From 7504e6bc13ab0531d783298329d40feb16af1026 Mon Sep 17 00:00:00 2001 From: abhimeda <138710508+abhimeda@users.noreply.github.com> Date: Thu, 30 May 2024 10:55:24 -0400 Subject: [PATCH] removing docker from external ci pipelines (#3177) * removed docker and pointed ROCm deps to our existing builds * removed vmImage tag for pool * added pip to apt list and renamed from rocFFT to hipFFT * fixed spelling mistakes in rocmDependencies * added correct apt dep for pip * removed leading slash in the cmake flags * changed cxx_compiler to /rocm/bin/hipcc * added llvm-project, ROCR-Runtime, clr, and rocminfo to rocm deps * added rocFFT as a rocm dependency * removed docker and added our builds for components * removed rocFFT from rocm deps * Fixed typo in rocFFT value * added rocprofiler-register to rocFFT and fixed typo in the dependencies-rocm file * changed cxx compiler to amdclang++ * fixed amdclang++ paths * moving to faster machine * added cmake module paths * switched back to medium build * added libopm-dev to apt deps * added libomp-14-dev to apt deps * added aomp as a rocm dep * added aomp as a rocm dep * added hipcc as the cxx_compiler * reverted back to clang++ as the cxx_compiler * removed unmentioned rocm deps from the readme * removed docker * added python3-pip as an apt dep * fixed compiler paths * added hipRAND as a rocm dep * added print statements to see directory structure * adding a print statement into /agent/_work/1/s/build/library * added -Tensile_rocm_assembler as a build flag * removed a broken script line * added D to tensile rocm assembler * added DROCM_PATH to build flags * fixed typo * changed build pool from medium to base * changed build pool from base to low * added env variables using josephs pr * removed docker from hipBLASLt and added rocm dependencies that point to our builds * added pip to the apt packages array * changed cmake_cxx_compiler env var ro amdclang++ * changed cmake_cxx_compiler env var to amdclang++ * changed cmake_cxx_compiler env var to hipcc * changed cmake_cxx_compiler env var to hipcc * changed clang to amdclang * changed all refs mentioning hipcc to amdclang * changed cmake_cxx_compiler back to hipcc * added a HIP_PATH env var based off Tensile/Source/FindHIP.cmake * added hipcc to HIP_PATH * added rocm-cmake to rocm deps * added rocRAND as a rocm dep * removed dcmake_module flag * added libomp-dev as an apt dep * added aomp as a rocm dep * added clang as an apt dep * reverted changes back to how they appear in develop since this branch will be submitted for review * removed unecessary flags * adding -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang back to see if these are vital to a successful build * removed newline character --- .azuredevops/components/hipBLASLt.yml | 2 +- .azuredevops/components/hipSPARSELt.yml | 45 +++++++++++++++++++++---- .azuredevops/components/rocFFT.yml | 39 ++++++++++++++++----- 3 files changed, 71 insertions(+), 15 deletions(-) diff --git a/.azuredevops/components/hipBLASLt.yml b/.azuredevops/components/hipBLASLt.yml index facb51dc2..61d4a70e1 100644 --- a/.azuredevops/components/hipBLASLt.yml +++ b/.azuredevops/components/hipBLASLt.yml @@ -51,4 +51,4 @@ jobs: -DTensile_LIBRARY_FORMAT=msgpack -DCMAKE_PREFIX_PATH="/opt/rocm" -GNinja - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml \ No newline at end of file diff --git a/.azuredevops/components/hipSPARSELt.yml b/.azuredevops/components/hipSPARSELt.yml index 0554aa775..e2576c571 100644 --- a/.azuredevops/components/hipSPARSELt.yml +++ b/.azuredevops/components/hipSPARSELt.yml @@ -12,21 +12,40 @@ parameters: - ninja-build - python3-venv - libmsgpack-dev - - hipsparse-dev - git + - python3-pip - name: pipModules type: object default: - joblib +# rocm dependencies should match dependencies-rocm.yml +- name: rocmDependencies + type: object + default: + - llvm-project + - ROCR-Runtime + - clr + - rocminfo + - rocprofiler-register + - hipSPARSE + - rocBLAS jobs: - job: hipSPARSELt variables: - group: common - template: /.azuredevops/variables-global.yml + - name: HIP_ROCCLR_HOME + value: $(Build.BinariesDirectory)/rocm + - name: TENSILE_ROCM_ASSEMBLER_PATH + value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang + - name: CMAKE_CXX_COMPILER + value: $(Agent.BuildDirectory)/rocm/llvm/bin/hipcc + - name: TENSILE_ROCM_OFFLOAD_BUNDLER_PATH + value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang-offload-bundler + - name: PATH + value: $(Agent.BuildDirectory)/rocm/llvm/bin:$(Agent.BuildDirectory)/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin pool: ${{ variables.MEDIUM_BUILD_POOL }} - container: - image: ${{ variables.DOCKER_IMAGE_NAME }}:${{ variables.LATEST_DOCKER_VERSION }} workspace: clean: all steps: @@ -38,17 +57,31 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml parameters: checkoutRepo: ${{ parameters.checkoutRepo }} +# CI case: download latest default branch build + - ${{ if eq(parameters.checkoutRef, '') }}: + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml + parameters: + dependencyList: ${{ parameters.rocmDependencies }} + dependencySource: staging +# manual build case: triggered by ROCm/ROCm repo + - ${{ if ne(parameters.checkoutRef, '') }}: + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml + parameters: + dependencyList: ${{ parameters.rocmDependencies }} + dependencySource: tag-builds + - script: pwd - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml parameters: extraBuildFlags: >- -DCMAKE_BUILD_TYPE=Release - -DCMAKE_CXX_COMPILER=/opt/rocm/bin/amdclang++ - -DCMAKE_C_COMPILER=/opt/rocm/bin/amdclang + -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ + -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang -DAMDGPU_TARGETS=all -DTensile_LOGIC= -DTensile_CPU_THREADS= -DTensile_CODE_OBJECT_VERSION=default -DTensile_LIBRARY_FORMAT=msgpack - -DCMAKE_PREFIX_PATH="/opt/rocm" + -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm" + -DROCM_PATH=$(Agent.BuildDirectory)/rocm -GNinja - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml diff --git a/.azuredevops/components/rocFFT.yml b/.azuredevops/components/rocFFT.yml index fcfb58ee3..4d16c0a56 100644 --- a/.azuredevops/components/rocFFT.yml +++ b/.azuredevops/components/rocFFT.yml @@ -10,20 +10,31 @@ parameters: default: - cmake - ninja-build - - rocrand - - hiprand - libboost-program-options-dev - libgtest-dev - libfftw3-dev - + - python3-pip +# rocm dependencies should match dependencies-rocm.yml +- name: rocmDependencies + type: object + default: + - llvm-project + - ROCR-Runtime + - clr + - rocminfo + - rocprofiler-register + - hipRAND + - rocRAND + - rocm-cmake + - aomp jobs: - job: rocFFT variables: - group: common - template: /.azuredevops/variables-global.yml + - name: HIP_ROCCLR_HOME + value: $(Build.BinariesDirectory)/rocm pool: ${{ variables.MEDIUM_BUILD_POOL }} - container: - image: ${{ variables.DOCKER_IMAGE_NAME }}:${{ variables.LATEST_DOCKER_VERSION }} workspace: clean: all steps: @@ -34,12 +45,24 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml parameters: checkoutRepo: ${{ parameters.checkoutRepo }} + # CI case: download latest default branch build + - ${{ if eq(parameters.checkoutRef, '') }}: + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml + parameters: + dependencyList: ${{ parameters.rocmDependencies }} + dependencySource: staging +# manual build case: triggered by ROCm/ROCm repo + - ${{ if ne(parameters.checkoutRef, '') }}: + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml + parameters: + dependencyList: ${{ parameters.rocmDependencies }} + dependencySource: tag-builds - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml parameters: extraBuildFlags: >- - -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/amdclang++ - -DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/amdclang - -DCMAKE_PREFIX_PATH=/opt/rocm + -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ + -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang + -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS=gfx1030;gfx1100 -DUSE_HIP_CLANG=ON