removing docker from external ci pipelines (#3177)

* removed docker and pointed ROCm deps to our existing builds

* removed vmImage tag for pool

* added pip to apt list and renamed from rocFFT to hipFFT

* fixed spelling mistakes in rocmDependencies

* added correct apt dep for pip

* removed leading slash in the cmake flags

* changed cxx_compiler to /rocm/bin/hipcc

* added llvm-project, ROCR-Runtime, clr, and rocminfo to rocm deps

* added rocFFT as a rocm dependency

* removed docker and added our builds for components

* removed rocFFT from rocm deps

* Fixed typo in rocFFT value

* added rocprofiler-register to rocFFT and fixed typo in the dependencies-rocm file

* changed cxx compiler to amdclang++

* fixed amdclang++ paths

* moving to faster machine

* added cmake module paths

* switched back to medium build

* added libopm-dev to apt deps

* added libomp-14-dev to apt deps

* added aomp as a rocm dep

* added aomp as a rocm dep

* added hipcc as the cxx_compiler

* reverted back to clang++ as the cxx_compiler

* removed unmentioned rocm deps from the readme

* removed docker

* added python3-pip as an apt dep

* fixed compiler paths

* added hipRAND as a rocm dep

* added print statements to see directory structure

* adding a print statement into /agent/_work/1/s/build/library

* added -Tensile_rocm_assembler as a build flag

* removed a broken script line

* added D to tensile rocm assembler

* added DROCM_PATH to build flags

* fixed typo

* changed build pool from medium to base

* changed build pool from base to low

* added env variables using josephs pr

* removed docker from hipBLASLt and added rocm dependencies that point to our builds

* added pip to the apt packages array

* changed cmake_cxx_compiler env var ro amdclang++

* changed cmake_cxx_compiler env var to amdclang++

* changed cmake_cxx_compiler env var to hipcc

* changed cmake_cxx_compiler env var to hipcc

* changed clang to amdclang

* changed all refs mentioning hipcc to amdclang

* changed cmake_cxx_compiler back to hipcc

* added a HIP_PATH env var based off Tensile/Source/FindHIP.cmake

* added hipcc to HIP_PATH

* added rocm-cmake to rocm deps

* added rocRAND as a rocm dep

* removed dcmake_module flag

* added libomp-dev as an apt dep

* added aomp as a rocm dep

* added clang as an apt dep

* reverted changes back to how they appear in develop since this branch will be submitted for review

* removed unecessary flags

* adding -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang back to see if these are vital to a successful build

* removed newline character

.azuredevops/components/hipBLASLt.yml

@@ -51,4 +51,4 @@ jobs:
         -DTensile_LIBRARY_FORMAT=msgpack
         -DCMAKE_PREFIX_PATH="/opt/rocm"
         -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/hipSPARSELt.yml

@@ -12,21 +12,40 @@ parameters:
     - ninja-build
     - python3-venv
     - libmsgpack-dev
-    - hipsparse-dev
     - git
+    - python3-pip
 - name: pipModules
   type: object
   default:
     - joblib
+# rocm dependencies should match dependencies-rocm.yml
+- name: rocmDependencies
+  type: object
+  default:
+    - llvm-project
+    - ROCR-Runtime
+    - clr
+    - rocminfo
+    - rocprofiler-register
+    - hipSPARSE
+    - rocBLAS
 
 jobs:
 - job: hipSPARSELt
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
+  - name: HIP_ROCCLR_HOME
+    value: $(Build.BinariesDirectory)/rocm
+  - name: TENSILE_ROCM_ASSEMBLER_PATH
+    value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang
+  - name: CMAKE_CXX_COMPILER
+    value: $(Agent.BuildDirectory)/rocm/llvm/bin/hipcc
+  - name: TENSILE_ROCM_OFFLOAD_BUNDLER_PATH
+    value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang-offload-bundler
+  - name: PATH
+    value: $(Agent.BuildDirectory)/rocm/llvm/bin:$(Agent.BuildDirectory)/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  container:
-    image: ${{ variables.DOCKER_IMAGE_NAME }}:${{ variables.LATEST_DOCKER_VERSION }}
   workspace:
     clean: all
   steps:
@@ -38,17 +57,31 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+# CI case: download latest default branch build
+  - ${{ if eq(parameters.checkoutRef, '') }}:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
+        dependencySource: staging
+# manual build case: triggered by ROCm/ROCm repo
+  - ${{ if ne(parameters.checkoutRef, '') }}:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
+        dependencySource: tag-builds 
+  - script: pwd
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_CXX_COMPILER=/opt/rocm/bin/amdclang++
-        -DCMAKE_C_COMPILER=/opt/rocm/bin/amdclang
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
         -DAMDGPU_TARGETS=all
         -DTensile_LOGIC=
         -DTensile_CPU_THREADS=
         -DTensile_CODE_OBJECT_VERSION=default
         -DTensile_LIBRARY_FORMAT=msgpack
-        -DCMAKE_PREFIX_PATH="/opt/rocm"
+        -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/rocFFT.yml

@@ -10,20 +10,31 @@ parameters:
   default:
     - cmake
     - ninja-build
-    - rocrand
-    - hiprand
     - libboost-program-options-dev
     - libgtest-dev
     - libfftw3-dev
-
+    - python3-pip
+# rocm dependencies should match dependencies-rocm.yml
+- name: rocmDependencies
+  type: object
+  default:
+    - llvm-project
+    - ROCR-Runtime
+    - clr
+    - rocminfo
+    - rocprofiler-register
+    - hipRAND
+    - rocRAND
+    - rocm-cmake
+    - aomp
 jobs:
 - job: rocFFT
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
+  - name: HIP_ROCCLR_HOME
+    value: $(Build.BinariesDirectory)/rocm
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  container:
-    image: ${{ variables.DOCKER_IMAGE_NAME }}:${{ variables.LATEST_DOCKER_VERSION }}
   workspace:
     clean: all
   steps:
@@ -34,12 +45,24 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+  # CI case: download latest default branch build
+  - ${{ if eq(parameters.checkoutRef, '') }}:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
+        dependencySource: staging
+# manual build case: triggered by ROCm/ROCm repo
+  - ${{ if ne(parameters.checkoutRef, '') }}:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
+        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/amdclang++
-        -DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/amdclang
-        -DCMAKE_PREFIX_PATH=/opt/rocm
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
         -DAMDGPU_TARGETS=gfx1030;gfx1100
         -DUSE_HIP_CLANG=ON