[Ex CI] consolidate artifact extraction and deletion in deps-rocm (#4961)

.azuredevops/components/ROCmValidationSuite.yml

@@ -86,8 +86,7 @@ jobs:
       value: $(Agent.BuildDirectory)/rocm
     - name: HIP_INC_DIR
       value: $(Agent.BuildDirectory)/rocm
-    pool:
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-      vmImage: ${{ variables.BASE_BUILD_POOL }}
     workspace:
       clean: all
     steps:

.azuredevops/components/Tensile.yml

@@ -163,6 +163,7 @@ jobs:
     workspace:
       clean: all
     steps:
+    - checkout: none
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
       parameters:
         aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/hipBLASLt.yml

@@ -245,6 +245,7 @@ jobs:
       workspace:
         clean: all
       steps:
+      - checkout: none
       - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
         parameters:
           aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/hipFFT.yml

@@ -156,6 +156,7 @@ jobs:
     workspace:
       clean: all
     steps:
+    - checkout: none
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
       parameters:
         aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/hipSOLVER.yml

@@ -70,8 +70,7 @@ jobs:
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
-    pool:
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-      vmImage: ${{ variables.BASE_BUILD_POOL }}
     workspace:
       clean: all
     steps:

.azuredevops/components/llvm-project.yml

@@ -67,7 +67,6 @@ jobs:
       parameters:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
-        skipLlvmSymlink: true
         aggregatePipeline: ${{ parameters.aggregatePipeline }}
         os: ${{ job.os }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml

.azuredevops/components/rocALUTION.yml

@@ -73,8 +73,7 @@ jobs:
     - template: /.azuredevops/variables-global.yml
     - name: HIP_ROCCLR_HOME
       value: $(Build.BinariesDirectory)/rocm
-    pool:
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-      vmImage: ${{ variables.BASE_BUILD_POOL }}
     workspace:
       clean: all
     steps:

.azuredevops/components/rocBLAS.yml

@@ -216,6 +216,7 @@ jobs:
       workspace:
         clean: all
       steps:
+      - checkout: none
       - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
         parameters:
           aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/rocFFT.yml

@@ -166,6 +166,7 @@ jobs:
     workspace:
       clean: all
     steps:
+    - checkout: none
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
       parameters:
         aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/rocSOLVER.yml

@@ -194,6 +194,7 @@ jobs:
       workspace:
         clean: all
       steps:
+      - checkout: none
       - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
         parameters:
           aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/rocminfo.yml

@@ -67,7 +67,6 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         aggregatePipeline: ${{ parameters.aggregatePipeline }}
-        skipLlvmSymlink: true
         os: ${{ job.os }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:

.azuredevops/nightly/pytorch.yml

@@ -407,7 +407,6 @@ jobs:
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
       dependencySource: staging
-      skipLlvmSymlink: true
 # get sources to run test scripts
   - task: Bash@3
     displayName: git clone upstream pytorch

.azuredevops/nightly/rocm-nightly.yml

@@ -119,7 +119,6 @@ jobs:
         dependencyList: ${{ parameters.rocmDependencies }}
         os: ${{ job.os }}
         gpuTarget: ${{ job.target }}
-        skipLibraryLinking: true
     - script: df -h
       displayName: System disk space after ROCm
     - script: du -sh $(Agent.BuildDirectory)/rocm

.azuredevops/templates/steps/artifact-download.yml

@@ -12,6 +12,9 @@ parameters:
 - name: fileFilter
   type: string
   default: ''
+- name: extractAndDeleteFiles
+  type: boolean
+  default: true
 # set to true if doing full build of ROCm stack
 # and dependencies are pulled from same pipeline
 - name: aggregatePipeline
@@ -37,16 +40,17 @@ steps:
         buildVersionToDownload: latest # aomp trigger lives in ROCm/ROCm, so cannot use ROCm/aomp branch names
       ${{ else }}:
         buildVersionToDownload: latestFromBranch
-- task: ExtractFiles@1
+- ${{ if eq(parameters.extractAndDeleteFiles, true) }}:
-  displayName: Extract ${{ parameters.componentName }}
+  - task: ExtractFiles@1
-  inputs:
+    displayName: Extract ${{ parameters.componentName }}
-    archiveFilePatterns: '$(Pipeline.Workspace)/d/**/*.tar.gz'
+    inputs:
-    destinationFolder: '$(Agent.BuildDirectory)/rocm'
+      archiveFilePatterns: '$(Pipeline.Workspace)/d/**/*.tar.gz'
-    cleanDestinationFolder: false
+      destinationFolder: '$(Agent.BuildDirectory)/rocm'
-    overwriteExistingFiles: true
+      cleanDestinationFolder: false
-- task: DeleteFiles@1
+      overwriteExistingFiles: true
-  displayName: Cleanup Compressed ${{ parameters.componentName }}
+  - task: DeleteFiles@1
-  inputs:
+    displayName: Clean up Compressed ${{ parameters.componentName }}
-    SourceFolder: '$(Pipeline.Workspace)/d'
+    inputs:
-    Contents: '**/*.tar.gz'
+      SourceFolder: '$(Pipeline.Workspace)/d'
-    RemoveDotFiles: true
+      Contents: '**/*.tar.gz'
+      RemoveDotFiles: true

.azuredevops/templates/steps/dependencies-rocm.yml

@@ -19,16 +19,6 @@ parameters:
 - name: gpuTarget
   type: string
   default: ''
-# set to true if you're calling this template file multiple files in same pipeline
-# only leave last call false to optimize sequence
-- name: skipLibraryLinking
-  type: boolean
-  default: false
-# set to true if llvm-project is not downloaded in a particular call
-# or if you just don't want the symlink
-- name: skipLlvmSymlink
-  type: boolean
-  default: false
 # set to true if dlopen calls for HIP libraries are causing failures
 # because they do not follow shared library symlink convention
 - name: setupHIPLibrarySymlinks
@@ -367,6 +357,7 @@ steps:
         componentName: ${{ split(dependency, ':')[0] }}
         pipelineId: ${{ parameters.componentVarList[split(dependency, ':')[0]].pipelineId }}
         aggregatePipeline: ${{ parameters.aggregatePipeline }}
+        extractAndDeleteFiles: false
         ${{ if parameters.componentVarList[split(dependency, ':')[0]].hasGpuTarget }}:
           fileFilter: "${{ split(dependency, ':')[1] }}*_${{ parameters.os }}_${{ parameters.gpuTarget }}"
         # dependencySource = staging
@@ -405,6 +396,7 @@ steps:
         componentName: ${{ dependency }}
         pipelineId: ${{ parameters.componentVarList[dependency].pipelineId }}
         aggregatePipeline: ${{ parameters.aggregatePipeline }}
+        extractAndDeleteFiles: false
         ${{ if parameters.componentVarList[dependency].hasGpuTarget }}:
           fileFilter: ${{ parameters.os }}_${{ parameters.gpuTarget }}
         ${{ else }}:
@@ -430,8 +422,20 @@ steps:
         # default = staging
         ${{ else }}:
           branchName: ${{ parameters.componentVarList[dependency].stagingBranch }}
-# Set link to redirect llvm folder
+- task: ExtractFiles@1
-- ${{ if eq(parameters.skipLlvmSymlink, false) }}:
+  displayName: Extract ROCm artifacts
+  inputs:
+    archiveFilePatterns: $(Pipeline.Workspace)/d/**/*.tar.gz
+    destinationFolder: $(Agent.BuildDirectory)/rocm
+    cleanDestinationFolder: false
+    overwriteExistingFiles: true
+- task: DeleteFiles@1
+  displayName: Clean up ROCm artifacts
+  inputs:
+    SourceFolder: $(Pipeline.Workspace)/d
+    Contents: '**/*.tar.gz'
+    RemoveDotFiles: true
+- ${{ if containsValue(parameters.dependencyList, 'llvm-project') }}:
   - task: Bash@3
     displayName: Symlink from rocm/llvm to rocm/lib/llvm
     inputs:
@@ -439,6 +443,7 @@ steps:
       script: |
         sudo mkdir -p $(Agent.BuildDirectory)/rocm/lib
         sudo ln -sr $(Agent.BuildDirectory)/rocm/llvm $(Agent.BuildDirectory)/rocm/lib/llvm
+        echo "Created symlink from rocm/llvm to rocm/lib/llvm"
   - task: Bash@3
     displayName: Symlink executables from rocm/llvm/bin to rocm/bin
     inputs:
@@ -446,7 +451,14 @@ steps:
       script: |
         for file in amdclang amdclang++ amdclang-cl amdclang-cpp amdflang amdlld aompcc mygpu mycpu offload-arch; do
           sudo ln -sr $(Agent.BuildDirectory)/rocm/llvm/bin/$file $(Agent.BuildDirectory)/rocm/bin/$file
+          echo "Created symlink from rocm/llvm/bin/$file to rocm/bin/$file"
         done
+- ${{ if containsValue(parameters.dependencyList, 'rocm-core') }}:
+  - task: Bash@3
+    displayName: Print rocm/.info/version
+    inputs:
+      targetType: inline
+      script: cat $(Agent.BuildDirectory)/rocm/.info/version
 # dlopen calls within a ctest or pytest sequence runs into issues when shared library symlink convention is not followed
 # the convention is as follows:
 # unversioned .so is a symlink to major version .so
@@ -483,17 +495,16 @@ steps:
   inputs:
     targetType: inline
     script: ls -la1R $(Agent.BuildDirectory)/rocm
-- ${{ if eq(parameters.skipLibraryLinking, false) }}:
+- task: Bash@3
-  - task: Bash@3
+  displayName: 'Link ROCm shared libraries'
-    displayName: 'Link ROCm shared libraries'
+  inputs:
-    inputs:
+    targetType: inline
-      targetType: inline
+    # OS ignores if the ROCm lib folder shows up more than once
-# OS ignores if the ROCm lib folder shows up more than once
+    script: |
-      script: |
+      echo $(Agent.BuildDirectory)/rocm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
-        echo $(Agent.BuildDirectory)/rocm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
+      echo $(Agent.BuildDirectory)/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
-        echo $(Agent.BuildDirectory)/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
+      echo $(Agent.BuildDirectory)/rocm/lib64 | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
-        echo $(Agent.BuildDirectory)/rocm/lib64 | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
+      echo $(Agent.BuildDirectory)/rocm/llvm/lib64 | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
-        echo $(Agent.BuildDirectory)/rocm/llvm/lib64 | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
+      sudo cat /etc/ld.so.conf.d/rocm-ci.conf
-        sudo cat /etc/ld.so.conf.d/rocm-ci.conf
+      sudo ldconfig -v
-        sudo ldconfig -v
+      ldconfig -p
-        ldconfig -p