Docs: remove system_debugging.md

* add initial draft

* add to toc and install page

* update wording

* improve documentation structure

* resturcture and expand content

* add to training section

* add to conf.py article_pages

* Update docs/how-to/rocm-for-ai/includes/system-health-benchmarks.rst

Co-authored-by: Leo Paoletti <164940351+lpaoletti@users.noreply.github.com>

* Update docs/how-to/rocm-for-ai/includes/system-health-benchmarks.rst

Co-authored-by: Leo Paoletti <164940351+lpaoletti@users.noreply.github.com>

* update wordlist.txt

* Update docs/how-to/rocm-for-ai/includes/system-health-benchmarks.rst

Co-authored-by: Leo Paoletti <164940351+lpaoletti@users.noreply.github.com>

* inference --> AI workloads

* udpate toc

* update article_pages in conf.py

* Update system validation notes in training docs

* fix links in prerequisite-system-validation

* wording

* add note

* consistency

* remove extra files

* fix links

* add links to training index page

---------

Co-authored-by: Leo Paoletti <164940351+lpaoletti@users.noreply.github.com>

.azuredevops/ci-builds/aomp-mainline.yml

@@ -8,29 +8,29 @@ resources:
     type: github
     endpoint: ROCm
     name: ROCm/aomp
-    ref: amd-mainline-open
+    ref: amd-mainline
   - repository: aomp-extras_repo
     type: github
     endpoint: ROCm
     name: ROCm/aomp-extras
-    ref: amd-mainline-open
+    ref: amd-mainline
   - repository: flang_repo
     type: github
     endpoint: ROCm
     name: ROCm/flang
-    ref: amd-mainline-open
+    ref: amd-mainline
   - repository: llvm-project_repo
     type: github
     endpoint: ROCm
     name: ROCm/llvm-project
-    ref: amd-mainline-open
+    ref: amd-mainline
   pipelines:
   - pipeline: rocr-runtime_pipeline
     source: \ROCR-Runtime
     trigger:
       branches:
         include:
-        - amd-master
+        - amd-mainline
 # this job will only be triggered after successful build sequence of llvm-project and ROCR-Runtime
 
 trigger: none

.azuredevops/ci-builds/aomp-staging.yml

@@ -8,17 +8,17 @@ resources:
     type: github
     endpoint: ROCm
     name: ROCm/aomp
-    ref: amd-staging
+    ref: aomp-dev
   - repository: aomp-extras_repo
     type: github
     endpoint: ROCm
     name: ROCm/aomp-extras
-    ref: amd-staging
+    ref: aomp-dev
   - repository: flang_repo
     type: github
     endpoint: ROCm
     name: ROCm/flang
-    ref: amd-staging
+    ref: aomp-dev
   - repository: llvm-project_repo
     type: github
     endpoint: ROCm

.azuredevops/components/AMDMIGraphX.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -27,12 +32,12 @@ parameters:
   type: object
   default:
     - https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
-    - onnx>=1.14.1
+    - "onnx>=1.14.1"
-    - numpy>=1.21.6
+    - "numpy>=1.21.6"
-    - typing>=3.7.4
+    - "typing>=3.7.4"
-    - pytest>=6.0.1
+    - "pytest>=6.0.1"
-    - packaging>=23.0
+    - "packaging>=23.0"
-    - protobuf>=3.20.2
+    - "protobuf>=3.20.2"
 - name: rocmDependencies
   type: object
   default:
@@ -72,124 +77,137 @@ parameters:
     - rocprofiler-register
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: AMDMIGraphX
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-# half version should be fixed to 5.6.0
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-    parameters:
-      buildType: specific
-      definitionId: ${{ variables.HALF560_PIPELINE_ID }}
-      buildId: ${{ variables.HALF560_BUILD_ID }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_BUILD_TYPE=Release
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
-        -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
-        -DBUILD_TESTING=ON
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: AMDMIGraphX_testing
+jobs:
-  dependsOn: AMDMIGraphX
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: AMDMIGraphX_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  - name: MIGRAPHX_TRACE_BENCHMARKING
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-    value: 1
+    workspace:
-  pool: $(JOB_TEST_POOL)
+      clean: all
-  workspace:
+    steps:
-    clean: all
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  strategy:
+      parameters:
-    matrix:
+        aptPackages: ${{ parameters.aptPackages }}
-      gfx942:
+        pipModules: ${{ parameters.pipModules }}
-        JOB_GPU_TARGET: gfx942
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  steps:
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+  # half version should be fixed to 5.6.0
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-      pipModules: ${{ parameters.pipModules }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        buildType: specific
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+        definitionId: ${{ variables.HALF560_PIPELINE_ID }}
-    parameters:
+        buildId: ${{ variables.HALF560_BUILD_ID }}
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+      parameters:
-# half version should be fixed to 5.6.0
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      buildType: specific
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      definitionId: ${{ variables.HALF560_PIPELINE_ID }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      buildId: ${{ variables.HALF560_BUILD_ID }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        extraBuildFlags: >-
-    parameters:
+          -DCMAKE_BUILD_TYPE=Release
-      checkoutRef: ${{ parameters.checkoutRef }}
+          -DGPU_TARGETS=${{ job.target }}
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+          -DAMDGPU_TARGETS=${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-  - task: CMake@1
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
-    displayName: MIGraphXTest CMake Flags
+          -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
-    inputs:
+          -DBUILD_TESTING=ON
-      cmakeArgs: >-
+          -GNinja
-        -DCMAKE_BUILD_TYPE=Release
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+      parameters:
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
-        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
+      parameters:
-        -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
+        gpuTarget: ${{ job.target }}
-        -DBUILD_TESTING=ON
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-        -DMIGRAPHX_ENABLE_C_API_TEST=ON
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-        ..
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+        pipModules: ${{ parameters.pipModules }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      componentName: AMDMIGraphX
+
-      testExecutable: make
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-      testParameters: -j$(nproc) check
+  - job: AMDMIGraphX_test_${{ job.target }}
-      testPublishResults: false
+    dependsOn: AMDMIGraphX_build_${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    condition:
-    parameters:
+      and(succeeded(),
-      aptPackages: ${{ parameters.aptPackages }}
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-      pipModules: ${{ parameters.pipModules }}
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-      environment: test
+        eq(${{ parameters.aggregatePipeline }}, False)
-      gpuTarget: $(JOB_GPU_TARGET)
+      )
-      extraEnvVars:
+    variables:
-        - MIGRAPHX_TRACE_BENCHMARKING:::1
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    - name: MIGRAPHX_TRACE_BENCHMARKING
+      value: 1
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+      parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  # half version should be fixed to 5.6.0
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        buildType: specific
+        definitionId: ${{ variables.HALF560_PIPELINE_ID }}
+        buildId: ${{ variables.HALF560_BUILD_ID }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - task: CMake@1
+      displayName: MIGraphXTest CMake Flags
+      inputs:
+        cmakeArgs: >-
+          -DCMAKE_BUILD_TYPE=Release
+          -DGPU_TARGETS=${{ job.target }}
+          -DAMDGPU_TARGETS=${{ job.target }}
+          -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
+          -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
+          -DBUILD_TESTING=ON
+          -DMIGRAPHX_ENABLE_C_API_TEST=ON
+          ..
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: AMDMIGraphX
+        testExecutable: make
+        testParameters: -j$(nproc) check
+        testPublishResults: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}
+        extraEnvVars:
+          - MIGRAPHX_TRACE_BENCHMARKING:::1

.azuredevops/components/HIP.yml

@@ -5,12 +5,18 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
     - cmake
     - libnuma-dev
     - mesa-common-dev
+    - ninja-build
     - ocl-icd-libopencl1
     - ocl-icd-opencl-dev
     - opencl-headers
@@ -41,7 +47,8 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -65,11 +72,13 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependenciesAMD }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
 # compile clr
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: clr
-      cmakeBuildDir: 'clr/build'
+      cmakeBuildDir: '$(Build.SourcesDirectory)/clr/build'
+      cmakeSourceDir: '$(Build.SourcesDirectory)/clr'
       extraBuildFlags: >-
         -DHIP_COMMON_DIR=$(Build.SourcesDirectory)/HIP
         -DHIP_PLATFORM=amd
@@ -78,6 +87,7 @@ jobs:
         -DHIPCC_BIN_DIR=$(Agent.BuildDirectory)/rocm/bin
         -DCLR_BUILD_HIP=ON
         -DCLR_BUILD_OCL=ON
+        -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
     parameters:
       artifactName: amd
@@ -85,18 +95,19 @@ jobs:
     parameters:
       artifactName: amd
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
+  #     pipModules: ${{ parameters.pipModules }}
-      environment: amd
+  #     environment: amd
 
 # HIP with Nvidia backend
 - job: hip_clr_combined_nvidia
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -121,13 +132,15 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependenciesNvidia }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - script: 'ls -1R $(Agent.BuildDirectory)/rocm'
     displayName: 'Artifact listing'
 # compile clr
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: clr
-      cmakeBuildDir: 'clr/build'
+      cmakeBuildDir: '$(Build.SourcesDirectory)/clr/build'
+      cmakeSourceDir: '$(Build.SourcesDirectory)/clr'
       extraBuildFlags: >-
         -DHIP_COMMON_DIR=$(Build.SourcesDirectory)/HIP
         -DHIP_PLATFORM=nvidia
@@ -135,12 +148,13 @@ jobs:
         -DCLR_BUILD_HIP=ON
         -DCLR_BUILD_OCL=OFF
         -DHIPNV_DIR=$(Build.SourcesDirectory)/hipother/hipnv
+        -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: nvidia
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
+  #     pipModules: ${{ parameters.pipModules }}
-      environment: nvidia
+  #     environment: nvidia

.azuredevops/components/HIPIFY.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -68,6 +73,7 @@ jobs:
     parameters:
       componentName: upstream-llvm
       cmakeBuildDir: $(Pipeline.Workspace)/llvm-project/llvm/build
+      cmakeSourceDir: $(Pipeline.Workspace)/llvm-project/llvm
       installDir: $(Pipeline.Workspace)/llvm
       extraBuildFlags: >-
         -DCMAKE_BUILD_TYPE=Release

.azuredevops/components/MIOpen.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -55,151 +60,166 @@ parameters:
     - rocRAND
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: MIOpen
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - task: Bash@3
-    displayName: Build and install other dependencies
-    inputs:
-      targetType: inline
-      workingDirectory: $(Build.SourcesDirectory)
-      script: |
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-        sed -i '/composable_kernel/d' requirements.txt
-        mkdir -p $(Agent.BuildDirectory)/miopen-deps
-        cmake -P install_deps.cmake --prefix $(Agent.BuildDirectory)/miopen-deps
-        sudo rm -rf /opt/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DMIOPEN_BACKEND=HIP
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/miopen-deps
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DMIOPEN_ENABLE_AI_KERNEL_TUNING=OFF
-        -DMIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK=OFF
-        -DCMAKE_BUILD_TYPE=Release
-        -DBUILD_TESTING=ON
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraCopyDirectories:
-        - miopen-deps
 
-- job: MIOpen_testing
+jobs:
-  timeoutInMinutes: 180
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  dependsOn: MIOpen
+  - job: MIOpen_build_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    variables:
-  variables:
+    - group: common
-  - group: common
+    - template: /.azuredevops/variables-global.yml
-  - template: /.azuredevops/variables-global.yml
+    - name: ROCM_PATH
-  - name: ROCM_PATH
+      value: $(Agent.BuildDirectory)/rocm
-    value: $(Agent.BuildDirectory)/rocm
+    pool: ${{ variables.HIGH_BUILD_POOL }}
-  pool: $(JOB_TEST_POOL)
+    workspace:
-  workspace:
+      clean: all
-    clean: all
+    steps:
-  strategy:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    matrix:
+      parameters:
-      gfx942:
+        aptPackages: ${{ parameters.aptPackages }}
-        JOB_GPU_TARGET: gfx942
+        pipModules: ${{ parameters.pipModules }}
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
-    parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml
-      pipModules: ${{ parameters.pipModules }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
+      parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    - task: Bash@3
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      displayName: Build and install other dependencies
-    parameters:
+      inputs:
-      checkoutRef: ${{ parameters.checkoutRef }}
+        targetType: inline
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        workingDirectory: $(Build.SourcesDirectory)
-      gpuTarget: $(JOB_GPU_TARGET)
+        script: |
-  - task: Bash@3
+          sed -i '/composable_kernel/d' requirements.txt
-    displayName: Build and install other dependencies
+          mkdir -p $(Agent.BuildDirectory)/miopen-deps
-    inputs:
+          export CXX=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-      targetType: inline
+          export CC=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-      workingDirectory: $(Build.SourcesDirectory)
+          cmake -P install_deps.cmake --prefix $(Agent.BuildDirectory)/miopen-deps --generator Ninja
-      script: |
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      parameters:
-        sed -i '/composable_kernel/d' requirements.txt
+        extraBuildFlags: >-
-        mkdir -p $(Agent.BuildDirectory)/miopen-deps
+          -DMIOPEN_BACKEND=HIP
-        cmake -P install_deps.cmake --prefix $(Agent.BuildDirectory)/miopen-deps
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        sudo rm -rf /opt/rocm
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/miopen-deps
-  - task: CMake@1
+          -DGPU_TARGETS=${{ job.target }}
-    displayName: 'MIOpen Test CMake Flags'
+          -DMIOPEN_ENABLE_AI_KERNEL_TUNING=OFF
-    inputs:
+          -DMIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK=OFF
-      cmakeArgs: >-
+          -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Build.SourcesDirectory)/bin;$(Agent.BuildDirectory)/miopen-deps
+          -DBUILD_TESTING=ON
-        -DCMAKE_INSTALL_PREFIX=$(Agent.BuildDirectory)/rocm
+          -GNinja
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+      parameters:
-        -DMIOPEN_BACKEND=HIP
+        gpuTarget: ${{ job.target }}
-        -DMIOPEN_TEST_FLAGS=" --disable-verification-cache"
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-        -DCMAKE_BUILD_TYPE=release
+      parameters:
-        -DBUILD_DEV=OFF
+        gpuTarget: ${{ job.target }}
-        -DMIOPEN_USE_MLIR=ON
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-        -DMIOPEN_GPU_SYNC=OFF
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-        ..
+      parameters:
-  - task: Bash@3
+        aptPackages: ${{ parameters.aptPackages }}
-    displayName: 'MIOpen Test Build'
+        pipModules: ${{ parameters.pipModules }}
-    inputs:
+        gpuTarget: ${{ job.target }}
-      targetType: inline
+        extraCopyDirectories:
-      script: |
+          - miopen-deps
-        cmake --build . --target tests -- -j$(nproc)
+
-      workingDirectory: $(Build.SourcesDirectory)/build
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - job: MIOpen_test_${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    timeoutInMinutes: 180
-    parameters:
+    dependsOn: MIOpen_build_${{ job.target }}
-      componentName: MIOpen
+    condition:
-      testParameters: '-VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml --exclude-regex test_rnn_seq_api'
+      and(succeeded(),
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-    parameters:
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-      aptPackages: ${{ parameters.aptPackages }}
+        eq(${{ parameters.aggregatePipeline }}, False)
-      pipModules: ${{ parameters.pipModules }}
+      )
-      environment: test
+    variables:
-      gpuTarget: $(JOB_GPU_TARGET)
+    - group: common
-      extraCopyDirectories:
+    - template: /.azuredevops/variables-global.yml
-        - miopen-deps
+    - name: ROCM_PATH
+      value: $(Agent.BuildDirectory)/rocm
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+      parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - task: Bash@3
+      displayName: Build and install other dependencies
+      inputs:
+        targetType: inline
+        workingDirectory: $(Build.SourcesDirectory)
+        script: |
+          sed -i '/composable_kernel/d' requirements.txt
+          mkdir -p $(Agent.BuildDirectory)/miopen-deps
+          export CXX=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+          export CC=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+          cmake -P install_deps.cmake --prefix $(Agent.BuildDirectory)/miopen-deps
+    - task: CMake@1
+      displayName: 'MIOpen Test CMake Flags'
+      inputs:
+        cmakeArgs: >-
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Build.SourcesDirectory)/bin;$(Agent.BuildDirectory)/miopen-deps
+          -DCMAKE_INSTALL_PREFIX=$(Agent.BuildDirectory)/rocm
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+          -DMIOPEN_BACKEND=HIP
+          -DMIOPEN_TEST_FLAGS=" --disable-verification-cache"
+          -DCMAKE_BUILD_TYPE=release
+          -DBUILD_DEV=OFF
+          -DMIOPEN_USE_MLIR=ON
+          -DMIOPEN_GPU_SYNC=OFF
+          ..
+    - task: Bash@3
+      displayName: 'MIOpen Test Build'
+      inputs:
+        targetType: inline
+        script: |
+          cmake --build . --target tests -- -j$(nproc)
+        workingDirectory: $(Build.SourcesDirectory)/build
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: MIOpen
+        testParameters: '--output-on-failure --force-new-ctest-process --output-junit test_output.xml --exclude-regex "test_rnn_seq_api|GPU_Conv2dTuningAsm_FP32"'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}
+        extraCopyDirectories:
+          - miopen-deps

.azuredevops/components/MIVisionX.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -65,97 +70,115 @@ parameters:
     - roctracer
     - rpp
 
-jobs:
+- name: jobMatrix
-- job: MIVisionX
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_BUILD_TYPE=Release
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DROCM_DEP_ROCMCORE=ON
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: MIVisionX_testing
+jobs:
-  dependsOn: MIVisionX
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: MIVisionX_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    pool:
-  workspace:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    clean: all
+    workspace:
-  strategy:
+      clean: all
-    matrix:
+    steps:
-      gfx942:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-        JOB_GPU_TARGET: gfx942
+      parameters:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+        aptPackages: ${{ parameters.aptPackages }}
-  steps:
+        pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        extraBuildFlags: >-
-  # anything in /opt may be persistent across runs
+          -DCMAKE_BUILD_TYPE=Release
-  # so we need to remove the symlink if it already exists
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-  - script: |
+          -DROCM_DEP_ROCMCORE=ON
-      sudo rm -f /opt/rocm
+          -GNinja
-      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      mkdir mivisionx-tests
+      parameters:
-      cd mivisionx-tests
+        gpuTarget: ${{ job.target }}
-      cmake /opt/rocm/share/mivisionx/test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-      componentName: MIVisionX
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      testDir: 'mivisionx-tests'
+    #   parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    #     aptPackages: ${{ parameters.aptPackages }}
-    parameters:
+    #     pipModules: ${{ parameters.pipModules }}
-      aptPackages: ${{ parameters.aptPackages }}
+    #     gpuTarget: ${{ job.target }}
-      pipModules: ${{ parameters.pipModules }}
+
-      environment: test
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-      gpuTarget: $(JOB_GPU_TARGET)
+  - job: MIVisionX_test_${{ job.target }}
-      optSymLink: true
+    dependsOn: MIVisionX_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    - name: LD_LIBRARY_PATH
+      value: $(Agent.BuildDirectory)/rocm/lib:$(Agent.BuildDirectory)/rocm/include/mivisionx/VX
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - task: Bash@3
+      displayName: Build MIVisionX tests
+      inputs:
+        targetType: inline
+        script: |
+          # Assuming that /opt is no longer persistent across runs, test environments are fully ephemeral
+          sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+          mkdir mivisionx-tests
+          cd mivisionx-tests
+          cmake $(Agent.BuildDirectory)/rocm/share/mivisionx/test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: MIVisionX
+        testDir: 'mivisionx-tests'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}
+        optSymLink: true

.azuredevops/components/ROCR-Runtime.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -13,6 +18,7 @@ parameters:
     - libdrm-dev
     - libelf-dev
     - libnuma-dev
+    - ninja-build
     - pkg-config
     - python3-pip
 - name: rocmDependencies
@@ -27,12 +33,22 @@ parameters:
     - rocm_smi_lib
     - rocprofiler-register
 
+- name: jobMatrix
+  type: object
+  default:
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 jobs:
-- job: ROCR_Runtime
+- job: ROCR_Runtime_build
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -47,100 +63,105 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_SHARED_LIBS=ON
         -DCMAKE_BUILD_TYPE=Release
+        -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
 
-- job: ROCR_Runtime_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  dependsOn: ROCR_Runtime
+  - job: ROCR_Runtime_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: ROCR_Runtime_build
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool: $(JOB_TEST_POOL)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  workspace:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    clean: all
+      )
-  strategy:
+    variables:
-    matrix:
+    - group: common
-      gfx942:
+    - template: /.azuredevops/variables-global.yml
-        JOB_GPU_TARGET: gfx942
+    pool: ${{ job.target }}_test_pool
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    workspace:
-  steps:
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    steps:
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-  - task: Bash@3
+        aptPackages: ${{ parameters.aptPackages }}
-    displayName: Install libhwloc5
+    - task: Bash@3
-    inputs:
+      displayName: Install libhwloc5
-      targetType: 'inline'
+      inputs:
-      script: |
+        targetType: 'inline'
-        wget http://ftp.us.debian.org/debian/pool/main/h/hwloc/libhwloc5_1.11.12-3_amd64.deb
+        script: |
-        wget http://ftp.us.debian.org/debian/pool/main/h/hwloc/libhwloc-dev_1.11.12-3_amd64.deb
+          wget http://ftp.us.debian.org/debian/pool/main/h/hwloc/libhwloc5_1.11.12-3_amd64.deb
-        sudo apt install -y --allow-downgrades ./libhwloc5_1.11.12-3_amd64.deb ./libhwloc-dev_1.11.12-3_amd64.deb
+          wget http://ftp.us.debian.org/debian/pool/main/h/hwloc/libhwloc-dev_1.11.12-3_amd64.deb
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+          sudo apt install -y --allow-downgrades ./libhwloc5_1.11.12-3_amd64.deb ./libhwloc-dev_1.11.12-3_amd64.deb
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmTestDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-      runRocminfo: false
+      parameters:
-  - task: Bash@3
+        runRocminfo: false
-    displayName: Build kfdtest
+    - task: Bash@3
-    continueOnError: true
+      displayName: Build kfdtest
-    inputs:
+      continueOnError: true
-      targetType: 'inline'
+      inputs:
-      workingDirectory: $(Build.SourcesDirectory)/libhsakmt/tests/kfdtest
+        targetType: 'inline'
-      script: |
+        workingDirectory: $(Build.SourcesDirectory)/libhsakmt/tests/kfdtest
-        mkdir build && cd build
+        script: |
-        cmake -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm ..
+          mkdir build && cd build
-        make
+          cmake -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm ..
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          make
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-      componentName: kfdtest
+      parameters:
-      testExecutable: BIN_DIR=$(Build.SourcesDirectory)/libhsakmt/tests/kfdtest/build ./run_kfdtest.sh
+        componentName: kfdtest
-      testParameters: '-p core --gtest_output=xml:./test_output.xml --gtest_color=yes'
+        testExecutable: BIN_DIR=$(Build.SourcesDirectory)/libhsakmt/tests/kfdtest/build ./run_kfdtest.sh
-      testDir: $(Build.SourcesDirectory)/libhsakmt/tests/kfdtest/scripts
+        testParameters: '-p core --gtest_output=xml:./test_output.xml --gtest_color=yes'
-  - task: Bash@3
+        testDir: $(Build.SourcesDirectory)/libhsakmt/tests/kfdtest/scripts
-    displayName: Build rocrtst
+    - task: Bash@3
-    continueOnError: true
+      displayName: Build rocrtst
-    inputs:
+      continueOnError: true
-      targetType: 'inline'
+      inputs:
-      workingDirectory: $(Build.SourcesDirectory)/rocrtst/suites/test_common
+        targetType: 'inline'
-      script: |
+        workingDirectory: $(Build.SourcesDirectory)/rocrtst/suites/test_common
-        mkdir build && cd build
+        script: |
-        cmake .. \
+          BASE_CLANG_DIR=$(Agent.BuildDirectory)/rocm/llvm/lib/clang
-          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm \
+          export NEWEST_CLANG_VER=$(ls -1 $BASE_CLANG_DIR | sort -V | tail -n 1)
-          -DTARGET_DEVICES=$(JOB_GPU_TARGET) \
+          mkdir build && cd build
-          -DROCM_DIR=$(Agent.BuildDirectory)/rocm \
+          cmake .. \
-          -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm/bin \
+            -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm \
-          -DOPENCL_INC_DIR=$(Agent.BuildDirectory)/rocm/llvm/lib/clang/21/include
+            -DTARGET_DEVICES=${{ job.target }} \
-        make
+            -DROCM_DIR=$(Agent.BuildDirectory)/rocm \
-        make rocrtst_kernels
+            -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm/bin \
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+            -DOPENCL_INC_DIR=$BASE_CLANG_DIR/$NEWEST_CLANG_VER/include
-    parameters:
+          make
-      componentName: rocrtst
+          make rocrtst_kernels
-      testExecutable: ./rocrtst64
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-      testParameters: '--gtest_filter="-rocrtstNeg.Memory_Negative_Tests:rocrtstFunc.Memory_Max_Mem" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+      parameters:
-      testDir: $(Build.SourcesDirectory)/rocrtst/suites/test_common/build/$(JOB_GPU_TARGET)
+        componentName: rocrtst
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        testExecutable: ./rocrtst64
-    parameters:
+        testParameters: '--gtest_filter="-rocrtstNeg.Memory_Negative_Tests:rocrtstFunc.Memory_Max_Mem" --gtest_output=xml:./test_output.xml --gtest_color=yes'
-      aptPackages: ${{ parameters.aptPackages }}
+        testDir: $(Build.SourcesDirectory)/rocrtst/suites/test_common/build/${{ job.target }}
-      environment: test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-# docker image will be missing libhwloc5
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}
+  # docker image will be missing libhwloc5

.azuredevops/components/ROCdbgapi.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -24,7 +29,8 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -39,6 +45,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -48,6 +55,6 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/ROCgdb.yml

@@ -5,11 +5,17 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 # reference: https://github.com/ROCm/ROCgdb/blob/amd-staging/README-ROCM.md
 - name: aptPackages
   type: object
   default:
     - bison
+    - cmake
     - dejagnu
     - flex
     - libbabeltrace-dev
@@ -17,6 +23,7 @@ parameters:
     - libgmp-dev
     - liblzma-dev
     - libmpfr-dev
+    - pkg-config
     - ncurses-dev
     - texinfo
     - zlib1g-dev
@@ -30,22 +37,26 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
+- name: jobMatrix
+  type: object
+  default:
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 jobs:
 - job: ROCgdb
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
   - name: PKG_CONFIG_PATH
     value: $(Agent.BuildDirectory)/rocm/share/pkgconfig
-  pool: $(JOB_TEST_POOL)
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
-  strategy:
-    matrix:
-      gfx942:
-        JOB_GPU_TARGET: gfx942
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -59,6 +70,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-autotools.yml
     parameters:
       configureFlags: >-
@@ -84,33 +96,88 @@ jobs:
       makeCallPrefix: LD_RUN_PATH='${ORIGIN}/../lib'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-  - task: Bash@3
-    displayName: Setup test environment
-    inputs:
-      targetType: inline
-      script: |
-        sudo rm -rf /opt/rocm
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-        echo "##vso[task.prependpath]/opt/rocm/bin"
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+
-  - task: Bash@3
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-    displayName: check-gdb
+  - job: ROCgdb_test_${{ job.target }}
-    continueOnError: true
+    dependsOn: ROCgdb
-    inputs:
+    condition:
-      targetType: inline
+      and(succeeded(),
-      script: make check-gdb TESTS=gdb.rocm/simple.exp
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-      workingDirectory: $(Build.SourcesDirectory)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  - task: Bash@3
+        eq(${{ parameters.aggregatePipeline }}, False)
-    displayName: print gdb log
+      )
-    inputs:
+    variables:
-      targetType: inline
+    - group: common
-      script: find -name gdb.log -exec cat {} \;
+    - template: /.azuredevops/variables-global.yml
-      workingDirectory: $(Build.SourcesDirectory)
+    - name: PKG_CONFIG_PATH
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      value: $(Agent.BuildDirectory)/rocm/share/pkgconfig
-    parameters:
+    pool: ${{ job.target }}_test_pool
-      aptPackages: ${{ parameters.aptPackages }}
+    workspace:
-      environment: combined
+      clean: all
-      gpuTarget: $(JOB_GPU_TARGET)
+    steps:
-      extraEnvVars:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-        - PKG_CONFIG_PATH:::/home/user/workspace/rocm/share/pkgconfig
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+      parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmDependencies }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-autotools.yml
+      parameters:
+        configureFlags: >-
+          --program-prefix=roc
+          --enable-64-bit-bfd
+          --enable-targets="x86_64-linux-gnu,amdgcn-amd-amdhsa"
+          --disable-ld
+          --disable-gas
+          --disable-gdbserver
+          --disable-sim
+          --enable-tui
+          --disable-gdbtk
+          --disable-shared
+          --disable-gprofng
+          --with-expat
+          --with-system-zlib
+          --without-guile
+          --with-babeltrace
+          --with-lzma
+          --with-python=python3
+          --with-rocm-dbgapi=$(Agent.BuildDirectory)/rocm
+          LDFLAGS="-Wl,--enable-new-dtags,-rpath=$(Agent.BuildDirectory)/rocm/lib"
+        makeCallPrefix: LD_RUN_PATH='${ORIGIN}/../lib'
+    - task: Bash@3
+      displayName: Setup test environment
+      inputs:
+        targetType: inline
+        script: |
+          # Assuming that /opt is no longer persistent across runs, test environments are fully ephemeral
+          sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+          echo "##vso[task.prependpath]/opt/rocm/bin"
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - task: Bash@3
+      displayName: check-gdb
+      continueOnError: true
+      inputs:
+        targetType: inline
+        script: make check-gdb TESTS=gdb.rocm/simple.exp
+        workingDirectory: $(Build.SourcesDirectory)
+    - task: Bash@3
+      displayName: print gdb log
+      inputs:
+        targetType: inline
+        script: find -name gdb.log -exec cat {} \;
+        workingDirectory: $(Build.SourcesDirectory)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: combined
+        gpuTarget: ${{ job.target }}
+        extraEnvVars:
+          - PKG_CONFIG_PATH:::/home/user/workspace/rocm/share/pkgconfig

.azuredevops/components/ROCmValidationSuite.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -21,6 +26,8 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
+    - amdsmi
+    - aomp
     - clr
     - hipBLAS-common
     - hipBLASLt
@@ -36,6 +43,8 @@ parameters:
 - name: rocmTestDependencies
   type: object
   default:
+    - amdsmi
+    - aomp
     - clr
     - hipBLAS-common
     - hipBLASLt
@@ -49,99 +58,116 @@ parameters:
     - rocRAND
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: ROCmValidationSuite
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  - name: HIP_ROCCLR_HOME
+        target: gfx942
-    value: $(Build.BinariesDirectory)/rocm
+      - gfx90a:
-  - name: ROCM_PATH
+        target: gfx90a
-    value: $(Agent.BuildDirectory)/rocm
+    testJobs:
-  - name: HIP_INC_DIR
+      - gfx942:
-    value: $(Agent.BuildDirectory)/rocm
+        target: gfx942
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        confDir: MI300X
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+        confDir: MI210
-    matrix:
-      gfx942:
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DCPACK_PACKAGING_INSTALL_PREFIX=$(Build.BinariesDirectory)
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
-        - ROCM_PATH:::/home/user/workspace/rocm
-        - HIP_INC_DIR:::/home/user/workspace/rocm
 
-- job: ROCmValidationSuite_testing
+jobs:
-  dependsOn: ROCmValidationSuite
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: ROCmValidationSuite_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    - name: HIP_ROCCLR_HOME
-  workspace:
+      value: $(Build.BinariesDirectory)/rocm
-    clean: all
+    - name: ROCM_PATH
-  strategy:
+      value: $(Agent.BuildDirectory)/rocm
-    matrix:
+    - name: HIP_INC_DIR
-      gfx942:
+      value: $(Agent.BuildDirectory)/rocm
-        JOB_GPU_TARGET: gfx942
+    pool:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-  steps:
+    workspace:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      clean: all
-    parameters:
+    steps:
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        aptPackages: ${{ parameters.aptPackages }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      componentName: ROCmValidationSuite
+      parameters:
-      testExecutable: $(Agent.BuildDirectory)/rocm/bin/rvs -c $(Agent.BuildDirectory)/rocm/share/rocm-validation-suite/conf/MI300X/gst_single.conf
+        extraBuildFlags: >-
-      testParameters: ''
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-      testDir: $(Agent.BuildDirectory)
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-      testPublishResults: false
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -DCMAKE_CXX_FLAGS=-I$(Agent.BuildDirectory)/rocm/llvm/include
-    parameters:
+          -DCPACK_PACKAGING_INSTALL_PREFIX=$(Build.BinariesDirectory)
-      aptPackages: ${{ parameters.aptPackages }}
+          -GNinja
-      environment: test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    #   parameters:
+    #     aptPackages: ${{ parameters.aptPackages }}
+    #     gpuTarget: ${{ job.target }}
+    #     extraEnvVars:
+    #       - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+    #       - ROCM_PATH:::/home/user/workspace/rocm
+    #       - HIP_INC_DIR:::/home/user/workspace/rocm
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: ROCmValidationSuite_test_${{ job.target }}
+    dependsOn: ROCmValidationSuite_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: ROCmValidationSuite
+        testExecutable: $(Agent.BuildDirectory)/rocm/bin/rvs -c $(Agent.BuildDirectory)/rocm/share/rocm-validation-suite/conf/${{ job.confDir }}/gst_single.conf
+        testParameters: ''
+        testDir: $(Agent.BuildDirectory)
+        testPublishResults: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/Tensile.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -30,12 +35,24 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
+- name: jobMatrix
+  type: object
+  default:
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 jobs:
-- job: Tensile
+- job: Tensile_build
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  - name: ROCM_PATH
+    value: $(Agent.BuildDirectory)/rocm
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -51,6 +68,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - task: Bash@3
     displayName: Create wheel file
@@ -58,8 +76,6 @@ jobs:
       targetType: inline
       script: python3 setup.py bdist_wheel
       workingDirectory: $(Build.SourcesDirectory)
-    env:
-      ROCM_PATH: $(Agent.BuildDirectory)/rocm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
       sourceDir: $(Build.SourcesDirectory)/dist
@@ -77,114 +93,84 @@ jobs:
       workingDirectory: $(Pipeline.Workspace)
       targetType: inline
       script: |
-        echo "$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_drop_$(JOB_GPU_TARGET).tar.gz" >> pipelineArtifacts.txt
         whlFile=$(find "$(Build.ArtifactStagingDirectory)" -type f -name "*.whl" | head -n 1)
         if [ -n "$whlFile" ]; then
           echo $(basename "$whlFile") >> pipelineArtifacts.txt
         fi
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
+  #     pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: Tensile_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  timeoutInMinutes: 90
+  - job: Tensile_test_${{ job.target }}
-  dependsOn: Tensile
+    timeoutInMinutes: 180
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: Tensile_build
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool: $(JOB_TEST_POOL)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  workspace:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    clean: all
+      )
-  strategy:
+    variables:
-    matrix:
+    - group: common
-      gfx942:
+    - template: /.azuredevops/variables-global.yml
-        JOB_GPU_TARGET: gfx942
+    - name: ROCM_PATH
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+      value: $(Agent.BuildDirectory)/rocm
-  steps:
+    pool: ${{ job.target }}_test_pool
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    workspace:
-    parameters:
+      clean: all
-      aptPackages: ${{ parameters.aptPackages }}
+    steps:
-      pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - task: DownloadPipelineArtifact@2
+        aptPackages: ${{ parameters.aptPackages }}
-    displayName: 'Download Pipeline Wheel Files'
+        pipModules: ${{ parameters.pipModules }}
-    inputs:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      itemPattern: '**/*.whl'
+    - task: DownloadPipelineArtifact@2
-      targetPath: $(Agent.BuildDirectory)
+      displayName: 'Download Pipeline Wheel Files'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+      inputs:
-    parameters:
+        itemPattern: '**/*.whl'
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+        targetPath: $(Agent.BuildDirectory)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
-    parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-      dependencyList: ${{ parameters.rocmDependencies }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - task: Bash@3
+        checkoutRef: ${{ parameters.checkoutRef }}
-    displayName: pip install
+        dependencyList: ${{ parameters.rocmDependencies }}
-    inputs:
+        gpuTarget: ${{ job.target }}
-      targetType: inline
+    - task: Bash@3
-      script: find -name *.whl -exec pip install {} \;
+      displayName: pip install
-      workingDirectory: $(Agent.BuildDirectory)
+      inputs:
-  - task: Bash@3
+        targetType: inline
-    displayName: Setup test environment
+        script: find -name *.whl -exec pip install {} \;
-    inputs:
+        workingDirectory: $(Agent.BuildDirectory)
-      targetType: inline
+    - task: Bash@3
-      script: |
+      displayName: Setup test environment
-        sudo rm -rf /opt/rocm
+      inputs:
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        targetType: inline
-  - task: Bash@3
+        script: |
-    displayName: Add Python site-packages binaries to path
+          # Assuming that /opt is no longer persistent across runs, test environments are fully ephemeral
-    inputs:
+          sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-      targetType: inline
+          USER_BASE=$(python3 -m site --user-base)
-      script: |
+          echo "##vso[task.prependpath]$USER_BASE/bin"
-        USER_BASE=$(python3 -m site --user-base)
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
-        echo "##vso[task.prependpath]$USER_BASE/bin"
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
-  - task: Bash@3
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-    displayName: Add ROCm binaries to PATH
+    - task: Bash@3
-    inputs:
+      displayName: tox test
-      targetType: inline
+      inputs:
-      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+        targetType: inline
-  - task: Bash@3
+        script: tox run -v -e ci -- -m pre_checkin
-    displayName: Add ROCm compilers to PATH
+        workingDirectory: $(Build.SourcesDirectory)
-    inputs:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      targetType: inline
+      parameters:
-      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        pipModules: ${{ parameters.pipModules }}
-  - task: Bash@3
+        environment: test
-    displayName: tox test
+        gpuTarget: ${{ job.target }}
-    inputs:
+        pythonEnvVars: true
-      targetType: inline
+        extraPaths: /home/user/workspace/rocm/llvm/bin:/home/user/workspace/rocm/bin
-      script: tox run -v -e ci -- -m pre_checkin
+  # docker image will not have python site-packages in path, but the env vars will make it easier
-      workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: Remove Python site-packages binaries from path
-    inputs:
-      targetType: inline
-      script: |
-        USER_BASE=$(python3 -m site --user-base)
-        echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$USER_BASE/bin;;' -e 's;^/;;' -e 's;/$;;')"
-  - task: Bash@3
-    displayName: Remove ROCm binaries from PATH
-    inputs:
-      targetType: inline
-      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
-  - task: Bash@3
-    displayName: Remove ROCm compilers from PATH
-    inputs:
-      targetType: inline
-      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      environment: test
-      gpuTarget: $(JOB_GPU_TARGET)
-      optSymLink: true
-      pythonEnvVars: true
-      extraPaths: /home/user/workspace/rocm/llvm/bin:/home/user/workspace/rocm/bin
-# docker image will not have python site-packages in path, but the env vars will make it easier

.azuredevops/components/TransferBench.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -30,94 +35,108 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
-jobs:
+- name: jobMatrix
-- job: TransferBench
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: TransferBench_testing
+jobs:
-  dependsOn: TransferBench
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: TransferBench_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    pool:
-  workspace:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    clean: all
+    workspace:
-  strategy:
+      clean: all
-    matrix:
+    steps:
-      gfx942:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-        JOB_GPU_TARGET: gfx942
+      parameters:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+        aptPackages: ${{ parameters.aptPackages }}
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
-    parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        extraBuildFlags: >-
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-    parameters:
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-      componentName: TransferBench All-to-all
+          -GNinja
-      testDir: '$(Agent.BuildDirectory)'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      testExecutable: './rocm/bin/TransferBench'
+      parameters:
-      testParameters: 'a2a'
+        gpuTarget: ${{ job.target }}
-      testPublishResults: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
-    parameters:
+        gpuTarget: ${{ job.target }}
-      componentName: TransferBench Peer-to-peer
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-      testDir: '$(Agent.BuildDirectory)'
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      testExecutable: './rocm/bin/TransferBench'
+    #   parameters:
-      testParameters: 'p2p'
+    #     aptPackages: ${{ parameters.aptPackages }}
-      testPublishResults: false
+    #     gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+
-    parameters:
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-      aptPackages: ${{ parameters.aptPackages }}
+  - job: TransferBench_test_${{ job.target }}
-      environment: test
+    dependsOn: TransferBench_build_${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: TransferBench All-to-all
+        testDir: '$(Agent.BuildDirectory)'
+        testExecutable: './rocm/bin/TransferBench'
+        testParameters: 'a2a'
+        testPublishResults: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: TransferBench Peer-to-peer
+        testDir: '$(Agent.BuildDirectory)'
+        testExecutable: './rocm/bin/TransferBench'
+        testParameters: 'p2p'
+        testPublishResults: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/amdsmi.yml

@@ -5,20 +5,36 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
     - cmake
     - libdrm-dev
+    - ninja-build
     - python3-pip
     - pkg-config
 
+- name: jobMatrix
+  type: object
+  default:
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 jobs:
-- job: amdsmi
+- job: amdsmi_build
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -33,44 +49,46 @@ jobs:
     parameters:
       extraBuildFlags: >-
         -DBUILD_TESTS=ON
+        -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
 
-- job: amdsmi_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  dependsOn: amdsmi
+  - job: amdsmi_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: amdsmi_build
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool: $(JOB_TEST_POOL)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  workspace:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    clean: all
+      )
-  strategy:
+    variables:
-    matrix:
+    - group: common
-      gfx942:
+    - template: /.azuredevops/variables-global.yml
-        JOB_GPU_TARGET: gfx942
+    pool: ${{ job.target }}_test_pool
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    workspace:
-  steps:
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    steps:
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-      runRocminfo: false
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+        runRocminfo: false
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-      componentName: amdsmi
+      parameters:
-      testDir: '$(Agent.BuildDirectory)'
+        componentName: amdsmi
-      testExecutable: './rocm/share/amd_smi/tests/amdsmitst'
+        testDir: '$(Agent.BuildDirectory)'
-      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+        testExecutable: './rocm/share/amd_smi/tests/amdsmitst'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      environment: test
+        aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/aomp-extras.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -23,7 +28,8 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -38,6 +44,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: aomp-extras
@@ -51,6 +58,6 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/aomp.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 # reference:
 # https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL_PREREQUISITE.md
 - name: aptPackages
@@ -43,9 +48,7 @@ parameters:
     - python3-dev
     - libudev-dev
     - parallel
-  # Referencing comment snippet below but excluding rocprofiler and roctracer.
+  # Referencing comment snippet.
-  # This is to remove need for separate build per gpu target.
-  # With our selected build flags, compilation and installation work fine without these two.
   #
   # snippet from https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L131-L134
   #
@@ -65,10 +68,21 @@ parameters:
     - rocm-core
     - rocminfo
     - rocm_smi_lib
+    - rocprofiler
     - rocprofiler-register
+    - rocprofiler-sdk
     - ROCR-Runtime
     - roctracer
 
+- name: jobMatrix
+  type: object
+  default:
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 jobs:
 - job: aomp
   variables:
@@ -99,425 +113,147 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-# Because clang is not being rebuilt and to separate downloaded ROCm
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
-# dependencies from the new artifacts, we use temporary symbolic links
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-# for the compilation and installation to go through.
-  - script: |
-      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-      mkdir -p $(Build.BinariesDirectory)/lib/llvm/bin
-      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang $(Build.BinariesDirectory)/lib/llvm/bin/clang
-      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang++ $(Build.BinariesDirectory)/lib/llvm/bin/clang++
-      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/llvm-config $(Build.BinariesDirectory)/lib/llvm/bin/llvm-config
-      ln -s $(Agent.BuildDirectory)/rocm/llvm $(Build.BinariesDirectory)/llvm
-      ls -1R $(Build.BinariesDirectory)
-    displayName: Extra build environment setup
-# We follow the sequence described in the aomp repo instructions
-# https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL.md
-# But instead of calling build_aomp.sh directly, we have to split up the calls to build each component
-# in order for the omp.h header file to be both be published as an artifact,
-# and be found when building subsequent components.
-# Environment variables specified per bash script were determined from looking through the
-# individual scripts and iteratively testing compilation.
-# Splitting up the build_aomp.sh call is also easier for debugging within Azure, as the former
-# method leads to a giant build log compared to separate logs per script call.
-#
-# Components compiled and the order for non-standalone build found at
-# https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L135-L142
-  - task: Bash@3
-    displayName: Build Prereq
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_prereq.sh
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: Build extras
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_extras.sh
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_EXTRAS: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Install extras
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_extras.sh
-      arguments: install
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_EXTRAS: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Build openmp
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_openmp.sh
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_LEGACY_OPENMP: 1
-      AOMP_USE_NINJA: 1
-      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_OPENMP: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Install openmp
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_openmp.sh
-      arguments: install
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_LEGACY_OPENMP: 1
-      AOMP_USE_NINJA: 1
-      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_OPENMP: $(Build.BinariesDirectory)
-  - script: ln -s $(Build.BinariesDirectory)/lib/llvm/include/omp.h $(Build.SourcesDirectory)/llvm-project/llvm/include/omp.h
-    displayName: Link omp header
-  - task: Bash@3
-    displayName: Build offload
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_offload.sh
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_LEGACY_OPENMP: 1
-      AOMP_USE_NINJA: 1
-      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_OPENMP: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Install offload
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_offload.sh
-      arguments: install
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_LEGACY_OPENMP: 1
-      AOMP_USE_NINJA: 1
-      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_OPENMP: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Build llvm-classic
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_llvm-classic.sh
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Install llvm-classic
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_llvm-classic.sh
-      arguments: install
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Build flang-classic
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-classic.sh
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Install flang-classic
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-classic.sh
-      arguments: install
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Build pgmath
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_pgmath.sh
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Install pgmath
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_pgmath.sh
-      arguments: install
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Build flang
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang.sh
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Install flang
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang.sh
-      arguments: install
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Build flang_runtime
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang_runtime.sh
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-  - task: Bash@3
-    displayName: Install flang_runtime
-    inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang_runtime.sh
-      arguments: install
-    env:
-      AOMP_REPOS: $(Build.SourcesDirectory)
-      AOMP_PROJECT_REPO_NAME: llvm-project
-      AOMP_STANDALONE_BUILD: 0
-      AOMP_BUILD_SANITIZER: 0
-      AOMP_BUILD_DEBUG: 0
-      AOMP_USE_NINJA: 1
-      INSTALL_PREFIX: $(Build.BinariesDirectory)
-      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
-      AOMP: $(Build.BinariesDirectory)
-      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-# Clean up build environment before publish artifact
-  - script: |
-      rm $(Build.BinariesDirectory)/lib/llvm/bin/clang
-      rm $(Build.BinariesDirectory)/lib/llvm/bin/clang++
-      rm $(Build.BinariesDirectory)/lib/llvm/bin/llvm-config
-      rm $(Build.BinariesDirectory)/lib/llvm/bin/flang
-      rm $(Build.BinariesDirectory)/llvm
-    displayName: Remove temporary symbolic links
-# aomp scripts changed where files get installed in scripts, copy to expected location
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
-      sourceDir: $(Build.BinariesDirectory)/lib/llvm
+      componentName: extras
-      targetDir: $(Build.ArtifactStagingDirectory)
+      cmakeBuildDir: '$(Build.SourcesDirectory)/aomp-extras/build'
-# Remove temporary directory used to deal with expected paths of scripts
+      cmakeSourceDir: '$(Build.SourcesDirectory)/aomp-extras'
-  - script: |
+      installDir: '$(Build.BinariesDirectory)/llvm'
-      rm -rf $(Build.BinariesDirectory)/lib/llvm
+      extraBuildFlags: >-
-    displayName: Remove temporary directories
+        -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm
-# Copy the files to artifact staging temporarily to clean up binaries directory
+        -DCMAKE_BUILD_TYPE=Release
-# and then copy files back to llvm subdirectory in the cleaned up binaries directory
+        -DAOMP_STANDALONE_BUILD=0
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+        -DAOMP_VERSION_STRING=9.99.99
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
-      sourceDir: $(Build.BinariesDirectory)
+      componentName: openmp
-      targetDir: $(Build.ArtifactStagingDirectory)
+      cmakeBuildDir: '$(Build.SourcesDirectory)/llvm-project/openmp/build'
-      clean: false
+      cmakeSourceDir: '$(Build.SourcesDirectory)/llvm-project/openmp'
-  - script: |
+      installDir: '$(Build.BinariesDirectory)/llvm'
-      ln -s $(Build.ArtifactStagingDirectory)/bin/flang-classic $(Build.ArtifactStagingDirectory)/bin/flang
+      extraBuildFlags: >-
-    displayName: Recreate flang symlink
+        -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm;$(Build.BinariesDirectory)"
-  - task: DeleteFiles@1
+        -DCMAKE_BUILD_TYPE=Release
-    displayName: 'Cleanup Binaries Directory'
+        -DOPENMP_TEST_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
+        -DOPENMP_TEST_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
+        -DOPENMP_ENABLE_LIBOMPTARGET=1
+        -DLIBOMP_COPY_EXPORTS=OFF
+        -DLIBOMP_OMPT_SUPPORT=ON
+        -DLIBOMP_OMPD_SUPPORT=ON
+        -DCMAKE_SKIP_INSTALL_RPATH=TRUE
+        -DLLVM_MAIN_INCLUDE_DIR=$(Build.SourcesDirectory)/llvm-project/llvm/include
+        -DLIBOMP_FORTRAN_MODULES_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/flang
+        -DLIBOMP_MODULES_INSTALL_PATH=$(Build.BinariesDirectory)/llvm/include/flang/
+        -GNinja
+  - task: Bash@3
+    displayName: 'ROCm symbolic link'
     inputs:
-      SourceFolder: '$(Build.BinariesDirectory)'
+      targetType: inline
-      Contents: '/**/*'
+      script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-      RemoveDotFiles: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
-      sourceDir: $(Build.ArtifactStagingDirectory)
+      componentName: offload
-      targetDir: $(Build.BinariesDirectory)/llvm
+      cmakeBuildDir: '$(Build.SourcesDirectory)/llvm-project/offload/build'
-  - task: DeleteFiles@1
+      cmakeSourceDir: '$(Build.SourcesDirectory)/llvm-project/offload'
-    displayName: 'Cleanup Staging Directory'
+      installDir: '$(Build.BinariesDirectory)/llvm'
-    inputs:
+      extraBuildFlags: >-
-      SourceFolder: $(Build.ArtifactStagingDirectory)
+        -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm;$(Build.BinariesDirectory)"
-      Contents: '/**/*'
+        -DCMAKE_BUILD_TYPE=Release
-      RemoveDotFiles: true
+        -DOPENMP_TEST_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
+        -DOPENMP_TEST_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
+        -DCMAKE_SKIP_INSTALL_RPATH=TRUE
+        -DLLVM_MAIN_INCLUDE_DIR=$(Build.SourcesDirectory)/llvm-project/llvm/include
+        -DLIBOMPTARGET_LLVM_INCLUDE_DIRS=$(Build.SourcesDirectory)/llvm-project/llvm/include
+        -DCMAKE_EXE_LINKER_FLAGS="-L$(Agent.BuildDirectory)/rocm/llvm/lib"
+        -DCMAKE_SHARED_LINKER_FLAGS="-L$(Agent.BuildDirectory)/rocm/llvm/lib"
+        -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
-      optSymLink: true
 
-- job: aomp_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  dependsOn: aomp
+  - job: aomp_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: aomp
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool: $(JOB_TEST_POOL)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  workspace:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    clean: all
+      )
-  strategy:
+    variables:
-    matrix:
+    - group: common
-      gfx942:
+    - template: /.azuredevops/variables-global.yml
-        JOB_GPU_TARGET: gfx942
+    pool: ${{ job.target }}_test_pool
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    workspace:
-  steps:
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    steps:
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmDependencies }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - task: Bash@3
+        dependencyList: ${{ parameters.rocmDependencies }}
-    displayName: ROCm symbolic link
+    - task: Bash@3
-    inputs:
+      displayName: ROCm symbolic link
-      targetType: inline
+      inputs:
-      script: |
+        targetType: inline
-        sudo rm -rf /opt/rocm
+        script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+      parameters:
-    parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+      parameters:
-    parameters:
+        checkoutRepo: aomp-extras_repo
-      checkoutRepo: aomp-extras_repo
+  # these copy steps are from the aomp prototype script for test prep
-# these copy steps are from the aomp prototype script for test prep
+    - task: CopyFiles@2
-  - task: CopyFiles@2
+      displayName: 'Copy AOMP contents'
-    displayName: 'Copy AOMP contents'
+      inputs:
-    inputs:
+        CleanTargetFolder: false
-      CleanTargetFolder: false
+        SourceFolder: $(Build.SourcesDirectory)/aomp
-      SourceFolder: $(Build.SourcesDirectory)/aomp
+        Contents: |
-      Contents: |
+          **
-        **
+          !**/.git/**
-        !**/.git/**
+          !**/.github/**
-        !**/.github/**
+          !**/.gitignore
-        !**/.gitignore
+        TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests
-      TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests
+        retryCount: 3
-      retryCount: 3
+    - task: CopyFiles@2
-  - task: CopyFiles@2
+      displayName: 'Copy FileCheck'
-    displayName: 'Copy FileCheck'
+      inputs:
-    inputs:
+        CleanTargetFolder: false
-      CleanTargetFolder: false
+        SourceFolder: $(Agent.BuildDirectory)/rocm/llvm/bin
-      SourceFolder: $(Agent.BuildDirectory)/rocm/llvm/bin
+        Contents: FileCheck
-      Contents: FileCheck
+        TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests/bin
-      TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests/bin
+        retryCount: 3
-      retryCount: 3
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - task: Bash@3
-  - task: Bash@3
+      displayName: Test AOMP
-    displayName: Test AOMP
+      continueOnError: true
-    continueOnError: true
+      inputs:
-    inputs:
+        targetType: inline
-      targetType: inline
+        script: ./run_rocm_test.sh
-      script: ./run_rocm_test.sh
+        workingDirectory: $(Build.SourcesDirectory)/aomp/bin
-      workingDirectory: $(Build.SourcesDirectory)/aomp/bin
+      env:
-    env:
+        AOMP: $(Agent.BuildDirectory)/rocm/llvm
-      AOMP: $(Agent.BuildDirectory)/rocm/llvm
+        AOMP_REPOS_TEST: $(Build.SourcesDirectory)/aomp-test
-      AOMP_REPOS_TEST: $(Build.SourcesDirectory)/aomp-test
+        AOMP_TEST_DIR: $(Build.SourcesDirectory)/aomp-test
-      AOMP_TEST_DIR: $(Build.SourcesDirectory)/aomp-test
+        SKIP_TEST_PACKAGE: 1
-      SKIP_TEST_PACKAGE: 1
+        MAINLINE_BUILD: 1
-      MAINLINE_BUILD: 1
+        SUITE_LIST: smoke
-      SUITE_LIST: smoke
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
-    parameters:
+        aptPackages: ${{ parameters.aptPackages }}
-      aptPackages: ${{ parameters.aptPackages }}
+        environment: test
-      environment: test
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        optSymLink: true
-      optSymLink: true

.azuredevops/components/composable_kernel.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -27,122 +32,134 @@ parameters:
   type: object
   default:
     - clr
-    - composable_kernel
     - llvm-project
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
 
-jobs:
+- name: jobMatrix
-- job: composable_kernel
+  type: object
-  timeoutInMinutes: 240
+  default:
-  variables:
+    buildJobs:
-  - group: common
+      - gfx942:
-  - template: /.azuredevops/variables-global.yml
+        target: gfx942
-  - name: DAY_STRING
+      - gfx90a:
-    value: $[format('{0:ddMMyyyy}', pipeline.startTime)]
+        target: gfx90a
-  pool: ${{ variables.ULTRA_BUILD_POOL }}
+    testJobs:
-  workspace:
+      - gfx942:
-    clean: all
+        target: gfx942
-  strategy:
+      - gfx90a:
-    matrix:
+        target: gfx90a
-      gfx942:
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - script: |
-      mkdir -p $(CCACHE_DIR)
-      echo "##vso[task.prependpath]/usr/lib/ccache"
-    displayName: Update path for ccache
-  - task: Cache@2
-    displayName: Ccache caching
-    inputs:
-      key: composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING) | $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-      path: $(CCACHE_DIR)
-      restoreKeys: |
-        composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING)
-        composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET)
-        composable_kernel | $(Agent.OS)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-        -DCMAKE_C_COMPILER_LAUNCHER=ccache
-        -DCMAKE_HIP_FLAGS="-Wno-missing-include-dirs"
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DCK_BUILD_JIT_LIB=ON
-        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
-        -DCMAKE_BUILD_TYPE=Release
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: composable_kernel_testing
+jobs:
-  timeoutInMinutes: 90
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  dependsOn: composable_kernel
+  - job: composable_kernel_build_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    timeoutInMinutes: 240
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  - name: TEST_LOG_FILE
+    - name: DAY_STRING
-    value: $(Pipeline.Workspace)/ckTestLog.log
+      value: $[format('{0:ddMMyyyy}', pipeline.startTime)]
-  pool: $(JOB_TEST_POOL)
+    pool: ${{ variables.ULTRA_BUILD_POOL }}
-  workspace:
+    workspace:
-    clean: all
+      clean: all
-  strategy:
+    steps:
-    matrix:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      gfx942:
+      parameters:
-        JOB_GPU_TARGET: gfx942
+        aptPackages: ${{ parameters.aptPackages }}
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
-    parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-    parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - script: |
-    parameters:
+        mkdir -p $(CCACHE_DIR)
-      checkoutRef: ${{ parameters.checkoutRef }}
+        echo "##vso[task.prependpath]/usr/lib/ccache"
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+      displayName: Update path for ccache
-      gpuTarget: $(JOB_GPU_TARGET)
+    - task: Cache@2
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+      displayName: Ccache caching
-  - task: Bash@3
+      inputs:
-    displayName: Iterate through test scripts
+        key: composable_kernel | $(Agent.OS) | ${{ job.target }} | $(DAY_STRING) | $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-    inputs:
+        path: $(CCACHE_DIR)
-      targetType: inline
+        restoreKeys: |
-      script: |
+          composable_kernel | $(Agent.OS) | ${{ job.target }} | $(DAY_STRING)
-        for file in ./test_*; do
+          composable_kernel | $(Agent.OS) | ${{ job.target }}
-          ./$file | tee -a $(TEST_LOG_FILE)
+          composable_kernel | $(Agent.OS)
-        done
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      workingDirectory: $(Agent.BuildDirectory)/rocm/bin
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        extraBuildFlags: >-
-    parameters:
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-      aptPackages: ${{ parameters.aptPackages }}
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-      environment: test
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache
+          -DCMAKE_HIP_FLAGS="-Wno-missing-include-dirs"
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+          -DCK_BUILD_JIT_LIB=ON
+          -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+          -DCMAKE_BUILD_TYPE=Release
+          -DGPU_TARGETS=${{ job.target }}
+          -GNinja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: composable_kernel_test_${{ job.target }}
+    timeoutInMinutes: 180
+    dependsOn: composable_kernel_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    - name: TEST_LOG_FILE
+      value: $(Pipeline.Workspace)/ckTestLog.log
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - task: Bash@3
+      displayName: Iterate through test scripts
+      inputs:
+        targetType: inline
+        script: |
+          for file in ./test_*; do
+            ./$file | tee -a $(TEST_LOG_FILE)
+          done
+        workingDirectory: $(Agent.BuildDirectory)/rocm/bin
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/copyHIP.yml

@@ -26,9 +26,11 @@ jobs:
     parameters:
       componentName: HIP
       pipelineId: $(HIP_PIPELINE_ID)
-  - template:  ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+  - task: Bash@3
-    parameters:
+    displayName: Copy HIP artifacts
-      sourceDir: $(Agent.BuildDirectory)/rocm
+    inputs:
+      targetType: inline
+      script: cp -a $(Agent.BuildDirectory)/rocm/* $(Build.BinariesDirectory)/
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml

.azuredevops/components/half.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -25,7 +30,8 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -40,6 +46,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -59,7 +66,7 @@ jobs:
         ./bin/test
       workingDirectory: $(Build.SourcesDirectory)/test
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
-      environment: combined
+  #     environment: combined

.azuredevops/components/hip-tests.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -37,111 +42,118 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
+- name: jobMatrix
+  type: object
+  default:
+    buildJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 # HIP with AMD backend
 jobs:
-- job: hip_tests
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  variables:
+  - job: hip_tests_build_${{ job.target }}
-  - group: common
+    variables:
-  - template: /.azuredevops/variables-global.yml
+    - group: common
-  - name: HIP_ROCCLR_HOME
+    - template: /.azuredevops/variables-global.yml
-    value: $(Agent.BuildDirectory)/rocm
+    - name: HIP_ROCCLR_HOME
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+      value: $(Agent.BuildDirectory)/rocm
-  workspace:
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-    clean: all
+    workspace:
-  strategy:
+      clean: all
-    matrix:
+    steps:
-      gfx942:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-        JOB_GPU_TARGET: gfx942
+      parameters:
-  steps:
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
+      parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+    # compile hip-tests
-      dependencyList: ${{ parameters.rocmDependencies }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-  # compile hip-tests
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+        componentName: hip-tests
-    parameters:
+        cmakeSourceDir: '../catch'
-      componentName: hip-tests
+        customBuildTarget: build_tests
-      cmakeSourceDir: '../catch'
+        extraBuildFlags: >-
-      customBuildTarget: build_tests
+          -DHIP_PLATFORM=amd
-      extraBuildFlags: >-
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DHIP_PLATFORM=amd
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+          -DHIP_PATH=$(Agent.BuildDirectory)/rocm
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+          -DOFFLOAD_ARCH_STR="--offload-arch=${{ job.target }}"
-        -DHIP_PATH=$(Agent.BuildDirectory)/rocm
+          -GNinja
-        -DOFFLOAD_ARCH_STR="--offload-arch=$(JOB_GPU_TARGET)"
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-        -GNinja
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        aptPackages: ${{ parameters.aptPackages }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      aptPackages: ${{ parameters.aptPackages }}
+        extraEnvVars:
-      gpuTarget: $(JOB_GPU_TARGET)
+          - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
 
-- job: hip_tests_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  timeoutInMinutes: 240
+  - job: hip_tests_test_${{ job.target }}
-  dependsOn: hip_tests
+    timeoutInMinutes: 240
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: hip_tests_build_${{ job.target }}
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool: $(JOB_TEST_POOL)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  workspace:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    clean: all
+      )
-  strategy:
+    variables:
-    matrix:
+    - group: common
-      gfx942:
+    - template: /.azuredevops/variables-global.yml
-        JOB_GPU_TARGET: gfx942
+    pool: ${{ job.target }}_test_pool
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    workspace:
-  steps:
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    steps:
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmTestDependencies }}
-  - task: Bash@3
+        gpuTarget: ${{ job.target }}
-    displayName: Symlink rocm_agent_enumerator
+    - task: Bash@3
-    inputs:
+      displayName: Symlink rocm_agent_enumerator
-      targetType: inline
+      inputs:
-      script: |
+        targetType: inline
-        sudo rm -rf /opt/rocm
+        script: |
-        sudo mkdir -p /opt/rocm/bin
+          # Assuming that /opt is no longer persistent across runs, test environments are fully ephemeral
-        sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rocm_agent_enumerator /opt/rocm/bin/rocm_agent_enumerator
+          sudo mkdir -p /opt/rocm/bin
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rocm_agent_enumerator /opt/rocm/bin/rocm_agent_enumerator
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-      componentName: hip_tests
+      parameters:
-      testDir: $(Agent.BuildDirectory)/rocm/share/hip
+        componentName: hip_tests
-  - task: Bash@3
+        testDir: $(Agent.BuildDirectory)/rocm/share/hip
-    displayName: Clean up symlink
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    condition: always()
+      parameters:
-    inputs:
+        aptPackages: ${{ parameters.aptPackages }}
-      targetType: inline
+        environment: test
-      script: sudo rm -rf /opt/rocm
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        optSymLink: true
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      environment: test
-      gpuTarget: $(JOB_GPU_TARGET)
-      optSymLink: true

.azuredevops/components/hipBLAS-common.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -29,7 +34,8 @@ jobs:
   - name: ROCM_PATH
     value: $(Agent.BuildDirectory)/rocm
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -44,6 +50,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -53,8 +60,8 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
-      extraEnvVars:
+  #     extraEnvVars:
-        - ROCM_PATH:::/home/user/workspace/rocm
+  #       - ROCM_PATH:::/home/user/workspace/rocm

.azuredevops/components/hipBLAS.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -51,98 +56,111 @@ parameters:
     - rocSOLVER
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: hipBLAS
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DHIP_PLATFORM=amd
-        -DBUILD_CLIENTS_TESTS=ON
-        -DBUILD_CLIENTS_BENCHMARKS=OFF
-        -DBUILD_CLIENTS_SAMPLES=OFF
-        -DCPACK_SET_DESTDIR=OFF
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      installAOCL: true
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: hipBLAS_testing
+jobs:
-  dependsOn: hipBLAS
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: hipBLAS_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  workspace:
+    workspace:
-    clean: all
+      clean: all
-  strategy:
+    steps:
-    matrix:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      gfx942:
+      parameters:
-        JOB_GPU_TARGET: gfx942
+        aptPackages: ${{ parameters.aptPackages }}
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+        pipModules: ${{ parameters.pipModules }}
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
-    parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        extraBuildFlags: >-
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DCMAKE_BUILD_TYPE=Release
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-    parameters:
+          -DHIP_PLATFORM=amd
-      componentName: hipBLAS
+          -DBUILD_CLIENTS_TESTS=ON
-      testExecutable: $(Agent.BuildDirectory)/rocm/bin/hipblas-test
+          -DBUILD_CLIENTS_BENCHMARKS=OFF
-      testParameters: '--yaml hipblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'
+          -DBUILD_CLIENTS_SAMPLES=OFF
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+          -DCPACK_SET_DESTDIR=OFF
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -GNinja
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      pipModules: ${{ parameters.pipModules }}
+        gpuTarget: ${{ job.target }}
-      environment: test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        installAOCL: true
+        gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: hipBLAS_test_${{ job.target }}
+    dependsOn: hipBLAS_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: hipBLAS
+        testExecutable: $(Agent.BuildDirectory)/rocm/bin/hipblas-test
+        testParameters: '--yaml hipblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/hipBLASLt.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -22,7 +27,7 @@ parameters:
   type: object
   default:
     - joblib
-    - packaging>=22.0
+    - "packaging>=22.0"
     - --upgrade
 - name: rocmDependencies
   type: object
@@ -49,151 +54,173 @@ parameters:
     - ROCR-Runtime
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: hipBLASLt
+  type: object
-  timeoutInMinutes: 300
+  default:
-  variables:
+    buildJobs:
-  - group: common
+      - gfx942:
-  - template: /.azuredevops/variables-global.yml
+        target: gfx942
-  - name: HIP_ROCCLR_HOME
+      - gfx90a:
-    value: $(Build.BinariesDirectory)/rocm
+        target: gfx90a
-  - name: TENSILE_ROCM_ASSEMBLER_PATH
+    testJobs:
-    value: $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+      - gfx942:
-  - name: TENSILE_ROCM_OFFLOAD_BUNDLER_PATH
+        target: gfx942
-    value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang-offload-bundler
+      - gfx90a:
-  - name: ROCM_PATH
+        target: gfx90a
-    value: $(Agent.BuildDirectory)/rocm
-  - name: DAY_STRING
-    value: $[format('{0:ddMMyyyy}', pipeline.startTime)]
-  pool: ${{ variables.ULTRA_BUILD_POOL }}
-  workspace:
-    clean: all
-  strategy:
-    matrix:
-      gfx942:
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-    displayName: ROCm symbolic link
-# Build and install gtest, lapack, hipBLAS-common
-# $(Pipeline.Workspace)/deps is a temporary folder for the build process
-# $(Pipeline.Workspace)/s/deps is part of the hipBLASLt repo
-  - script: mkdir $(Pipeline.Workspace)/deps
-    displayName: Create temp folder for external dependencies
-# hipBLASLt already has a CMake script for external deps, so we can just run that
-# https://github.com/ROCm/hipBLASLt/blob/develop/deps/CMakeLists.txt
-  - script: cmake $(Pipeline.Workspace)/s/deps
-    displayName: Configure hipBLASLt external dependencies
-    workingDirectory: $(Pipeline.Workspace)/deps
-  - script: make
-    displayName: Build hipBLASLt external dependencies
-    workingDirectory: $(Pipeline.Workspace)/deps
-  - script: sudo make install
-    displayName: Install hipBLASLt external dependencies
-    workingDirectory: $(Pipeline.Workspace)/deps
-  - script: |
-      mkdir -p $(CCACHE_DIR)
-      echo "##vso[task.prependpath]/usr/lib/ccache"
-    displayName: Update path for ccache
-  - task: Cache@2
-    displayName: Ccache caching
-    inputs:
-      key: hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING) | $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-      path: $(CCACHE_DIR)
-      restoreKeys: |
-        hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING)
-        hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET)
-        hipBLASLt | $(Agent.OS)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-        -DCMAKE_C_COMPILER_LAUNCHER=ccache
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DTensile_LOGIC=
-        -DTensile_CPU_THREADS=
-        -DTensile_LIBRARY_FORMAT=msgpack
-        -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
-        -DBUILD_CLIENTS_TESTS=ON
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraPaths: /home/user/workspace/rocm/llvm/bin:/home/user/workspace/rocm/bin
-      installLatestCMake: true
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
-        - TENSILE_ROCM_ASSEMBLER_PATH:::/home/user/workspace/rocm/llvm/bin/amdclang
-        - TENSILE_ROCM_OFFLOAD_BUNDLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang-offload-bundler
-        - ROCM_PATH:::/home/user/workspace/rocm
-      extraCopyDirectories:
-        - deps
 
-- job: hipBLASLt_testing
+jobs:
-  dependsOn: hipBLASLt
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: hipBLASLt_build_${{ job.target }}
-  variables:
+    timeoutInMinutes: 300
-  - group: common
+    variables:
-  - template: /.azuredevops/variables-global.yml
+    - group: common
-  pool: $(JOB_TEST_POOL)
+    - template: /.azuredevops/variables-global.yml
-  workspace:
+    - name: HIP_ROCCLR_HOME
-    clean: all
+      value: $(Build.BinariesDirectory)/rocm
-  strategy:
+    - name: TENSILE_ROCM_ASSEMBLER_PATH
-    matrix:
+      value: $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-      gfx942:
+    - name: TENSILE_ROCM_OFFLOAD_BUNDLER_PATH
-        JOB_GPU_TARGET: gfx942
+      value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang-offload-bundler
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - name: ROCM_PATH
-  steps:
+      value: $(Agent.BuildDirectory)/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - name: DAY_STRING
-    parameters:
+      value: $[format('{0:ddMMyyyy}', pipeline.startTime)]
-      aptPackages: ${{ parameters.aptPackages }}
+    pool: ${{ variables.ULTRA_BUILD_POOL }}
-      pipModules: ${{ parameters.pipModules }}
+    workspace:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    steps:
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        pipModules: ${{ parameters.pipModules }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
+      parameters:
-      componentName: hipBLASLt
+        checkoutRef: ${{ parameters.checkoutRef }}
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        dependencyList: ${{ parameters.rocmDependencies }}
-      testExecutable: './hipblaslt-test'
+        gpuTarget: ${{ job.target }}
-      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    - task: Bash@3
-    parameters:
+      displayName: Add ROCm binaries to PATH
-      aptPackages: ${{ parameters.aptPackages }}
+      inputs:
-      pipModules: ${{ parameters.pipModules }}
+        targetType: inline
-      environment: test
+        script: |
-      gpuTarget: $(JOB_GPU_TARGET)
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  # Build and install gtest, lapack, hipBLAS-common
+  # $(Pipeline.Workspace)/deps is a temporary folder for the build process
+  # $(Pipeline.Workspace)/s/deps is part of the hipBLASLt repo
+    - script: mkdir $(Pipeline.Workspace)/deps
+      displayName: Create temp folder for external dependencies
+  # hipBLASLt already has a CMake script for external deps, so we can just run that
+  # https://github.com/ROCm/hipBLASLt/blob/develop/deps/CMakeLists.txt
+    - script: cmake $(Pipeline.Workspace)/s/deps
+      displayName: Configure hipBLASLt external dependencies
+      workingDirectory: $(Pipeline.Workspace)/deps
+    - script: make
+      displayName: Build hipBLASLt external dependencies
+      workingDirectory: $(Pipeline.Workspace)/deps
+    - script: sudo make install
+      displayName: Install hipBLASLt external dependencies
+      workingDirectory: $(Pipeline.Workspace)/deps
+    - script: |
+        mkdir -p $(CCACHE_DIR)
+        echo "##vso[task.prependpath]/usr/lib/ccache"
+      displayName: Update path for ccache
+    - task: Cache@2
+      displayName: Ccache caching
+      inputs:
+        key: hipBLASLt | $(Agent.OS) | ${{ job.target }} | $(DAY_STRING) | $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+        path: $(CCACHE_DIR)
+        restoreKeys: |
+          hipBLASLt | $(Agent.OS) | ${{ job.target }} | $(DAY_STRING)
+          hipBLASLt | $(Agent.OS) | ${{ job.target }}
+          hipBLASLt | $(Agent.OS)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+      parameters:
+        extraBuildFlags: >-
+          -DCMAKE_BUILD_TYPE=Release
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache
+          -DAMDGPU_TARGETS=${{ job.target }}
+          -DTensile_LOGIC=
+          -DTensile_CPU_THREADS=
+          -DTensile_LIBRARY_FORMAT=msgpack
+          -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
+          -DBUILD_CLIENTS_TESTS=ON
+          -GNinja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        gpuTarget: ${{ job.target }}
+        extraPaths: /home/user/workspace/rocm/llvm/bin:/home/user/workspace/rocm/bin
+        installLatestCMake: true
+        extraEnvVars:
+          - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+          - TENSILE_ROCM_ASSEMBLER_PATH:::/home/user/workspace/rocm/llvm/bin/amdclang
+          - TENSILE_ROCM_OFFLOAD_BUNDLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang-offload-bundler
+          - ROCM_PATH:::/home/user/workspace/rocm
+        extraCopyDirectories:
+          - deps
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: hipBLASLt_test_${{ job.target }}
+    timeoutInMinutes: 300
+    dependsOn: hipBLASLt_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    - name: ROCM_PATH
+      value: $(Agent.BuildDirectory)/rocm
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: hipBLASLt
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        testExecutable: './hipblaslt-test'
+        testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/hipCUB.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -31,87 +36,100 @@ parameters:
     - ROCR-Runtime
     - rocprofiler-register
 
-jobs:
+- name: jobMatrix
-- job: hipCUB
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DBUILD_TEST=ON
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: hipCUB_testing
+jobs:
-  dependsOn: hipCUB
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: hipCUB_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  workspace:
+    workspace:
-    clean: all
+      clean: all
-  strategy:
+    steps:
-    matrix:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      gfx942:
+      parameters:
-        JOB_GPU_TARGET: gfx942
+        aptPackages: ${{ parameters.aptPackages }}
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
-    parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-    parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
+      parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
+        extraBuildFlags: >-
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DBUILD_TEST=ON
-    parameters:
+          -DAMDGPU_TARGETS=${{ job.target }}
-      componentName: hipCUB
+          -GNinja
-      testDir: '$(Agent.BuildDirectory)/rocm/bin/hipcub'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
-    parameters:
+        gpuTarget: ${{ job.target }}
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-      environment: test
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: hipCUB_test_${{ job.target }}
+    dependsOn: hipCUB_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: hipCUB
+        testDir: '$(Agent.BuildDirectory)/rocm/bin/hipcub'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/hipFFT.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -40,97 +45,111 @@ parameters:
     - ROCR-Runtime
     - rocRAND
 
-jobs:
+- name: jobMatrix
-- job: hipFFT
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  - name: HIP_ROCCLR_HOME
+        target: gfx942
-    value: $(Build.BinariesDirectory)/rocm
+      - gfx90a:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx90a
-  workspace:
+    testJobs:
-    clean: all
+      - gfx942:
-  strategy:
+        target: gfx942
-    matrix:
+      - gfx90a:
-      gfx942:
+        target: gfx90a
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DUSE_HIP_CLANG=ON
-        -DHIP_COMPILER=clang
-        -DBUILD_CLIENTS_TESTS=ON
-        -DBUILD_CLIENTS_BENCHMARKS=OFF
-        -DBUILD_CLIENTS_SAMPLES=OFF
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: hipFFT_testing
+jobs:
-  dependsOn: hipFFT
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: hipFFT_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    - name: HIP_ROCCLR_HOME
-  workspace:
+      value: $(Build.BinariesDirectory)/rocm
-    clean: all
+    pool:
-  strategy:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    matrix:
+    workspace:
-      gfx942:
+      clean: all
-        JOB_GPU_TARGET: gfx942
+    steps:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  steps:
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+        aptPackages: ${{ parameters.aptPackages }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        extraBuildFlags: >-
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-    parameters:
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-      componentName: hipFFT
+          -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-      testExecutable: './hipfft-test'
+          -DCMAKE_BUILD_TYPE=Release
-      testParameters: '--test_prob 0.002 --gtest_output=xml:./test_output.xml --gtest_color=yes'
+          -DAMDGPU_TARGETS=${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -DUSE_HIP_CLANG=ON
-    parameters:
+          -DHIP_COMPILER=clang
-      aptPackages: ${{ parameters.aptPackages }}
+          -DBUILD_CLIENTS_TESTS=ON
-      environment: test
+          -DBUILD_CLIENTS_BENCHMARKS=OFF
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DBUILD_CLIENTS_SAMPLES=OFF
+          -GNinja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    #   parameters:
+    #     aptPackages: ${{ parameters.aptPackages }}
+    #     gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: hipFFT_test_${{ job.target }}
+    dependsOn: hipFFT_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: hipFFT
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        testExecutable: './hipfft-test'
+        testParameters: '--test_prob 0.002 --gtest_output=xml:./test_output.xml --gtest_color=yes'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/hipRAND.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -31,93 +36,107 @@ parameters:
     - ROCR-Runtime
     - rocRAND
 
-jobs:
+- name: jobMatrix
-- job: hipRAND
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  - name: HIP_ROCCLR_HOME
+        target: gfx942
-    value: $(Build.BinariesDirectory)/rocm
+      - gfx90a:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx90a
-  workspace:
+    testJobs:
-    clean: all
+      - gfx942:
-  strategy:
+        target: gfx942
-    matrix:
+      - gfx90a:
-      gfx942:
+        target: gfx90a
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DBUILD_TEST=ON
-        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
 
-- job: hipRAND_testing
+jobs:
-  dependsOn: hipRAND
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: hipRAND_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    - name: HIP_ROCCLR_HOME
-  workspace:
+      value: $(Build.BinariesDirectory)/rocm
-    clean: all
+    pool:
-  strategy:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    matrix:
+    workspace:
-      gfx942:
+      clean: all
-        JOB_GPU_TARGET: gfx942
+    steps:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  steps:
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+        aptPackages: ${{ parameters.aptPackages }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        extraBuildFlags: >-
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-    parameters:
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-      componentName: hipRAND
+          -DBUILD_TEST=ON
-      testDir: '$(Agent.BuildDirectory)/rocm/bin/hipRAND'
+          -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-    parameters:
+          -DCMAKE_BUILD_TYPE=Release
-      aptPackages: ${{ parameters.aptPackages }}
+          -DAMDGPU_TARGETS=${{ job.target }}
-      environment: test
+          -GNinja
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    #   parameters:
+    #     aptPackages: ${{ parameters.aptPackages }}
+    #     gpuTarget: ${{ job.target }}
+    #     extraEnvVars:
+    #       - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: hipRAND_test_${{ job.target }}
+    dependsOn: hipRAND_build_${{ job.target }}
+    condition:
+        and(succeeded(),
+          eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+          not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+          eq(${{ parameters.aggregatePipeline }}, False)
+        )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: hipRAND
+        testDir: '$(Agent.BuildDirectory)/rocm/bin/hipRAND'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/hipSOLVER.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -45,102 +50,117 @@ parameters:
     - rocSPARSE
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: hipSOLVER
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-# build external gtest and lapack
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      componentName: external
-      cmakeBuildDir: 'deps/build'
-      installDir: '$(Pipeline.Workspace)/deps-install'
-      extraBuildFlags: >-
-        -DBUILD_BOOST=OFF
-        -DBUILD_GTEST=OFF
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Pipeline.Workspace)/deps-install
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DBUILD_CLIENTS_TESTS=ON
-        -DUSE_CUDA=OFF
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraCopyDirectories:
-        - deps-install
 
-- job: hipSOLVER_testing
+jobs:
-  dependsOn: hipSOLVER
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: hipSOLVER_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    pool:
-  workspace:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    clean: all
+    workspace:
-  strategy:
+      clean: all
-    matrix:
+    steps:
-      gfx942:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-        JOB_GPU_TARGET: gfx942
+      parameters:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+        aptPackages: ${{ parameters.aptPackages }}
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
-    parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-    parameters:
+  # build external gtest and lapack
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        componentName: external
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        cmakeBuildDir: '$(Build.SourcesDirectory)/deps/build'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+        cmakeSourceDir: '$(Build.SourcesDirectory)/deps'
-    parameters:
+        installDir: '$(Pipeline.Workspace)/deps-install'
-      componentName: hipSOLVER
+        extraBuildFlags: >-
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+          -DBUILD_BOOST=OFF
-      testExecutable: './hipsolver-test'
+          -DBUILD_GTEST=OFF
-      testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+          -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        extraBuildFlags: >-
-      environment: test
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Pipeline.Workspace)/deps-install
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DCMAKE_BUILD_TYPE=Release
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+          -DBUILD_CLIENTS_TESTS=ON
+          -DUSE_CUDA=OFF
+          -GNinja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    #   parameters:
+    #     aptPackages: ${{ parameters.aptPackages }}
+    #     gpuTarget: ${{ job.target }}
+    #     extraCopyDirectories:
+    #       - deps-install
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: hipSOLVER_test_${{ job.target }}
+    dependsOn: hipSOLVER_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: hipSOLVER
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        testExecutable: './hipsolver-test'
+        testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/hipSPARSE.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -40,101 +45,115 @@ parameters:
     - rocSPARSE
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: hipSPARSE
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/share/rocm/cmake/
-        -DBUILD_CLIENTS_TESTS=ON
-        -DBUILD_CLIENTS_SAMPLES=OFF
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      artifactName: hipSPARSE
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      artifactName: hipSPARSE
-      gpuTarget: $(JOB_GPU_TARGET)
-      publish: false
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
-    parameters:
-      sourceDir: $(Build.SourcesDirectory)/build/clients
-      contentsString: matrices/**
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      artifactName: testMatrices
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      environment: test
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: hipSPARSE_testing
+jobs:
-  dependsOn: hipSPARSE
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: hipSPARSE_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    pool:
-  workspace:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    clean: all
+    workspace:
-  strategy:
+      clean: all
-    matrix:
+    steps:
-      gfx942:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-        JOB_GPU_TARGET: gfx942
+      parameters:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+        aptPackages: ${{ parameters.aptPackages }}
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
-    parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        extraBuildFlags: >-
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DCMAKE_BUILD_TYPE=Release
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/share/rocm/cmake/
-    parameters:
+          -DBUILD_CLIENTS_TESTS=ON
-      componentName: hipSPARSE
+          -DBUILD_CLIENTS_SAMPLES=OFF
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+          -GNinja
-      testExecutable: './hipsparse-test'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        artifactName: hipSPARSE
-    parameters:
+        gpuTarget: ${{ job.target }}
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-      environment: test
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        artifactName: hipSPARSE
+        gpuTarget: ${{ job.target }}
+        publish: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+      parameters:
+        sourceDir: $(Build.SourcesDirectory)/build/clients
+        contentsString: matrices/**
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        artifactName: testMatrices
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    #   parameters:
+    #     aptPackages: ${{ parameters.aptPackages }}
+    #     environment: test
+    #     gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: hipSPARSE_test_${{ job.target }}
+    dependsOn: hipSPARSE_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: hipSPARSE
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        testExecutable: './hipsparse-test'
+        testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/hipSPARSELt.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -28,6 +33,7 @@ parameters:
     - hipSPARSE
     - llvm-project
     - rocBLAS
+    - rocm_smi_lib
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
@@ -46,132 +52,142 @@ parameters:
     - ROCR-Runtime
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: hipSPARSELt
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  - name: HIP_ROCCLR_HOME
+        target: gfx942
-    value: $(Build.BinariesDirectory)/rocm
+    testJobs:
-  - name: TENSILE_ROCM_ASSEMBLER_PATH
+      - gfx942:
-    value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang
+        target: gfx942
-  - name: CMAKE_CXX_COMPILER
-    value: $(Agent.BuildDirectory)/rocm/llvm/bin/hipcc
-  - name: TENSILE_ROCM_OFFLOAD_BUNDLER_PATH
-    value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang-offload-bundler
-  - name: PATH
-    value: $(Agent.BuildDirectory)/rocm/llvm/bin:$(Agent.BuildDirectory)/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  workspace:
-    clean: all
-  strategy:
-    matrix:
-      gfx942:
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-# Build and install gtest and lapack
-# $(Pipeline.Workspace)/deps is a temporary folder for the build process
-# $(Pipeline.Workspace)/s/deps is part of the hipSPARSELt repo
-  - script: mkdir $(Pipeline.Workspace)/deps
-    displayName: Create temp folder for external dependencies
-# hipSPARSELt already has a CMake script for external deps, so we can just run that
-# https://github.com/ROCm/hipSPARSELt/blob/develop/deps/CMakeLists.txt
-  - script: cmake $(Pipeline.Workspace)/s/deps
-    displayName: Configure hipSPARSELt external dependencies
-    workingDirectory: $(Pipeline.Workspace)/deps
-  - script: make
-    displayName: Build hipSPARSELt external dependencies
-    workingDirectory: $(Pipeline.Workspace)/deps
-  - script: sudo make install
-    displayName: Install hipSPARSELt external dependencies
-    workingDirectory: $(Pipeline.Workspace)/deps
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DTensile_LOGIC=
-        -DTensile_CPU_THREADS=
-        -DTensile_LIBRARY_FORMAT=msgpack
-        -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DBUILD_CLIENTS_TESTS=ON
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraCopyDirectories:
-        - deps
-      extraPaths: /home/user/workspace/rocm/llvm/bin:/home/user/workspace/rocm/bin
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
-        - TENSILE_ROCM_ASSEMBLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang
-        - CMAKE_CXX_COMPILER:::/home/user/workspace/rocm/llvm/bin/hipcc
-        - TENSILE_ROCM_OFFLOAD_BUNDLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang-offload-bundler
-      installLatestCMake: true
 
-- job: hipSPARSELt_testing
+jobs:
-  dependsOn: hipSPARSELt
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: hipSPARSELt_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    - name: HIP_ROCCLR_HOME
-  workspace:
+      value: $(Build.BinariesDirectory)/rocm
-    clean: all
+    - name: TENSILE_ROCM_ASSEMBLER_PATH
-  strategy:
+      value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang
-    matrix:
+    - name: CMAKE_CXX_COMPILER
-      gfx942:
+      value: $(Agent.BuildDirectory)/rocm/llvm/bin/hipcc
-        JOB_GPU_TARGET: gfx942
+    - name: TENSILE_ROCM_OFFLOAD_BUNDLER_PATH
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+      value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang-offload-bundler
-  steps:
+    - name: PATH
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      value: $(Agent.BuildDirectory)/rocm/llvm/bin:$(Agent.BuildDirectory)/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
-    parameters:
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-      aptPackages: ${{ parameters.aptPackages }}
+    workspace:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    steps:
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        aptPackages: ${{ parameters.aptPackages }}
-    parameters:
+        pipModules: ${{ parameters.pipModules }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      componentName: hipSPARSELt
+      parameters:
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        checkoutRef: ${{ parameters.checkoutRef }}
-      testExecutable: './hipsparselt-test'
+        dependencyList: ${{ parameters.rocmDependencies }}
-      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-    parameters:
+  # Build and install gtest and lapack
-      aptPackages: ${{ parameters.aptPackages }}
+  # $(Pipeline.Workspace)/deps is a temporary folder for the build process
-      pipModules: ${{ parameters.pipModules }}
+  # $(Pipeline.Workspace)/s/deps is part of the hipSPARSELt repo
-      environment: test
+    - script: mkdir $(Pipeline.Workspace)/deps
-      gpuTarget: $(JOB_GPU_TARGET)
+      displayName: Create temp folder for external dependencies
+  # hipSPARSELt already has a CMake script for external deps, so we can just run that
+  # https://github.com/ROCm/hipSPARSELt/blob/develop/deps/CMakeLists.txt
+    - script: cmake $(Pipeline.Workspace)/s/deps
+      displayName: Configure hipSPARSELt external dependencies
+      workingDirectory: $(Pipeline.Workspace)/deps
+    - script: make
+      displayName: Build hipSPARSELt external dependencies
+      workingDirectory: $(Pipeline.Workspace)/deps
+    - script: sudo make install
+      displayName: Install hipSPARSELt external dependencies
+      workingDirectory: $(Pipeline.Workspace)/deps
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+      parameters:
+        extraBuildFlags: >-
+          -DCMAKE_BUILD_TYPE=Release
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+          -DCMAKE_Fortran_COMPILER=f95
+          -DAMDGPU_TARGETS=${{ job.target }}
+          -DTensile_LOGIC=
+          -DTensile_CPU_THREADS=
+          -DTensile_LIBRARY_FORMAT=msgpack
+          -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+          -DBUILD_CLIENTS_TESTS=ON
+          -GNinja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        gpuTarget: ${{ job.target }}
+        extraCopyDirectories:
+          - deps
+        extraPaths: /home/user/workspace/rocm/llvm/bin:/home/user/workspace/rocm/bin
+        extraEnvVars:
+          - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+          - TENSILE_ROCM_ASSEMBLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang
+          - CMAKE_CXX_COMPILER:::/home/user/workspace/rocm/llvm/bin/hipcc
+          - TENSILE_ROCM_OFFLOAD_BUNDLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang-offload-bundler
+        installLatestCMake: true
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: hipSPARSELt_test_${{ job.target }}
+    dependsOn: hipSPARSELt_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: hipSPARSELt
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        testExecutable: './hipsparselt-test'
+        testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/hipTensor.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -30,90 +35,103 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
-jobs:
+- name: jobMatrix
-- job: hipTensor
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/llvm
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_BUILD_TYPE=Release
-        -DHIPTENSOR_BUILD_TESTS=ON
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-      multithreadFlag: -- -j32
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: hipTensor_testing
+jobs:
-  timeoutInMinutes: 90
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  dependsOn: hipTensor
+  - job: hipTensor_build_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    variables:
-  variables:
+    - group: common
-  - group: common
+    - template: /.azuredevops/variables-global.yml
-  - template: /.azuredevops/variables-global.yml
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  pool: $(JOB_TEST_POOL)
+    workspace:
-  workspace:
+      clean: all
-    clean: all
+    steps:
-  strategy:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    matrix:
+      parameters:
-      gfx942:
+        aptPackages: ${{ parameters.aptPackages }}
-        JOB_GPU_TARGET: gfx942
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  steps:
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
-    parameters:
+        extraBuildFlags: >-
-      checkoutRef: ${{ parameters.checkoutRef }}
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/llvm
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DCMAKE_BUILD_TYPE=Release
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DHIPTENSOR_BUILD_TESTS=ON
-    parameters:
+          -DAMDGPU_TARGETS=${{ job.target }}
-      componentName: hipTensor
+          -GNinja
-      testDir: '$(Agent.BuildDirectory)/rocm/bin/hiptensor'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      testParameters: '-E ".*-extended" -VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml'
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      environment: test
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: hipTensor_test_${{ job.target }}
+    timeoutInMinutes: 90
+    dependsOn: hipTensor_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: hipTensor
+        testDir: '$(Agent.BuildDirectory)/rocm/bin/hiptensor'
+        testParameters: '-E ".*-extended" --output-on-failure --force-new-ctest-process --output-junit test_output.xml'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/hipfort.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -39,110 +44,123 @@ parameters:
     - rocSPARSE
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: hipfort
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/llvm
-        -DHIPFORT_INSTALL_DIR=$(Build.BinariesDirectory)
-        -DHIPFORT_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/flang
-        -DCMAKE_Fortran_FLAGS="-Mfree -fPIC"
-        -DCMAKE_Fortran_FLAGS_DEBUG=""
-        -DHIPFORT_COMPILER_FLAGS="-cpp"
-        -DHIPFORT_AR=$(Agent.BuildDirectory)/rocm/llvm/bin/llvm-ar
-        -DHIPFORT_RANLIB=$(Agent.BuildDirectory)/rocm/llvm/bin/llvm-ranlib
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DBUILD_TESTING=ON
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      installLatestCMake: true
 
-- job: hipfort_testing
+jobs:
-  dependsOn: hipfort
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: hipfort_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  workspace:
+    workspace:
-    clean: all
+      clean: all
-  strategy:
+    steps:
-    matrix:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      gfx942:
+      parameters:
-        JOB_GPU_TARGET: gfx942
+        aptPackages: ${{ parameters.aptPackages }}
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
-    parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmDependencies }}
+        extraBuildFlags: >-
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+          -DCMAKE_BUILD_TYPE=Release
-    parameters:
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/llvm
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+          -DHIPFORT_INSTALL_DIR=$(Build.BinariesDirectory)
-  - task: Bash@3
+          -DHIPFORT_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/flang
-    displayName: 'ROCm symbolic link'
+          -DCMAKE_Fortran_FLAGS="-Mfree -fPIC"
-    inputs:
+          -DCMAKE_Fortran_FLAGS_DEBUG=""
-      targetType: inline
+          -DHIPFORT_COMPILER_FLAGS="-cpp"
-      script: |
+          -DHIPFORT_AR=$(Agent.BuildDirectory)/rocm/llvm/bin/llvm-ar
-        sudo rm -rf /opt/rocm
+          -DHIPFORT_RANLIB=$(Agent.BuildDirectory)/rocm/llvm/bin/llvm-ranlib
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+          -DAMDGPU_TARGETS=${{ job.target }}
-      workingDirectory: $(Build.SourcesDirectory)
+          -DBUILD_TESTING=ON
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -GNinja
-  - task: Bash@3
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    displayName: 'Test hipfort'
+      parameters:
-    inputs:
+        gpuTarget: ${{ job.target }}
-      targetType: inline
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-      script: PATH=$(Agent.BuildDirectory)/rocm/bin:$PATH make run_all
+      parameters:
-      workingDirectory: $(Build.SourcesDirectory)/test
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      environment: test
+        aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
-      optSymLink: true
+        installLatestCMake: true
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: hipfort_test_${{ job.target }}
+    dependsOn: hipfort_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+      parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
+    - task: Bash@3
+      displayName: 'ROCm symbolic link'
+      inputs:
+        targetType: inline
+        script: |
+          # Assuming that /opt is no longer persistent across runs, test environments are fully ephemeral
+          sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        workingDirectory: $(Build.SourcesDirectory)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - task: Bash@3
+      displayName: 'Test hipfort'
+      inputs:
+        targetType: inline
+        script: PATH=$(Agent.BuildDirectory)/rocm/bin:$PATH make run_all
+        workingDirectory: $(Build.SourcesDirectory)/test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}
+        optSymLink: true

.azuredevops/components/llvm-project.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -29,7 +34,7 @@ jobs:
     value: '$(Build.BinariesDirectory)/amdgcn/bitcode'
   - name: HIP_PATH
     value: '$(Agent.BuildDirectory)/rocm'
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  pool: ${{ variables.ULTRA_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -45,13 +50,14 @@ jobs:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       skipLlvmSymlink: true
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: rocm-llvm
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH="$(Build.BinariesDirectory)/llvm;$(Build.BinariesDirectory)"
         -DCMAKE_BUILD_TYPE=Release
-        -DLLVM_ENABLE_PROJECTS=clang;lld;clang-tools-extra;mlir
+        -DLLVM_ENABLE_PROJECTS=clang;lld;clang-tools-extra;mlir;flang
         -DLLVM_ENABLE_RUNTIMES=compiler-rt;libunwind;libcxx;libcxxabi
         -DCLANG_ENABLE_AMDCLANG=ON
         -DLLVM_TARGETS_TO_BUILD=AMDGPU;X86
@@ -77,7 +83,8 @@ jobs:
         -DROCM_LLVM_BACKWARD_COMPAT_LINK=$(Build.BinariesDirectory)/llvm
         -DROCM_LLVM_BACKWARD_COMPAT_LINK_TARGET=./lib/llvm
         -GNinja
-      cmakeBuildDir: 'llvm/build'
+      cmakeBuildDir: '$(Build.SourcesDirectory)/llvm/build'
+      cmakeSourceDir: '$(Build.SourcesDirectory)/llvm'
       installDir: '$(Build.BinariesDirectory)/llvm'
 # use llvm-lit to run unit tests for llvm, clang, and lld
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
@@ -85,7 +92,7 @@ jobs:
       componentName: check-llvm
       testDir: 'llvm/build'
       testExecutable: './bin/llvm-lit'
-      testParameters: '-q --xunit-xml-output=llvm_test_output.xml ./test'
+      testParameters: '-q --xunit-xml-output=llvm_test_output.xml --filter-out="live-debug-values-spill-tracking" ./test'
       testOutputFile: llvm_test_output.xml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
@@ -115,7 +122,8 @@ jobs:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH="$(Build.SourcesDirectory)/llvm/build"
         -DCMAKE_BUILD_TYPE=Release
-      cmakeBuildDir: 'amd/device-libs/build'
+      cmakeBuildDir: '$(Build.SourcesDirectory)/amd/device-libs/build'
+      cmakeSourceDir: '$(Build.SourcesDirectory)/amd/device-libs'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: comgr
@@ -123,7 +131,8 @@ jobs:
         -DCMAKE_PREFIX_PATH="$(Build.SourcesDirectory)/llvm/build;$(Build.SourcesDirectory)/amd/device-libs/build"
         -DCOMGR_DISABLE_SPIRV=1
         -DCMAKE_BUILD_TYPE=Release
-      cmakeBuildDir: 'amd/comgr/build'
+      cmakeBuildDir: '$(Build.SourcesDirectory)/amd/comgr/build'
+      cmakeSourceDir: '$(Build.SourcesDirectory)/amd/comgr'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: comgr
@@ -136,7 +145,8 @@ jobs:
       extraBuildFlags: >-
         -DCMAKE_BUILD_TYPE=Release
         -DHIPCC_BACKWARD_COMPATIBILITY=OFF
-      cmakeBuildDir: 'amd/hipcc/build'
+      cmakeBuildDir: '$(Build.SourcesDirectory)/amd/hipcc/build'
+      cmakeSourceDir: '$(Build.SourcesDirectory)/amd/hipcc'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml

.azuredevops/components/omnitrace.yml

@@ -1,140 +0,0 @@
-# largely referenced from: https://github.com/ROCm/omnitrace/blob/main/.github/workflows/ubuntu-jammy.yml
-parameters:
-- name: checkoutRepo
-  type: string
-  default: 'self'
-- name: checkoutRef
-  type: string
-  default: ''
-- name: aptPackages
-  type: object
-  default:
-    - autoconf
-    - autotools-dev
-    - bison
-    - build-essential
-    - bzip2
-    - clang
-    - cmake
-    - environment-modules
-    - g++-12
-    - libdrm-dev
-    - libfabric-dev
-    - libiberty-dev
-    - libpapi-dev
-    - libpfm4-dev
-    - libtool
-    - libopenmpi-dev
-    - m4
-    - openmpi-bin
-    - software-properties-common
-    - python3-pip
-    - texinfo
-    - zlib1g-dev
-- name: pipModules
-  type: object
-  default:
-    - numpy
-    - perfetto
-    - dataclasses
-- name: rocmDependencies
-  type: object
-  default:
-    - aomp
-    - clr
-    - llvm-project
-    - rccl
-    - rocm-core
-    - rocm_smi_lib
-    - rocminfo
-    - ROCR-Runtime
-    - rocprofiler
-    - rocprofiler-register
-    - roctracer
-
-jobs:
-- job: omnitrace
-  variables:
-  - group: common
-  - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  workspace:
-    clean: all
-  strategy:
-    matrix:
-      gfx942:
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - task: Bash@3
-    displayName: ROCm symbolic link
-    inputs:
-      targetType: inline
-      script: |
-        sudo rm -rf /opt/rocm
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-  - task: Bash@3
-    displayName: Add ROCm binaries to PATH
-    inputs:
-      targetType: inline
-      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
-  - task: Bash@3
-    displayName: Add ROCm compilers to PATH
-    inputs:
-      targetType: inline
-      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-# build flags reference: https://rocm.docs.amd.com/projects/omnitrace/en/latest/install/install.html
-      extraBuildFlags: >-
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DOMNITRACE_BUILD_TESTING=ON
-        -DOMNITRACE_BUILD_DYNINST=ON
-        -DOMNITRACE_BUILD_LIBUNWIND=ON
-        -DDYNINST_BUILD_TBB=ON
-        -DDYNINST_BUILD_ELFUTILS=ON
-        -DDYNINST_BUILD_LIBIBERTY=ON
-        -DDYNINST_BUILD_BOOST=ON
-        -DOMNITRACE_USE_PAPI=ON
-        -DOMNITRACE_USE_MPI=ON
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-      multithreadFlag: -- -j32
-  - task: Bash@3
-    displayName: Set up omnitrace env
-    inputs:
-      targetType: inline
-      script: source share/omnitrace/setup-env.sh
-      workingDirectory: build
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-    parameters:
-      componentName: omnitrace
-  - task: Bash@3
-    displayName: Remove ROCm binaries from PATH
-    condition: always()
-    inputs:
-      targetType: inline
-      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
-  - task: Bash@3
-    displayName: Remove ROCm compilers from PATH
-    condition: always()
-    inputs:
-      targetType: inline
-      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rccl.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -49,101 +54,114 @@ parameters:
     - ROCR-Runtime
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: rccl
+  type: object
-  timeoutInMinutes: 90
+  default:
-  variables:
+    buildJobs:
-  - group: common
+      - gfx942:
-  - template: /.azuredevops/variables-global.yml
+        target: gfx942
-  - name: HIP_ROCCLR_HOME
+      - gfx90a:
-    value: $(Build.BinariesDirectory)/rocm
+        target: gfx90a
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+    testJobs:
-  workspace:
+      - gfx942:
-    clean: all
+        target: gfx942
-  strategy:
+      - gfx90a:
-    matrix:
+        target: gfx90a
-      gfx942:
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-      submoduleBehaviour: recursive
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-        -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
-        -DCMAKE_BUILD_TYPE=Release
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DBUILD_TESTS=ON
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/share/rocm/cmake;$(Agent.BuildDirectory)/rocm/libexec/hipify
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
-      installLatestCMake: true
 
-- job: rccl_testing
+jobs:
-  timeoutInMinutes: 120
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  dependsOn: rccl
+  - job: rccl_build_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    timeoutInMinutes: 90
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    - name: HIP_ROCCLR_HOME
-  workspace:
+      value: $(Build.BinariesDirectory)/rocm
-    clean: all
+    pool: ${{ variables.HIGH_BUILD_POOL }}
-  strategy:
+    workspace:
-    matrix:
+      clean: all
-      gfx942:
+    steps:
-        JOB_GPU_TARGET: gfx942
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+      parameters:
-  steps:
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+        submoduleBehaviour: recursive
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-    parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+        gpuTarget: ${{ job.target }}
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+        extraBuildFlags: >-
-    parameters:
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-      componentName: rccl
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+          -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
-      testExecutable: './rccl-UnitTests'
+          -DCMAKE_BUILD_TYPE=Release
-      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -DBUILD_TESTS=ON
-    parameters:
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/share/rocm/cmake;$(Agent.BuildDirectory)/rocm/libexec/hipify
-      aptPackages: ${{ parameters.aptPackages }}
+          -DAMDGPU_TARGETS=${{ job.target }}
-      environment: test
+          -GNinja
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+        extraEnvVars:
+          - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+        installLatestCMake: true
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rccl_test_${{ job.target }}
+    timeoutInMinutes: 120
+    dependsOn: rccl_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rccl
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        testExecutable: './rccl-UnitTests'
+        testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rdc.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -53,123 +58,138 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
-jobs:
+- name: jobMatrix
-- job: rdc
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-# Build grpc
-  - task: Bash@3
-    displayName: 'git clone grpc'
-    inputs:
-      targetType: inline
-      script: git clone -b v1.61.0 https://github.com/grpc/grpc --depth=1 --shallow-submodules --recurse-submodules
-      workingDirectory: $(Build.SourcesDirectory)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      cmakeBuildDir: $(Build.SourcesDirectory)/grpc/build
-      installDir: $(Build.SourcesDirectory)/bin
-      extraBuildFlags: >-
-        -DgRPC_INSTALL=ON
-        -DgRPC_BUILD_TESTS=OFF
-        -DBUILD_SHARED_LIBS=ON
-        -DCMAKE_INSTALL_LIBDIR=lib
-        -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DGRPC_ROOT="$(Build.SourcesDirectory)/bin"
-        -DBUILD_RVS=ON
-        -DBUILD_PROFILER=ON
-        -DBUILD_TESTS=ON
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: rdc_testing
+jobs:
-  dependsOn: rdc
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rdc_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  workspace:
+    workspace:
-    clean: all
+      clean: all
-  strategy:
+    steps:
-    matrix:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      gfx942:
+      parameters:
-        JOB_GPU_TARGET: gfx942
+        aptPackages: ${{ parameters.aptPackages }}
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
-    parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-    parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+  # Build grpc
-    parameters:
+    - task: Bash@3
-      checkoutRef: ${{ parameters.checkoutRef }}
+      displayName: 'git clone grpc'
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+      inputs:
-      gpuTarget: $(JOB_GPU_TARGET)
+        targetType: inline
-  - task: Bash@3
+        script: git clone -b v1.67.1 https://github.com/grpc/grpc --depth=1 --shallow-submodules --recurse-submodules
-    displayName: Setup test environment
+        workingDirectory: $(Build.SourcesDirectory)
-    inputs:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      targetType: inline
+      parameters:
-      script: |
+        cmakeBuildDir: $(Build.SourcesDirectory)/grpc/build
-        sudo rm -rf /opt/rocm
+        cmakeSourceDir: $(Build.SourcesDirectory)/grpc
-        sudo rm -rf /usr/sbin/rdcd
+        installDir: $(Build.SourcesDirectory)/bin
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        extraBuildFlags: >-
-        sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rdcd /usr/sbin/rdcd
+          -DgRPC_INSTALL=ON
-        echo $(Agent.BuildDirectory)/rocm/lib/rdc/grpc/lib | sudo tee /etc/ld.so.conf.d/grpc.conf
+          -DgRPC_BUILD_TESTS=OFF
-        sudo ldconfig -v
+          -DBUILD_SHARED_LIBS=ON
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DCMAKE_INSTALL_LIBDIR=lib
-  - task: Bash@3
+          -DCMAKE_BUILD_TYPE=Release
-    displayName: Test rdc
+          -DAMDGPU_TARGETS=${{ job.target }}
-    inputs:
+          -GNinja
-      targetType: inline
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      script: >-
+      parameters:
-        $(Agent.BuildDirectory)/rocm/share/rdc/rdctst_tests/rdctst
+        extraBuildFlags: >-
-        --batch_mode
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        --start_rdcd
+          -DGRPC_ROOT="$(Build.SourcesDirectory)/bin"
-        --unauth_comm
+          -DBUILD_RVS=ON
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -DBUILD_PROFILER=ON
-    parameters:
+          -DBUILD_TESTS=ON
-      aptPackages: ${{ parameters.aptPackages }}
+          -DAMDGPU_TARGETS=${{ job.target }}
-      environment: test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-      extraPaths: /home/user/workspace/rocm/bin
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rdc_test_${{ job.target }}
+    dependsOn: rdc_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    - name: ROCM_PATH
+      value: $(Agent.BuildDirectory)/rocm
+    - name: ROCM_DIR
+      value: $(Agent.BuildDirectory)/rocm
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - task: Bash@3
+      displayName: Setup test environment
+      inputs:
+        targetType: inline
+        script: |
+          sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rdcd /usr/sbin/rdcd
+          echo $(Agent.BuildDirectory)/rocm/lib/rdc/grpc/lib | sudo tee /etc/ld.so.conf.d/grpc.conf
+          sudo ldconfig -v
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - task: Bash@3
+      displayName: Test rdc
+      inputs:
+        targetType: inline
+        script: >-
+          $(Agent.BuildDirectory)/rocm/share/rdc/rdctst_tests/rdctst
+          --batch_mode
+          --start_rdcd
+          --unauth_comm
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}
+        extraPaths: /home/user/workspace/rocm/bin

.azuredevops/components/rocAL.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -61,188 +66,203 @@ parameters:
     - ROCR-Runtime
     - rpp
 
-jobs:
+- name: jobMatrix
-- job: rocAL
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - task: Bash@3
-    displayName: 'Register libjpeg-turbo packages'
-    inputs:
-      targetType: inline
-      script: |
-        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
-        wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
-        echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
-        sudo apt update
-        apt-cache show libjpeg-turbo-official | grep Version
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - task: Bash@3
-    displayName: 'Clone PyBind11'
-    inputs:
-      targetType: inline
-      script: git clone --depth 1 -b v2.11.1 https://github.com/pybind/pybind11
-      workingDirectory: '$(Build.SourcesDirectory)'
-  - task: Bash@3
-    displayName: 'Clone RapidJSON'
-    inputs:
-      targetType: inline
-      script: git clone --depth 1 https://github.com/Tencent/rapidjson.git
-      workingDirectory: '$(Build.SourcesDirectory)'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      componentName: PyBind11
-      cmakeBuildDir: '$(Build.SourcesDirectory)/pybind11/build'
-      customInstallPath: false
-      installEnabled: false
-      extraBuildFlags: >-
-        -DDOWNLOAD_CATCH=ON
-        -DDOWNLOAD_EIGEN=ON
-        -GNinja
-  - task: Bash@3
-    displayName: 'Install PyBind11'
-    inputs:
-      targetType: inline
-      script: sudo cmake --build . --target install
-      workingDirectory: '$(Build.SourcesDirectory)/pybind11/build'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      componentName: RapidJSON
-      cmakeBuildDir: '$(Build.SourcesDirectory)/rapidjson/build'
-      customInstallPath: false
-      installEnabled: false
-      extraBuildFlags: >-
-        -GNinja
-  - task: Bash@3
-    displayName: 'Install RapidJSON'
-    inputs:
-      targetType: inline
-      script: sudo cmake --build . --target install
-      workingDirectory: '$(Build.SourcesDirectory)/rapidjson/build'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;/opt/libjpeg-turbo
-        -DCMAKE_INSTALL_PREFIX_PYTHON=$Python3_STDARCH
-        -DCMAKE_BUILD_TYPE=Release
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      registerJPEGPackages: true
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraCopyDirectories:
-        - /opt/libjpeg-turbo
 
-- job: rocAL_testing
+jobs:
-  dependsOn: rocAL
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rocAL_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  - name: ROCM_PATH
+    pool:
-    value: $(Agent.BuildDirectory)/rocm
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-  - name: CMAKE_INCLUDE_PATH
+    workspace:
-    value: $(Agent.BuildDirectory)/rocm/include/rocal
+      clean: all
-  pool:
+    steps:
-    name: $(JOB_TEST_POOL)
+    - task: Bash@3
-    demands: firstRenderDeviceAccess
+      displayName: 'Register libjpeg-turbo packages'
-  workspace:
+      inputs:
-    clean: all
+        targetType: inline
-  strategy:
+        script: |
-    matrix:
+          sudo mkdir --parents --mode=0755 /etc/apt/keyrings
-      gfx942:
+          wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
-        JOB_GPU_TARGET: gfx942
+          echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+          sudo apt update
-  steps:
+          apt-cache show libjpeg-turbo-official | grep Version
-  - task: Bash@3
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    displayName: 'Register libjpeg-turbo packages'
+      parameters:
-    inputs:
+        aptPackages: ${{ parameters.aptPackages }}
-      targetType: inline
+        pipModules: ${{ parameters.pipModules }}
-      script: |
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-        wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
+      parameters:
-        echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-        sudo apt update
+    - task: Bash@3
-        apt-cache show libjpeg-turbo-official | grep Version
+      displayName: 'Clone PyBind11'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      inputs:
-    parameters:
+        targetType: inline
-      aptPackages: ${{ parameters.aptPackages }}
+        script: git clone --depth 1 -b v2.11.1 https://github.com/pybind/pybind11
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        workingDirectory: '$(Build.SourcesDirectory)'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - task: Bash@3
-    parameters:
+      displayName: 'Clone RapidJSON'
-      gpuTarget: $(JOB_GPU_TARGET)
+      inputs:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        targetType: inline
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        script: git clone --depth 1 https://github.com/Tencent/rapidjson.git
-    parameters:
+        workingDirectory: '$(Build.SourcesDirectory)'
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        componentName: PyBind11
-  - task: Bash@3
+        cmakeBuildDir: '$(Build.SourcesDirectory)/pybind11/build'
-    displayName: Link libjpeg-turbo
+        cmakeSourceDir: '$(Build.SourcesDirectory)/pybind11'
-    inputs:
+        customInstallPath: false
-      targetType: inline
+        installEnabled: false
-      script: |
+        extraBuildFlags: >-
-        echo /opt/libjpeg-turbo/lib64 | sudo tee /etc/ld.so.conf.d/libjpeg-turbo.conf
+          -DDOWNLOAD_CATCH=ON
-        sudo ldconfig -v
+          -DDOWNLOAD_EIGEN=ON
-  - task: Bash@3
+          -GNinja
-    displayName: Build rocAL tests
+    - task: Bash@3
-    inputs:
+      displayName: 'Install PyBind11'
-      targetType: inline
+      inputs:
-      script: |
+        targetType: inline
-        mkdir rocAL-tests
+        script: sudo cmake --build . --target install
-        cd rocAL-tests
+        workingDirectory: '$(Build.SourcesDirectory)/pybind11/build'
-        cmake $(Agent.BuildDirectory)/rocm/share/rocal/test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+        componentName: RapidJSON
-    parameters:
+        cmakeBuildDir: '$(Build.SourcesDirectory)/rapidjson/build'
-      componentName: rocAL
+        cmakeSourceDir: '$(Build.SourcesDirectory)/rapidjson'
-      testDir: rocAL-tests
+        customInstallPath: false
-  - task: Bash@3
+        installEnabled: false
-    displayName: Clean up libjpeg-turbo
+        extraBuildFlags: >-
-    condition: always()
+          -GNinja
-    inputs:
+    - task: Bash@3
-      targetType: inline
+      displayName: 'Install RapidJSON'
-      script: |
+      inputs:
-        sudo rm /etc/ld.so.conf.d/libjpeg-turbo.conf
+        targetType: inline
-        sudo ldconfig -v
+        script: sudo cmake --build . --target install
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        workingDirectory: '$(Build.SourcesDirectory)/rapidjson/build'
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      pipModules: ${{ parameters.pipModules }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-      registerJPEGPackages: true
+        dependencyList: ${{ parameters.rocmDependencies }}
-      environment: test
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      extraCopyDirectories:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-        - /opt/libjpeg-turbo
+      parameters:
-# docker image will be missing the ldconfig to libjpeg-turbo
+        extraBuildFlags: >-
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;/opt/libjpeg-turbo
+          -DCMAKE_INSTALL_PREFIX_PYTHON=$Python3_STDARCH
+          -DCMAKE_BUILD_TYPE=Release
+          -DGPU_TARGETS=${{ job.target }}
+          -GNinja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    #   parameters:
+    #     aptPackages: ${{ parameters.aptPackages }}
+    #     pipModules: ${{ parameters.pipModules }}
+    #     registerJPEGPackages: true
+    #     gpuTarget: ${{ job.target }}
+    #     extraCopyDirectories:
+    #       - /opt/libjpeg-turbo
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocAL_test_${{ job.target }}
+    dependsOn: rocAL_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    - name: ROCM_PATH
+      value: $(Agent.BuildDirectory)/rocm
+    - name: CMAKE_INCLUDE_PATH
+      value: $(Agent.BuildDirectory)/rocm/include/rocal
+    pool:
+      name: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - task: Bash@3
+      displayName: 'Register libjpeg-turbo packages'
+      inputs:
+        targetType: inline
+        script: |
+          sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+          wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
+          echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
+          sudo apt update
+          apt-cache show libjpeg-turbo-official | grep Version
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - task: Bash@3
+      displayName: Link libjpeg-turbo
+      inputs:
+        targetType: inline
+        script: |
+          echo /opt/libjpeg-turbo/lib64 | sudo tee /etc/ld.so.conf.d/libjpeg-turbo.conf
+          sudo ldconfig -v
+    - task: Bash@3
+      displayName: Build rocAL tests
+      inputs:
+        targetType: inline
+        script: |
+          mkdir rocAL-tests
+          cd rocAL-tests
+          cmake $(Agent.BuildDirectory)/rocm/share/rocal/test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocAL
+        testDir: rocAL-tests
+    - task: Bash@3
+      displayName: Clean up libjpeg-turbo
+      condition: always()
+      inputs:
+        targetType: inline
+        script: |
+          sudo rm /etc/ld.so.conf.d/libjpeg-turbo.conf
+          sudo ldconfig -v
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        registerJPEGPackages: true
+        environment: test
+        gpuTarget: ${{ job.target }}
+        extraCopyDirectories:
+          - /opt/libjpeg-turbo
+  # docker image will be missing the ldconfig to libjpeg-turbo

.azuredevops/components/rocALUTION.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -46,96 +51,110 @@ parameters:
     - rocSPARSE
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: rocALUTION
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  - name: HIP_ROCCLR_HOME
+        target: gfx942
-    value: $(Build.BinariesDirectory)/rocm
+      - gfx90a:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx90a
-  workspace:
+    testJobs:
-    clean: all
+      - gfx942:
-  strategy:
+        target: gfx942
-    matrix:
+      - gfx90a:
-      gfx942:
+        target: gfx90a
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/share/rocm/cmake/
-        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DBUILD_CLIENTS_TESTS=ON
-        -DBUILD_CLIENTS_BENCHMARKS=OFF
-        -DBUILD_CLIENTS_SAMPLES=OFF
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
 
-- job: rocALUTION_testing
+jobs:
-  dependsOn: rocALUTION
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rocALUTION_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    - name: HIP_ROCCLR_HOME
-  workspace:
+      value: $(Build.BinariesDirectory)/rocm
-    clean: all
+    pool:
-  strategy:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    matrix:
+    workspace:
-      gfx942:
+      clean: all
-        JOB_GPU_TARGET: gfx942
+    steps:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  steps:
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+        aptPackages: ${{ parameters.aptPackages }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        extraBuildFlags: >-
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-    parameters:
+          -DCMAKE_BUILD_TYPE=Release
-      componentName: rocALUTION
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/share/rocm/cmake/
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+          -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-      testExecutable: './rocalution-test'
+          -DAMDGPU_TARGETS=${{ job.target }}
-      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+          -DBUILD_CLIENTS_TESTS=ON
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -DBUILD_CLIENTS_BENCHMARKS=OFF
-    parameters:
+          -DBUILD_CLIENTS_SAMPLES=OFF
-      aptPackages: ${{ parameters.aptPackages }}
+          -GNinja
-      environment: test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    #   parameters:
+    #     aptPackages: ${{ parameters.aptPackages }}
+    #     gpuTarget: ${{ job.target }}
+    #     extraEnvVars:
+    #       - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocALUTION_test_${{ job.target }}
+    dependsOn: rocALUTION_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocALUTION
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        testExecutable: './rocalution-test'
+        testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocBLAS.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -55,118 +60,131 @@ parameters:
     - ROCR-Runtime
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: rocBLAS
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  - name: HIP_ROCCLR_HOME
+        target: gfx942
-    value: $(Build.BinariesDirectory)/rocm
+      - gfx90a:
-  - name: TENSILE_ROCM_ASSEMBLER_PATH
+        target: gfx90a
-    value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang
+    testJobs:
-  - name: TENSILE_ROCM_OFFLOAD_BUNDLER_PATH
+      - gfx942:
-    value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang-offload-bundler
+        target: gfx942
-  - name: ROCM_PATH
+      - gfx90a:
-    value: $(Agent.BuildDirectory)/rocm
+        target: gfx90a
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  workspace:
-    clean: all
-  strategy:
-    matrix:
-      gfx942:
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_TOOLCHAIN_FILE=toolchain-linux.cmake
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/amdclang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/amdclang
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DTensile_CODE_OBJECT_VERSION=default
-        -DTensile_LOGIC=asm_full
-        -DTensile_SEPARATE_ARCHITECTURES=ON
-        -DTensile_LAZY_LIBRARY_LOADING=ON
-        -DTensile_LIBRARY_FORMAT=msgpack
-        -DBUILD_CLIENTS_TESTS=ON
-        -DBUILD_CLIENTS_BENCHMARKS=OFF
-        -DBUILD_CLIENTS_SAMPLES=OFF
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      installAOCL: true
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
-        - TENSILE_ROCM_ASSEMBLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang
-        - TENSILE_ROCM_OFFLOAD_BUNDLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang-offload-bundler
-        - ROCM_PATH:::/home/user/workspace/rocm
 
-- job: rocBLAS_testing
+jobs:
-  dependsOn: rocBLAS
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rocBLAS_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    - name: HIP_ROCCLR_HOME
-  workspace:
+      value: $(Build.BinariesDirectory)/rocm
-    clean: all
+    - name: TENSILE_ROCM_ASSEMBLER_PATH
-  strategy:
+      value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang
-    matrix:
+    - name: TENSILE_ROCM_OFFLOAD_BUNDLER_PATH
-      gfx942:
+      value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang-offload-bundler
-        JOB_GPU_TARGET: gfx942
+    - name: ROCM_PATH
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+      value: $(Agent.BuildDirectory)/rocm
-  steps:
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    workspace:
-    parameters:
+      clean: all
-      aptPackages: ${{ parameters.aptPackages }}
+    steps:
-      pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        aptPackages: ${{ parameters.aptPackages }}
-    parameters:
+        pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
-    parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      componentName: rocBLAS
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      testExecutable: './rocblas-test'
+      parameters:
-      testParameters: '--yaml rocblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'
+        extraBuildFlags: >-
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -DCMAKE_TOOLCHAIN_FILE=toolchain-linux.cmake
-    parameters:
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
-      aptPackages: ${{ parameters.aptPackages }}
+          -DCMAKE_BUILD_TYPE=Release
-      pipModules: ${{ parameters.pipModules }}
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/amdclang++
-      environment: test
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/amdclang
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DGPU_TARGETS=${{ job.target }}
+          -DTensile_CODE_OBJECT_VERSION=default
+          -DTensile_LOGIC=asm_full
+          -DTensile_SEPARATE_ARCHITECTURES=ON
+          -DTensile_LAZY_LIBRARY_LOADING=ON
+          -DTensile_LIBRARY_FORMAT=msgpack
+          -DBUILD_CLIENTS_TESTS=ON
+          -DBUILD_CLIENTS_BENCHMARKS=OFF
+          -DBUILD_CLIENTS_SAMPLES=OFF
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+          -GNinja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        installAOCL: true
+        gpuTarget: ${{ job.target }}
+        extraEnvVars:
+          - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+          - TENSILE_ROCM_ASSEMBLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang
+          - TENSILE_ROCM_OFFLOAD_BUNDLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang-offload-bundler
+          - ROCM_PATH:::/home/user/workspace/rocm
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocBLAS_test_${{ job.target }}
+    dependsOn: rocBLAS_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocBLAS
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        testExecutable: './rocblas-test'
+        testParameters: '--yaml rocblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocDecode.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -40,12 +45,24 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
+- name: jobMatrix
+  type: object
+  default:
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 jobs:
-- job: rocDecode
+- job: rocDecode_build
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  - name: ROCM_PATH
+    value: $(Agent.BuildDirectory)/rocm
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -61,68 +78,67 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
-      registerROCmPackages: true
+  #     registerROCmPackages: true
 
-- job: rocDecode_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  dependsOn: rocDecode
+  - job: rocDecode_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: rocDecode_build
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool:
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-    name: $(JOB_TEST_POOL)
+        eq(${{ parameters.aggregatePipeline }}, False)
-    demands: firstRenderDeviceAccess
+      )
-  workspace:
+    variables:
-    clean: all
+    - group: common
-  strategy:
+    - template: /.azuredevops/variables-global.yml
-    matrix:
+    - name: ROCM_PATH
-      gfx942:
+      value: $(Agent.BuildDirectory)/rocm
-        JOB_GPU_TARGET: gfx942
+    pool:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+      name: ${{ job.target }}_test_pool
-  steps:
+    workspace:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      clean: all
-    parameters:
+    steps:
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      registerROCmPackages: true
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        registerROCmPackages: true
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        checkoutRef: ${{ parameters.checkoutRef }}
-  # anything in /opt may be persistent across runs
+        dependencyList: ${{ parameters.rocmTestDependencies }}
-  # so we need to remove the symlink if it already exists
+        gpuTarget: ${{ job.target }}
-  - script: |
+    - task: Bash@3
-      sudo rm -rf /opt/rocm
+      displayName: Build rocDecode tests
-      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      inputs:
-      mkdir rocDecode-tests
+        targetType: inline
-      cd rocDecode-tests
+        script: |
-      cmake /opt/rocm/share/rocdecode/test
+          mkdir rocDecode-tests
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          cd rocDecode-tests
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          cmake $(Agent.BuildDirectory)/rocm/share/rocdecode/test
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-      componentName: rocDecode
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-      testDir: 'rocDecode-tests'
+      parameters:
-  - script: sudo rm /opt/rocm
+        componentName: rocDecode
-    condition: always()
+        testDir: 'rocDecode-tests'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        aptPackages: ${{ parameters.aptPackages }}
-      registerROCmPackages: true
+        registerROCmPackages: true
-      environment: test
+        environment: test
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocFFT.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -41,98 +46,111 @@ parameters:
     - ROCR-Runtime
     - rocRAND
 
-jobs:
+- name: jobMatrix
-- job: rocFFT
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  - name: HIP_ROCCLR_HOME
+        target: gfx942
-    value: $(Build.BinariesDirectory)/rocm
+      - gfx90a:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx90a
-  workspace:
+    testJobs:
-    clean: all
+      - gfx942:
-  strategy:
+        target: gfx942
-    matrix:
+      - gfx90a:
-      gfx942:
+        target: gfx90a
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DUSE_HIP_CLANG=ON
-        -DHIP_COMPILER=clang
-        -DBUILD_CLIENTS_TESTS=ON
-        -DBUILD_CLIENTS_BENCHMARKS=OFF
-        -DBUILD_CLIENTS_SAMPLES=OFF
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
 
-- job: rocFFT_testing
+jobs:
-  dependsOn: rocFFT
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rocFFT_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    - name: HIP_ROCCLR_HOME
-  workspace:
+      value: $(Build.BinariesDirectory)/rocm
-    clean: all
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  strategy:
+    workspace:
-    matrix:
+      clean: all
-      gfx942:
+    steps:
-        JOB_GPU_TARGET: gfx942
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+      parameters:
-  steps:
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        extraBuildFlags: >-
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-    parameters:
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-      componentName: rocFFT
+          -DCMAKE_BUILD_TYPE=Release
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+          -DAMDGPU_TARGETS=${{ job.target }}
-      testExecutable: './rocfft-test'
+          -DUSE_HIP_CLANG=ON
-      testParameters: '--test_prob 0.004 --gtest_output=xml:./test_output.xml --gtest_color=yes'
+          -DHIP_COMPILER=clang
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -DBUILD_CLIENTS_TESTS=ON
-    parameters:
+          -DBUILD_CLIENTS_BENCHMARKS=OFF
-      aptPackages: ${{ parameters.aptPackages }}
+          -DBUILD_CLIENTS_SAMPLES=OFF
-      environment: test
+          -GNinja
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+        extraEnvVars:
+          - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocFFT_test_${{ job.target }}
+    dependsOn: rocFFT_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocFFT
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        testExecutable: './rocfft-test'
+        testParameters: '--test_prob 0.004 --gtest_output=xml:./test_output.xml --gtest_color=yes'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocJPEG.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -35,97 +40,113 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
-jobs:
+- name: jobMatrix
-- job: rocJPEG
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      registerROCmPackages: true
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_BUILD_TYPE=Release
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      registerROCmPackages: true
 
-- job: rocJPEG_testing
+jobs:
-  dependsOn: rocJPEG
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rocJPEG_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool:
+    - name: ROCM_PATH
-    name: $(JOB_TEST_POOL)
+      value: $(Agent.BuildDirectory)/rocm
-    demands: firstRenderDeviceAccess
+    pool:
-  workspace:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    clean: all
+    workspace:
-  strategy:
+      clean: all
-    matrix:
+    steps:
-      gfx942:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-        JOB_GPU_TARGET: gfx942
+      parameters:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+        aptPackages: ${{ parameters.aptPackages }}
-  steps:
+        registerROCmPackages: true
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      registerROCmPackages: true
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  # anything in /opt may be persistent across runs
+        extraBuildFlags: >-
-  # so we need to remove the symlink if it already exists
+          -DCMAKE_BUILD_TYPE=Release
-  - script: |
+          -DGPU_TARGETS=${{ job.target }}
-      sudo rm -rf /opt/rocm
+          -GNinja
-      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      mkdir rocJPEG-tests
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-      cd rocJPEG-tests
+      parameters:
-      cmake $(Agent.BuildDirectory)/rocm/share/rocjpeg/test
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+    #   parameters:
-      componentName: rocJPEG
+    #     aptPackages: ${{ parameters.aptPackages }}
-      testDir: 'rocJPEG-tests'
+    #     gpuTarget: ${{ job.target }}
-  - script: sudo rm /opt/rocm
+    #     registerROCmPackages: true
-    condition: always()
+
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-    parameters:
+  - job: rocJPEG_test_${{ job.target }}
-      aptPackages: ${{ parameters.aptPackages }}
+    dependsOn: rocJPEG_build_${{ job.target }}
-      environment: test
+    condition:
-      gpuTarget: $(JOB_GPU_TARGET)
+      and(succeeded(),
-      registerROCmPackages: true
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-      optSymLink: true
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    - name: ROCM_PATH
+      value: $(Agent.BuildDirectory)/rocm
+    pool:
+      name: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        registerROCmPackages: true
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - task: Bash@3
+      displayName: Build rocJPEG tests
+      inputs:
+        targetType: inline
+        script: |
+          mkdir rocJPEG-tests
+          cd rocJPEG-tests
+          cmake $(Agent.BuildDirectory)/rocm/share/rocjpeg/test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocJPEG
+        testDir: 'rocJPEG-tests'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}
+        registerROCmPackages: true

.azuredevops/components/rocMLIR.yml

@@ -5,31 +5,49 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
     - cmake
-    - ninja-build
     - git
-    - python3-pip
     - libdrm-dev
     - libstdc++-12-dev
+    - ninja-build
+    - python3-pip
 - name: pipModules
   type: object
   default:
+    - hip-python --extra-index-url https://test.pypi.org/simple
+    - ml_dtypes
+    - numpy
     - tomli
+    - scipy
 - name: rocmDependencies
   type: object
   default:
+    - clr
     - llvm-project
     - rocm-cmake
-    - clr
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
 
+- name: jobMatrix
+  type: object
+  default:
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 jobs:
-- job: rocMLIR
+- job: rocMLIR_build
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -49,6 +67,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -56,6 +75,7 @@ jobs:
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_FAT_LIBROCKCOMPILER=1
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
@@ -66,64 +86,69 @@ jobs:
       aptPackages: ${{ parameters.aptPackages }}
       pipModules: ${{ parameters.pipModules }}
 
-# compiling and running test on the test system together
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-- job: rocMLIR_testing
+  # compiling and running test on the test system together
-  dependsOn: rocMLIR
+  - job: rocMLIR_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: rocMLIR_build
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool: $(JOB_TEST_POOL)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  workspace:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    clean: all
+      )
-  strategy:
+    variables:
-    matrix:
+    - group: common
-      gfx942:
+    - template: /.azuredevops/variables-global.yml
-        JOB_GPU_TARGET: gfx942
+    pool: ${{ job.target }}_test_pool
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    workspace:
-  steps:
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    steps:
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      pipModules: ${{ parameters.pipModules }}
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmDependencies }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - task: Bash@3
+        gpuTarget: ${{ job.target }}
-    displayName: ROCm symbolic link
+    - task: Bash@3
-    inputs:
+      displayName: Symlink rocm_agent_enumerator
-      targetType: inline
+      inputs:
-      script: |
+        targetType: inline
-        sudo rm -rf /opt/rocm
+        script: |
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+          # Assuming that /opt is no longer persistent across runs, test environments are fully ephemeral
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+          sudo mkdir -p /opt/rocm/bin
-    parameters:
+          sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rocm_agent_enumerator /opt/rocm/bin/rocm_agent_enumerator
-      installEnabled: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      extraBuildFlags: >-
+      parameters:
-        -DCMAKE_BUILD_TYPE=Release
+        installEnabled: false
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
+        extraBuildFlags: >-
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
+          -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-        -DROCM_TEST_CHIPSET=$(JOB_GPU_TARGET)
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/lib
-        -GNinja
+          -DCMAKE_INCLUDE_PATH=$(Agent.BuildDirectory)/rocm/lib
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DAMDGPU_TARGETS=${{ job.target }}
-    parameters:
+          -DROCM_TEST_CHIPSET=${{ job.target }}
-      componentName: rocMLIR
+          -GNinja
-      testDir: $(Build.SourcesDirectory)/build
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-      testExecutable: ninja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-      testParameters: check-rocmlir
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        componentName: rocMLIR
-    parameters:
+        testDir: $(Build.SourcesDirectory)/build
-      aptPackages: ${{ parameters.aptPackages }}
+        testExecutable: ninja
-      pipModules: ${{ parameters.pipModules }}
+        testParameters: check-rocmlir
-      environment: test
+        testPublishResults: false
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocPRIM.yml

@@ -5,6 +5,17 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+- name: sparseCheckout
+  type: boolean
+  default: false
+- name: sparseCheckoutDir
+  type: string
+  default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -30,87 +41,102 @@ parameters:
     - ROCR-Runtime
     - rocprofiler-register
 
-jobs:
+- name: jobMatrix
-- job: rocPRIM
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DBUILD_BENCHMARK=ON
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DBUILD_TEST=ON
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: rocPRIM_testing
+jobs:
-  dependsOn: rocPRIM
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rocPRIM_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  workspace:
+    workspace:
-    clean: all
+      clean: all
-  strategy:
+    steps:
-    matrix:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      gfx942:
+      parameters:
-        JOB_GPU_TARGET: gfx942
+        aptPackages: ${{ parameters.aptPackages }}
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
-    parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      aptPackages: ${{ parameters.aptPackages }}
+        sparseCheckout: ${{ parameters.sparseCheckout }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        extraBuildFlags: >-
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DBUILD_BENCHMARK=ON
-    parameters:
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-      componentName: rocPRIM
+          -DAMDGPU_TARGETS=${{ job.target }}
-      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocprim'
+          -DBUILD_TEST=ON
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -GNinja
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      environment: test
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocPRIM_test_${{ job.target }}
+    dependsOn: rocPRIM_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocPRIM
+        testDir: '$(Agent.BuildDirectory)/rocm/bin/rocprim'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocPyDecode.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -22,229 +27,224 @@ parameters:
 - name: pipModules
   type: object
   default:
+    - hip-python --extra-index-url https://test.pypi.org/simple
     - numpy
     - pybind11
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
     - clr
-    - rocminfo
+    - llvm-project
-    - rocm-core
-    - rocprofiler-register
     - rocDecode
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler-register
+
+- name: jobMatrix
+  type: object
+  default:
+    buildJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
 
 jobs:
-- job: rocPyDecode
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  variables:
+  - job: rocPyDecode_build_${{ job.target }}
-  - group: common
+    variables:
-  - template: /.azuredevops/variables-global.yml
+    - group: common
-  pool: ${{ variables.LOW_BUILD_POOL }}
+    - template: /.azuredevops/variables-global.yml
-  workspace:
+    - name: ROCM_PATH
-    clean: all
+      value: $(Agent.BuildDirectory)/rocm
-  strategy:
+    pool:
-    matrix:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-      gfx942:
+    workspace:
-        JOB_GPU_TARGET: gfx942
+      clean: all
-  steps:
+    steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
+        pipModules: ${{ parameters.pipModules }}
-  - task: Bash@3
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-    displayName: 'pip install hip-python'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    inputs:
+      parameters:
-      targetType: inline
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      script: pip install -i https://test.pypi.org/simple hip-python
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-    parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-    parameters:
+    - task: Bash@3
-      checkoutRef: ${{ parameters.checkoutRef }}
+      displayName: 'Save Python Package Paths'
-      dependencyList: ${{ parameters.rocmDependencies }}
+      inputs:
-      gpuTarget: $(JOB_GPU_TARGET)
+        targetType: inline
-  - task: Bash@3
+        script: |
-    displayName: 'ROCm symbolic link'
+          echo "##vso[task.setvariable variable=PYTHON_USER_SITE;]$(python3 -m site --user-site)"
-    inputs:
+          echo "##vso[task.setvariable variable=PYTHON_DIST_PACKAGES;]$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
-      targetType: inline
+          echo "##vso[task.setvariable variable=PYBIND11_PATH;]$(python3 -c 'import pybind11; print(pybind11.get_cmake_dir())')"
-      script: |
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-        sudo rm -rf /opt/rocm
+      parameters:
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        installEnabled: false
-  - task: Bash@3
+        extraBuildFlags: >-
-    displayName: 'Save Python Package Paths'
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(PYTHON_USER_SITE)/pybind11;$(PYTHON_DIST_PACKAGES)/pybind11;$(PYBIND11_PATH)
-    inputs:
+          -DCMAKE_BUILD_TYPE=Release
-      targetType: inline
+          -DGPU_TARGETS=${{ job.target }}
-      script: |
+          -DCMAKE_INSTALL_PREFIX_PYTHON=$(Build.BinariesDirectory)
-        echo "##vso[task.setvariable variable=PYTHON_USER_SITE;]$(python3 -m site --user-site)"
+          -GNinja
-        echo "##vso[task.setvariable variable=PYTHON_DIST_PACKAGES;]$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
+    - task: Bash@3
-        echo "##vso[task.setvariable variable=PYBIND11_PATH;]$(python3 -c 'import pybind11; print(pybind11.get_cmake_dir())')"
+      displayName: 'rocPyDecode install'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+      inputs:
-    parameters:
+        targetType: inline
-      installEnabled: false
+        script: |
-      extraBuildFlags: >-
+          sudo cmake --build . --target install
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+          sudo chown -R $(whoami):$(id -gn) $(Build.BinariesDirectory)
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(PYTHON_USER_SITE)/pybind11;$(PYTHON_DIST_PACKAGES)/pybind11;$(PYBIND11_PATH)
+        workingDirectory: $(Build.SourcesDirectory)/build
-        -DCMAKE_BUILD_TYPE=Release
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+      parameters:
-        -DCMAKE_INSTALL_PREFIX_PYTHON=$(Build.BinariesDirectory)
+        gpuTarget: ${{ job.target }}
-        -GNinja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-  - task: Bash@3
+      parameters:
-    displayName: 'rocPyDecode install'
+        gpuTarget: ${{ job.target }}
-    inputs:
+        publish: false
-      targetType: inline
+    - task: Bash@3
-      script: |
+      displayName: Create wheel file
-        sudo cmake --build . --target install
+      inputs:
-        sudo chown -R $(whoami):$(id -gn) $(Build.BinariesDirectory)
+        targetType: inline
-      workingDirectory: $(Build.SourcesDirectory)/build
+        script: |
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+          export HIP_INCLUDE_DIRS=$(Agent.BuildDirectory)/rocm/include/hip
-    parameters:
+          sudo python3 setup.py bdist_wheel
-      gpuTarget: $(JOB_GPU_TARGET)
+          sudo chown -R $(whoami):$(id -gn) $(find . -name "*.whl")
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+        workingDirectory: $(Build.SourcesDirectory)
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-      publish: false
+        sourceDir: $(Build.SourcesDirectory)/dist
-  - task: Bash@3
+        contentsString: '*.whl'
-    displayName: Create wheel file
+        targetDir: $(Build.ArtifactStagingDirectory)
-    inputs:
+        clean: false
-      targetType: inline
+    - task: PublishPipelineArtifact@1
-      script: |
+      displayName: 'wheel file Publish'
-        export ROCM_PATH=$(Agent.BuildDirectory)/rocm
+      retryCountOnTaskFailure: 3
-        export HIP_INCLUDE_DIRS=$(Agent.BuildDirectory)/rocm/include/hip
+      inputs:
-        sudo python3 setup.py bdist_wheel
+        targetPath: $(Build.ArtifactStagingDirectory)
-        sudo chown -R $(whoami):$(id -gn) $(find . -name "*.whl")
+    - task: Bash@3
-      workingDirectory: $(Build.SourcesDirectory)
+      displayName: Save pipeline artifact file names
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+      inputs:
-    parameters:
+        workingDirectory: $(Pipeline.Workspace)
-      sourceDir: $(Build.SourcesDirectory)/dist
+        targetType: inline
-      contentsString: '*.whl'
+        script: |
-      targetDir: $(Build.ArtifactStagingDirectory)
+          whlFile=$(find "$(Build.ArtifactStagingDirectory)" -type f -name "*.whl" | head -n 1)
-      clean: false
+          if [ -n "$whlFile" ]; then
-  - task: PublishPipelineArtifact@1
+            echo $(basename "$whlFile") >> pipelineArtifacts.txt
-    displayName: 'wheel file Publish'
+          fi
-    retryCountOnTaskFailure: 3
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-    inputs:
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      targetPath: $(Build.ArtifactStagingDirectory)
+    #   parameters:
-  - task: Bash@3
+    #     aptPackages: ${{ parameters.aptPackages }}
-    displayName: Save pipeline artifact file names
+    #     pipModules: ${{ parameters.pipModules }}
-    inputs:
+    #     gpuTarget: ${{ job.target }}
-      workingDirectory: $(Pipeline.Workspace)
-      targetType: inline
-      script: |
-        whlFile=$(find "$(Build.ArtifactStagingDirectory)" -type f -name "*.whl" | head -n 1)
-        if [ -n "$whlFile" ]; then
-          echo $(basename "$whlFile") >> pipelineArtifacts.txt
-        fi
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      optSymLink: true
 
-- job: rocPyDecode_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  dependsOn: rocPyDecode
+  - job: rocPyDecode_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: rocPyDecode_build_${{ job.target }}
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool:
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-    name: $(JOB_TEST_POOL)
+        eq(${{ parameters.aggregatePipeline }}, False)
-    demands: firstRenderDeviceAccess
+      )
-  workspace:
+    variables:
-    clean: all
+    - group: common
-  strategy:
+    - template: /.azuredevops/variables-global.yml
-    matrix:
+    - name: ROCM_PATH
-      gfx942:
+      value: $(Agent.BuildDirectory)/rocm
-        JOB_GPU_TARGET: gfx942
+    pool:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+      name: ${{ job.target }}_test_pool
-  steps:
+    workspace:
-  - task: Bash@3
+      clean: all
-    displayName: Ensure pybind11-dev is not installed
+    steps:
-    inputs:
+    - task: Bash@3
-      targetType: inline
+      displayName: Ensure pybind11-dev is not installed
-      script: |
+      inputs:
-        if dpkg -l | grep -q pybind11-dev; then
+        targetType: inline
-          echo "Removing pybind11-dev..."
+        script: |
-          sudo apt remove -y pybind11-dev
+          if dpkg -l | grep -q pybind11-dev; then
-        else
+            echo "Removing pybind11-dev..."
-          echo "pybind11-dev is not installed."
+            sudo apt remove -y pybind11-dev
-        fi
+          else
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+            echo "pybind11-dev is not installed."
-    parameters:
+          fi
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      pipModules: ${{ parameters.pipModules }}
+      parameters:
-  - task: Bash@3
+        aptPackages: ${{ parameters.aptPackages }}
-    displayName: 'pip install hip-python'
+        pipModules: ${{ parameters.pipModules }}
-    inputs:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      targetType: inline
+    - task: DownloadPipelineArtifact@2
-      script: pip install -i https://test.pypi.org/simple hip-python
+      displayName: 'Download Pipeline Wheel Files'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      inputs:
-  - task: DownloadPipelineArtifact@2
+        itemPattern: '**/*.whl'
-    displayName: 'Download Pipeline Wheel Files'
+        targetPath: $(Agent.BuildDirectory)
-    inputs:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-      itemPattern: '**/*.whl'
+      parameters:
-      targetPath: $(Agent.BuildDirectory)
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+        setupHIPLibrarySymlinks: true
-      dependencyList: ${{ parameters.rocmDependencies }}
+    - task: Bash@3
-      gpuTarget: $(JOB_GPU_TARGET)
+      displayName: pip install
-      setupHIPLibrarySymlinks: true
+      inputs:
-  - task: Bash@3
+        targetType: inline
-    displayName: pip install
+        script: |
-    inputs:
+          pip uninstall -y rocPyDecode
-      targetType: inline
+          find -name *.whl -exec pip install {} \;
-      script: |
+        workingDirectory: $(Agent.BuildDirectory)
-        pip uninstall -y rocPyDecode
+    - task: Bash@3
-        find -name *.whl -exec pip install {} \;
+      displayName: Setup search paths
-      workingDirectory: $(Agent.BuildDirectory)
+      inputs:
-  - task: Bash@3
+        targetType: inline
-    displayName: Setup search paths
+        script: |
-    inputs:
+          echo "##vso[task.setvariable variable=PYTHON_USER_SITE;]$(python3 -m site --user-site)"
-      targetType: inline
+          echo "##vso[task.setvariable variable=PYTHON_DIST_PACKAGES;]$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
-      script: |
+          echo "##vso[task.setvariable variable=PYBIND11_PATH;]$(python3 -c 'import pybind11; print(pybind11.get_cmake_dir())')"
-        sudo rm -rf /opt/rocm
+    - task: CMake@1
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      displayName: 'rocPyDecode Test CMake Flags'
-        echo "##vso[task.setvariable variable=PYTHON_USER_SITE;]$(python3 -m site --user-site)"
+      inputs:
-        echo "##vso[task.setvariable variable=PYTHON_DIST_PACKAGES;]$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
+        cmakeArgs: >-
-        echo "##vso[task.setvariable variable=PYBIND11_PATH;]$(python3 -c 'import pybind11; print(pybind11.get_cmake_dir())')"
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(PYTHON_USER_SITE)/pybind11;$(PYTHON_DIST_PACKAGES)/pybind11;$(PYBIND11_PATH)
-  - task: CMake@1
+          -DCMAKE_BUILD_TYPE=Release
-    displayName: 'rocPyDecode Test CMake Flags'
+          -DGPU_TARGETS=${{ job.target }}
-    inputs:
+          ..
-      cmakeArgs: >-
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(PYTHON_USER_SITE)/pybind11;$(PYTHON_DIST_PACKAGES)/pybind11;$(PYBIND11_PATH)
+      parameters:
-        -DCMAKE_BUILD_TYPE=Release
+        componentName: rocPyDecode
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        testDir: $(Build.SourcesDirectory)/build
-        ..
+  # sudo required for pip install but screws up permissions for next pipeline run
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - task: Bash@3
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      displayName: Clean up test environment
-    parameters:
+      condition: always()
-      componentName: rocPyDecode
+      inputs:
-      testDir: $(Build.SourcesDirectory)/build
+        targetType: inline
-# sudo required for pip install but screws up permissions for next pipeline run
+        script: |
-  - task: Bash@3
+          pip uninstall -y rocPyDecode
-    displayName: Clean up test environment
+          pip uninstall -y hip-python
-    condition: always()
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    inputs:
+      parameters:
-      targetType: inline
+        aptPackages: ${{ parameters.aptPackages }}
-      script: |
+        pipModules: ${{ parameters.pipModules }}
-        pip uninstall -y rocPyDecode
+        environment: test
-        pip uninstall -y hip-python
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        pythonEnvVars: true
-    parameters:
+    # note that this docker won't have hip-python installed via pip
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      environment: test
-      gpuTarget: $(JOB_GPU_TARGET)
-      pythonEnvVars: true
-  # note that this docker won't have hip-python installed via pip

.azuredevops/components/rocRAND.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -31,90 +36,104 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
-jobs:
+- name: jobMatrix
-- job: rocRAND
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  - name: HIP_ROCCLR_HOME
+        target: gfx942
-    value: $(Build.BinariesDirectory)/rocm
+      - gfx90a:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx90a
-  workspace:
+    testJobs:
-    clean: all
+      - gfx942:
-  strategy:
+        target: gfx942
-    matrix:
+      - gfx90a:
-      gfx942:
+        target: gfx90a
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DBUILD_TEST=ON
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
 
-- job: rocRAND_testing
+jobs:
-  dependsOn: rocRAND
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rocRAND_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    - name: HIP_ROCCLR_HOME
-  workspace:
+      value: $(Build.BinariesDirectory)/rocm
-    clean: all
+    pool:
-  strategy:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    matrix:
+    workspace:
-      gfx942:
+      clean: all
-        JOB_GPU_TARGET: gfx942
+    steps:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  steps:
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+        aptPackages: ${{ parameters.aptPackages }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        extraBuildFlags: >-
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-    parameters:
+          -DBUILD_TEST=ON
-      componentName: rocRAND
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocRAND'
+          -DAMDGPU_TARGETS=${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -GNinja
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      environment: test
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    #   parameters:
+    #     aptPackages: ${{ parameters.aptPackages }}
+    #     gpuTarget: ${{ job.target }}
+    #     extraEnvVars:
+    #       - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocRAND_test_${{ job.target }}
+    dependsOn: rocRAND_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocRAND
+        testDir: '$(Agent.BuildDirectory)/rocm/bin/rocRAND'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocSOLVER.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -46,111 +51,125 @@ parameters:
     - rocSPARSE
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: rocSOLVER
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - task: Bash@3
-    displayName: 'Clone lapack'
-    inputs:
-      targetType: inline
-      script: git clone --depth 1 --branch v3.9.1 https://github.com/Reference-LAPACK/lapack
-      workingDirectory: '$(Build.SourcesDirectory)'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      componentName: lapack
-      extraBuildFlags: >-
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_Fortran_FLAGS=-fno-optimize-sibling-calls
-        -DBUILD_TESTING=OFF
-        -DCBLAS=ON
-        -DLAPACKE=OFF
-        -GNinja
-      cmakeBuildDir: '$(Build.SourcesDirectory)/lapack/build'
-      installDir: '$(Pipeline.Workspace)/deps-install'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Pipeline.Workspace)/deps-install
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DBUILD_CLIENTS_TESTS=ON
-        -DBUILD_CLIENTS_BENCHMARKS=OFF
-        -DBUILD_CLIENTS_SAMPLES=OFF
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraCopyDirectories:
-        - deps-install
 
-- job: rocSOLVER_testing
+jobs:
-  dependsOn: rocSOLVER
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rocSOLVER_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  workspace:
+    workspace:
-    clean: all
+      clean: all
-  strategy:
+    steps:
-    matrix:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      gfx942:
+      parameters:
-        JOB_GPU_TARGET: gfx942
+        aptPackages: ${{ parameters.aptPackages }}
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
-    parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      aptPackages: ${{ parameters.aptPackages }}
+    - task: Bash@3
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      displayName: 'Clone lapack'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      inputs:
-    parameters:
+        targetType: inline
-      gpuTarget: $(JOB_GPU_TARGET)
+        script: git clone --depth 1 --branch v3.9.1 https://github.com/Reference-LAPACK/lapack
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        workingDirectory: '$(Build.SourcesDirectory)'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
+      parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
+      parameters:
-      componentName: rocSOLVER
+        componentName: lapack
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        extraBuildFlags: >-
-      testExecutable: './rocsolver-test'
+          -DCMAKE_BUILD_TYPE=Release
-      testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+          -DCMAKE_Fortran_FLAGS=-fno-optimize-sibling-calls
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -DBUILD_TESTING=OFF
-    parameters:
+          -DCBLAS=ON
-      aptPackages: ${{ parameters.aptPackages }}
+          -DLAPACKE=OFF
-      environment: test
+          -GNinja
-      gpuTarget: $(JOB_GPU_TARGET)
+        cmakeBuildDir: '$(Build.SourcesDirectory)/lapack/build'
+        cmakeSourceDir: '$(Build.SourcesDirectory)/lapack'
+        installDir: '$(Pipeline.Workspace)/deps-install'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+      parameters:
+        extraBuildFlags: >-
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Pipeline.Workspace)/deps-install
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+          -DAMDGPU_TARGETS=${{ job.target }}
+          -DBUILD_CLIENTS_TESTS=ON
+          -DBUILD_CLIENTS_BENCHMARKS=OFF
+          -DBUILD_CLIENTS_SAMPLES=OFF
+          -GNinja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+        extraCopyDirectories:
+          - deps-install
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocSOLVER_test_${{ job.target }}
+    dependsOn: rocSOLVER_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocSOLVER
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        testExecutable: './rocsolver-test'
+        testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocSPARSE.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -29,6 +34,7 @@ parameters:
     - rocminfo
     - rocPRIM
     - rocprofiler-register
+    - roctracer
 - name: rocmTestDependencies
   type: object
   default:
@@ -42,110 +48,123 @@ parameters:
     - ROCR-Runtime
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: rocSPARSE
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  - name: HIP_ROCCLR_HOME
+        target: gfx942
-    value: $(Build.BinariesDirectory)/rocm
+      - gfx90a:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx90a
-  workspace:
+    testJobs:
-    clean: all
+      - gfx942:
-  strategy:
+        target: gfx942
-    matrix:
+      - gfx90a:
-      gfx942:
+        target: gfx90a
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DBUILD_CLIENTS_SAMPLES=OFF
-        -DBUILD_CLIENTS_TESTS=ON
-        -DBUILD_CLIENTS_BENCHMARKS=OFF
-        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip;$(Agent.BuildDirectory)/rocm/hip/cmake
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      artifactName: rocSPARSE
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      artifactName: rocSPARSE
-      gpuTarget: $(JOB_GPU_TARGET)
-      publish: false
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
-    parameters:
-      sourceDir: $(Build.SourcesDirectory)/build/clients
-      contentsString: matrices/**
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      artifactName: testMatrices
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
 
-- job: rocSPARSE_testing
+jobs:
-  timeoutInMinutes: 90
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  dependsOn: rocSPARSE
+  - job: rocSPARSE_build_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    variables:
-  variables:
+    - group: common
-  - group: common
+    - template: /.azuredevops/variables-global.yml
-  - template: /.azuredevops/variables-global.yml
+    - name: HIP_ROCCLR_HOME
-  pool: $(JOB_TEST_POOL)
+      value: $(Build.BinariesDirectory)/rocm
-  workspace:
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-    clean: all
+    workspace:
-  strategy:
+      clean: all
-    matrix:
+    steps:
-      gfx942:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-        JOB_GPU_TARGET: gfx942
+      parameters:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+        aptPackages: ${{ parameters.aptPackages }}
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
-    parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        extraBuildFlags: >-
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-    parameters:
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-      componentName: rocSPARSE
+          -DCMAKE_BUILD_TYPE=Release
-      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+          -DAMDGPU_TARGETS=${{ job.target }}
-      testExecutable: './rocsparse-test'
+          -DBUILD_CLIENTS_SAMPLES=OFF
-      testParameters: '--gtest_filter="*quick*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+          -DBUILD_CLIENTS_TESTS=ON
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -DBUILD_CLIENTS_BENCHMARKS=OFF
-    parameters:
+          -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip;$(Agent.BuildDirectory)/rocm/hip/cmake
-      aptPackages: ${{ parameters.aptPackages }}
+          -GNinja
-      environment: test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
+        artifactName: rocSPARSE
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        artifactName: rocSPARSE
+        gpuTarget: ${{ job.target }}
+        publish: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+      parameters:
+        sourceDir: $(Build.SourcesDirectory)/build/clients
+        contentsString: matrices/**
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        artifactName: testMatrices
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+        extraEnvVars:
+          - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocSPARSE_test_${{ job.target }}
+    timeoutInMinutes: 90
+    dependsOn: rocSPARSE_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocSPARSE
+        testDir: '$(Agent.BuildDirectory)/rocm/bin'
+        testExecutable: './rocsparse-test'
+        testParameters: '--gtest_filter="*quick*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocThrust.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -35,87 +40,100 @@ parameters:
     - hipRAND
     - rocprofiler-register
 
-jobs:
+- name: jobMatrix
-- job: rocThrust
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -GNinja
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DBUILD_TEST=ON
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: rocThrust_testing
+jobs:
-  dependsOn: rocThrust
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rocThrust_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-  workspace:
+    workspace:
-    clean: all
+      clean: all
-  strategy:
+    steps:
-    matrix:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      gfx942:
+      parameters:
-        JOB_GPU_TARGET: gfx942
+        aptPackages: ${{ parameters.aptPackages }}
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
-    parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-    parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
+      parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
+        extraBuildFlags: >-
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+          -GNinja
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-    parameters:
+          -DAMDGPU_TARGETS=${{ job.target }}
-      componentName: rocThrust
+          -DBUILD_TEST=ON
-      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocthrust'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
-    parameters:
+        gpuTarget: ${{ job.target }}
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-      environment: test
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocThrust_test_${{ job.target }}
+    dependsOn: rocThrust_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocThrust
+        testDir: '$(Agent.BuildDirectory)/rocm/bin/rocthrust'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocWMMA.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -45,92 +50,105 @@ parameters:
     - ROCR-Runtime
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: rocWMMA
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_BUILD_TYPE=Release
-        -DROCWMMA_BUILD_TESTS=ON
-        -DROCWMMA_BUILD_SAMPLES=OFF
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON
-        -DROCM_PLATFORM_VERSION=$(NEXT_RELEASE_VERSION)
-        -GNinja
-# gfx1030 not supported in documentation
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: rocWMMA_testing
+jobs:
-  timeoutInMinutes: 90
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  dependsOn: rocWMMA
+  - job: rocWMMA_build_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    variables:
-  variables:
+    - group: common
-  - group: common
+    - template: /.azuredevops/variables-global.yml
-  - template: /.azuredevops/variables-global.yml
+    pool: ${{ variables.HIGH_BUILD_POOL }}
-  pool: $(JOB_TEST_POOL)
+    workspace:
-  workspace:
+      clean: all
-    clean: all
+    steps:
-  strategy:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    matrix:
+      parameters:
-      gfx942:
+        aptPackages: ${{ parameters.aptPackages }}
-        JOB_GPU_TARGET: gfx942
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  steps:
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
-    parameters:
+        extraBuildFlags: >-
-      checkoutRef: ${{ parameters.checkoutRef }}
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DCMAKE_BUILD_TYPE=Release
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DROCWMMA_BUILD_TESTS=ON
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DROCWMMA_BUILD_SAMPLES=OFF
-    parameters:
+          -DGPU_TARGETS=${{ job.target }}
-      componentName: rocWMMA
+          -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON
-      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocwmma'
+          -DROCM_PLATFORM_VERSION=$(NEXT_RELEASE_VERSION)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -GNinja
-    parameters:
+  # gfx1030 not supported in documentation
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      environment: test
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocWMMA_test_${{ job.target }}
+    timeoutInMinutes: 270
+    dependsOn: rocWMMA_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocWMMA
+        testDir: '$(Agent.BuildDirectory)/rocm/bin/rocwmma'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocm-cmake.yml

@@ -5,10 +5,14 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
-    - cmake
     - doxygen
     - doxygen-doc
     - ninja-build
@@ -18,14 +22,17 @@ parameters:
   type: object
   default:
     - cget
+    - cmake==3.20.5
     - ninja
+    - rocm-docs-core
 
 jobs:
 - job: rocm_cmake
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -33,27 +40,34 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
       pipModules: ${{ parameters.pipModules }}
+  - task: Bash@3
+    displayName: Add CMake to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(python3 -m site --user-base)/bin"
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-# extra steps for ctest suite
+  - task: Bash@3
-  - script: |
+    displayName: CTest setup
-      python -m pip install -r $(Build.SourcesDirectory)/docs/requirements.txt
+    inputs:
-      python -m pip install -r $(Build.SourcesDirectory)/test/docsphinx/docs/.sphinx/requirements.txt
+      targetType: inline
-      git config --global user.email "you@example.com"
+      script: |
-      git config --global user.name "Your Name"
+        python -m pip install -r $(Build.SourcesDirectory)/docs/requirements.txt
-    displayName: "ctest setup"
+        python -m pip install -r $(Build.SourcesDirectory)/test/docsphinx/docs/.sphinx/requirements.txt
+        git config --global user.email "you@example.com"
+        git config --global user.name "Your Name"
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocm-cmake
+      testParameters: '-E "pass-version-parent" --output-on-failure --force-new-ctest-process --output-junit test_output.xml'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
+  #     pipModules: ${{ parameters.pipModules }}
-      environment: combined
+  #     environment: combined
-      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocm-core.yml

@@ -5,10 +5,16 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
     - cmake
+    - ninja-build
     - python3-pip
 
 jobs:
@@ -16,7 +22,8 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -37,9 +44,10 @@ jobs:
         -DCPACK_DEBIAN_PACKAGE_RELEASE="local.9999~99.99"
         -DCPACK_RPM_PACKAGE_RELEASE="local.9999"
         -DROCM_VERSION="$(NEXT_RELEASE_VERSION)"
+        -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/rocm-examples.yml

@@ -5,12 +5,18 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
     - cmake
     - libglfw3-dev
     - libtbb-dev
+    - ninja-build
     - python3-pip
 - name: rocmDependencies
   type: object
@@ -63,107 +69,123 @@ parameters:
     - rocThrust
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: rocm_examples
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      # https://github.com/ROCm/HIP/issues/2203
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DROCM_ROOT=$(Agent.BuildDirectory)/rocm
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
-        -DCMAKE_EXE_LINKER_FLAGS=-fgpu-rdc
-  - task: Bash@3
-    displayName: Move rocm-examples binaries to rocm/examples
-    inputs:
-      targetType: inline
-      script: |
-        mkdir -p $(Build.BinariesDirectory)/examples
-        mv $(Build.BinariesDirectory)/bin/* $(Build.BinariesDirectory)/examples
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: rocm_examples_testing
+jobs:
-  dependsOn: rocm_examples
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rocm_examples_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  - name: TEST_LOG_FILE
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-    value: $(Pipeline.Workspace)/rocm-examplesTestLog.log
+    workspace:
-  pool: $(JOB_TEST_POOL)
+      clean: all
-  workspace:
+    steps:
-    clean: all
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  strategy:
+      parameters:
-    matrix:
+        aptPackages: ${{ parameters.aptPackages }}
-      gfx942:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-        JOB_GPU_TARGET: gfx942
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+      parameters:
-  steps:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        # https://github.com/ROCm/HIP/issues/2203
-    parameters:
+        extraBuildFlags: >-
-      checkoutRef: ${{ parameters.checkoutRef }}
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-      gpuTarget: $(JOB_GPU_TARGET)
+          -DROCM_ROOT=$(Agent.BuildDirectory)/rocm
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+          -DAMDGPU_TARGETS=${{ job.target }}
-    parameters:
+          -DCMAKE_HIP_ARCHITECTURES=${{ job.target }}
-      # https://github.com/ROCm/HIP/issues/2203
+          -DCMAKE_EXE_LINKER_FLAGS=-fgpu-rdc
-      extraBuildFlags: >-
+          -GNinja
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+    - task: Bash@3
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+      displayName: Move rocm-examples binaries to rocm/examples
-        -DROCM_ROOT=$(Agent.BuildDirectory)/rocm
+      inputs:
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        targetType: inline
-        -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
+        script: |
-        -DCMAKE_EXE_LINKER_FLAGS=-fgpu-rdc
+          mkdir -p $(Build.BinariesDirectory)/examples
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          mv $(Build.BinariesDirectory)/bin/* $(Build.BinariesDirectory)/examples
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
+      parameters:
-      componentName: rocm-examples
+        gpuTarget: ${{ job.target }}
-      testDir: $(Build.SourcesDirectory)/build
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
-    parameters:
+        gpuTarget: ${{ job.target }}
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-      environment: test
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocm_examples_test_${{ job.target }}
+    dependsOn: rocm_examples_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    - name: TEST_LOG_FILE
+      value: $(Pipeline.Workspace)/rocm-examplesTestLog.log
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+      parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+      parameters:
+        # https://github.com/ROCm/HIP/issues/2203
+        extraBuildFlags: >-
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+          -DROCM_ROOT=$(Agent.BuildDirectory)/rocm
+          -DAMDGPU_TARGETS=${{ job.target }}
+          -DCMAKE_HIP_ARCHITECTURES=${{ job.target }}
+          -DCMAKE_EXE_LINKER_FLAGS=-fgpu-rdc
+          -GNinja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocm-examples
+        testDir: $(Build.SourcesDirectory)/build
+        testParameters: '--output-on-failure --force-new-ctest-process --output-junit test_output.xml --exclude-regex "rocfft_callback"'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocm_bandwidth_test.yml

@@ -5,6 +5,12 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
+
 - name: aptPackages
   type: object
   default:
@@ -31,8 +37,17 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
+- name: jobMatrix
+  type: object
+  default:
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 jobs:
-- job: rocm_bandwidth_test
+- job: rocm_bandwidth_test_build
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -40,7 +55,8 @@ jobs:
     value: $(Agent.BuildDirectory)/rocm
   - name: ROCR_LIB_DIR
     value: $(Agent.BuildDirectory)/rocm
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -56,6 +72,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -66,51 +83,52 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
+  #     pipModules: ${{ parameters.pipModules }}
-      extraEnvVars:
+  #     extraEnvVars:
-        - ROCR_INC_DIR:::/home/user/workspace/rocm
+  #       - ROCR_INC_DIR:::/home/user/workspace/rocm
-        - ROCR_LIB_DIR:::/home/user/workspace/rocm
+  #       - ROCR_LIB_DIR:::/home/user/workspace/rocm
 
-- job: rocm_bandwidth_test_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  dependsOn: rocm_bandwidth_test
+  - job: rocm_bandwidth_test_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: rocm_bandwidth_test_build
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool: $(JOB_TEST_POOL)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  workspace:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    clean: all
+      )
-  strategy:
+    variables:
-    matrix:
+    - group: common
-      gfx942:
+    - template: /.azuredevops/variables-global.yml
-        JOB_GPU_TARGET: gfx942
+    pool: ${{ job.target }}_test_pool
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    workspace:
-  steps:
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    steps:
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      pipModules: ${{ parameters.pipModules }}
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmTestDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-      componentName: rocm_bandwidth_test
+      parameters:
-      testDir: '$(Agent.BuildDirectory)'
+        componentName: rocm_bandwidth_test
-      testExecutable: './rocm/bin/rocm-bandwidth-test'
+        testDir: '$(Agent.BuildDirectory)'
-      testParameters: ''
+        testExecutable: './rocm/bin/rocm-bandwidth-test'
-      testPublishResults: false
+        testParameters: ''
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        testPublishResults: false
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      pipModules: ${{ parameters.pipModules }}
+        aptPackages: ${{ parameters.aptPackages }}
-      environment: test
+        pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocm_smi_lib.yml

@@ -5,19 +5,36 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
     - cmake
     - libdrm-dev
+    - ninja-build
+    - pkg-config
     - python3-pip
 
+- name: jobMatrix
+  type: object
+  default:
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 jobs:
-- job: rocm_smi_lib
+- job: rocm_smi_lib_build
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -33,42 +50,46 @@ jobs:
       extraBuildFlags: >-
         -DBUILD_TESTS=ON
         -DROCM_DEP_ROCMCORE=ON
+        -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: rocm_smi_lib_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  dependsOn: rocm_smi_lib
+  - job: rocm_smi_lib_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: rocm_smi_lib_build
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool: $(JOB_TEST_POOL)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  workspace:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    clean: all
+      )
-  strategy:
+    variables:
-    matrix:
+    - group: common
-      gfx942:
+    - template: /.azuredevops/variables-global.yml
-        JOB_GPU_TARGET: gfx942
+    pool: ${{ job.target }}_test_pool
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    workspace:
-  steps:
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+      parameters:
-    parameters:
+        aptPackages: ${{ parameters.aptPackages }}
-      runRocminfo: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-      componentName: rocm_smi_lib
+      parameters:
-      testDir: '$(Agent.BuildDirectory)'
+        runRocminfo: false
-      testExecutable: './rocm/share/rocm_smi/rsmitst_tests/rsmitst'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        componentName: rocm_smi_lib
-    parameters:
+        testDir: '$(Agent.BuildDirectory)'
-      aptPackages: ${{ parameters.aptPackages }}
+        testExecutable: './rocm/share/rocm_smi/rsmitst_tests/rsmitst'
-      environment: test
+        testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocminfo.yml

@@ -5,12 +5,18 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
     - cmake
     - libdrm-amdgpu-dev
     - libdrm-dev
+    - ninja-build
     - python3-pip
 - name: rocmDependencies
   type: object
@@ -23,12 +29,22 @@ parameters:
     - ROCR-Runtime
     - rocprofiler-register
 
+- name: jobMatrix
+  type: object
+  default:
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 jobs:
 - job: rocminfo
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -44,65 +60,65 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
       skipLlvmSymlink: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCRTST_BLD_TYPE=release
+        -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      registerROCmPackages: true
 
-- job: rocminfo_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  dependsOn: rocminfo
+  - job: rocminfo_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: rocminfo
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool: $(JOB_TEST_POOL)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  workspace:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    clean: all
+      )
-  strategy:
+    variables:
-    matrix:
+    - group: common
-      gfx942:
+    - template: /.azuredevops/variables-global.yml
-        JOB_GPU_TARGET: gfx942
+    pool: ${{ job.target }}_test_pool
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    workspace:
-  steps:
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    steps:
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        registerROCmPackages: true
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        dependencyList: ${{ parameters.rocmTestDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      runRocminfo: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
-    parameters:
+        runRocminfo: false
-      componentName: rocminfo
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-      testDir: '$(Agent.BuildDirectory)'
+      parameters:
-      testExecutable: './rocm/bin/rocminfo'
+        componentName: rocminfo
-      testParameters: ''
+        testDir: '$(Agent.BuildDirectory)'
-      testPublishResults: false
+        testExecutable: './rocm/bin/rocminfo'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+        testParameters: ''
-    parameters:
+        testPublishResults: false
-      componentName: rocm_agent_enumerator
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-      testDir: '$(Agent.BuildDirectory)'
+      parameters:
-      testExecutable: './rocm/bin/rocm_agent_enumerator'
+        componentName: rocm_agent_enumerator
-      testParameters: ''
+        testDir: '$(Agent.BuildDirectory)'
-      testPublishResults: false
+        testExecutable: './rocm/bin/rocm_agent_enumerator'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        testParameters: ''
-    parameters:
+        testPublishResults: false
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      registerROCmPackages: true
+      parameters:
-      environment: test
+        aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        registerROCmPackages: true
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocprofiler-compute.yml

@@ -5,21 +5,29 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
     - cmake
+    - libdw-dev
+    - libtbb-dev
     - locales
+    - ninja-build
     - python3-pip
 - name: pipModules
   type: object
   default:
     - astunparse==1.6.2
     - colorlover
-    - dash>=1.12.0
+    - "dash>=1.12.0"
     - matplotlib
-    - numpy>=1.17.5
+    - "numpy>=1.17.5"
-    - pandas>=1.4.3
+    - "pandas>=1.4.3"
     - pymongo
     - pyyaml
     - tabulate
@@ -36,6 +44,7 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
+    - amdsmi
     - clr
     - llvm-project
     - rocm-cmake
@@ -44,133 +53,168 @@ parameters:
     - rocm_smi_lib
     - ROCR-Runtime
     - rocprofiler
+    - rocprofiler-sdk
     - rocprofiler-register
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: rocprofiler_compute
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942-staging:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        name: gfx942_staging
-  workspace:
+        target: gfx942
-    clean: all
+        dependencySource: staging
-  strategy:
+      - gfx942-mainline:
-    matrix:
+        name: gfx942_mainline
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+        dependencySource: mainline
-  steps:
+      - gfx90a-staging:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+        name: gfx90a_staging
-    parameters:
+        target: gfx90a
-      aptPackages: ${{ parameters.aptPackages }}
+        dependencySource: staging
-      pipModules: ${{ parameters.pipModules }}
+      - gfx90a-mainline:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        name: gfx90a_mainline
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+        target: gfx90a
-    parameters:
+        dependencySource: mainline
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+    testJobs:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+      - gfx942-staging:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        name: gfx942_staging
-    parameters:
+        target: gfx942
-      checkoutRef: ${{ parameters.checkoutRef }}
+        dependencySource: staging
-      dependencyList: ${{ parameters.rocmDependencies }}
+      - gfx942-mainline:
-      gpuTarget: $(JOB_GPU_TARGET)
+        name: gfx942_mainline
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+        target: gfx942
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+        dependencySource: mainline
-    parameters:
+      - gfx90a-staging:
-      gpuTarget: $(JOB_GPU_TARGET)
+        name: gfx90a_staging
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+        target: gfx90a
-    parameters:
+        dependencySource: staging
-      gpuTarget: $(JOB_GPU_TARGET)
+      - gfx90a-mainline:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+        name: gfx90a_mainline
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        target: gfx90a
-    parameters:
+        dependencySource: mainline
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: rocprofiler_compute_testing
+jobs:
-  timeoutInMinutes: 120
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  dependsOn: rocprofiler_compute
+  - job: rocprofiler_compute_build_${{ job.name }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    variables:
-  variables:
+    - group: common
-  - group: common
+    - template: /.azuredevops/variables-global.yml
-  - template: /.azuredevops/variables-global.yml
+    pool:
-  - name: PYTHON_VERSION
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    value: 3.10
+    workspace:
-  pool: $(JOB_TEST_POOL)
+      clean: all
-  workspace:
+    steps:
-    clean: all
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  strategy:
+      parameters:
-    matrix:
+        aptPackages: ${{ parameters.aptPackages }}
-      gfx942:
+        pipModules: ${{ parameters.pipModules }}
-        JOB_GPU_TARGET: gfx942
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  steps:
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      pipModules: ${{ parameters.pipModules }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - task: Bash@3
+        dependencyList: ${{ parameters.rocmDependencies }}
-    displayName: Add en_US.UTF-8 locale
+        dependencySource: ${{ job.dependencySource }}
-    inputs:
+        gpuTarget: ${{ job.target }}
-      targetType: inline
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      script: |
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-        sudo locale-gen en_US.UTF-8
+      parameters:
-        sudo update-locale
+        extraBuildFlags: >-
-        locale -a
+          -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
+      parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+        artifactName: ${{ job.dependencySource }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
-    parameters:
+        artifactName: ${{ job.dependencySource }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+        gpuTarget: ${{ job.target }}
-      dependencyList: ${{ parameters.rocmDependencies }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-  - task: Bash@3
+    #   parameters:
-    displayName: Add ROCm binaries to PATH
+    #     aptPackages: ${{ parameters.aptPackages }}
-    inputs:
+    #     pipModules: ${{ parameters.pipModules }}
-      targetType: inline
+    #     gpuTarget: ${{ job.target }}
-      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+
-  - task: Bash@3
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-    displayName: Add ROCm compilers to PATH
+  - job: rocprofiler_compute_test_${{ job.name }}
-    inputs:
+    timeoutInMinutes: 120
-      targetType: inline
+    dependsOn: rocprofiler_compute_build_${{ job.name }}
-      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+    condition:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+      and(succeeded(),
-    parameters:
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-      extraBuildFlags: >-
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-        -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
+        eq(${{ parameters.aggregatePipeline }}, False)
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+      )
-        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+    variables:
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+    - group: common
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+    - template: /.azuredevops/variables-global.yml
-        -DCMAKE_BUILD_TYPE=Release
+    - name: PYTHON_VERSION
-        -DENABLE_TESTS=ON
+      value: 3.10
-        -DINSTALL_TESTS=ON
+    pool: ${{ job.target }}_test_pool
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    workspace:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      clean: all
-    parameters:
+    steps:
-      componentName: rocprofiler-compute
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      testDir: $(Build.BinariesDirectory)/libexec/rocprofiler-compute
+      parameters:
-      testExecutable: ROCPROFCOMPUTE_ARCH_OVERRIDE="MI300X" ROCM_PATH=$(Agent.BuildDirectory)/rocm ctest
+        aptPackages: ${{ parameters.aptPackages }}
-  - task: Bash@3
+        pipModules: ${{ parameters.pipModules }}
-    displayName: Remove ROCm binaries from PATH
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-    condition: always()
+    - task: Bash@3
-    inputs:
+      displayName: Add en_US.UTF-8 locale
-      targetType: inline
+      inputs:
-      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+        targetType: inline
-  - task: Bash@3
+        script: |
-    displayName: Remove ROCm compilers from PATH
+          sudo locale-gen en_US.UTF-8
-    condition: always()
+          sudo update-locale
-    inputs:
+          locale -a
-      targetType: inline
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      pipModules: ${{ parameters.pipModules }}
+        postTargetFilter: ${{ job.dependencySource }}
-      environment: test
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmDependencies }}
+        dependencySource: ${{ job.dependencySource }}
+        gpuTarget: ${{ job.target }}
+    - task: Bash@3
+      displayName: Add ROCm binaries to PATH
+      inputs:
+        targetType: inline
+        script: |
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+      parameters:
+        extraBuildFlags: >-
+          -DCMAKE_HIP_ARCHITECTURES=${{ job.target }}
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+          -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+          -DCMAKE_BUILD_TYPE=Release
+          -DENABLE_TESTS=ON
+          -DINSTALL_TESTS=ON
+          -GNinja
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocprofiler-compute
+        testDir: $(Build.BinariesDirectory)/libexec/rocprofiler-compute
+        testExecutable: ROCM_PATH=$(Agent.BuildDirectory)/rocm ctest
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocprofiler-register.yml

@@ -5,10 +5,16 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
     - cmake
+    - ninja-build
     - python3-pip
 
 jobs:
@@ -16,7 +22,8 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -30,21 +37,18 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: rocprofiler-register
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      componentName: rocprofiler-register-tests
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Build.BinariesDirectory)
-      cmakeBuildDir: 'tests/build'
+        -DROCPROFILER_REGISTER_BUILD_TESTS=ON
-      installEnabled: false
+        -DROCPROFILER_REGISTER_BUILD_SAMPLES=ON
+        -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocprofiler-register
-      testDir: 'tests/build'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
-      environment: combined
+  #     environment: combined

.azuredevops/components/rocprofiler-sdk.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -13,6 +18,8 @@ parameters:
     - libdrm-dev
     - libdw-dev
     - libelf-dev
+    - libva-dev
+    - ninja-build
     - pkg-config
     - python3-pip
 - name: pipModules
@@ -29,11 +36,12 @@ parameters:
     - pandas
     - perfetto
     - pycobertura
-    - pytest>=6.2.5
+    - "pytest>=6.2.5"
     - pyyaml
 - name: rocmDependencies
   type: object
   default:
+    - aomp
     - clr
     - llvm-project
     - rccl
@@ -44,118 +52,135 @@ parameters:
     - rocminfo
     - ROCR-Runtime
     - rocprofiler-register
+    - roctracer
+
+- name: jobMatrix
+  type: object
+  default:
+    buildJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
 
 jobs:
-- job: rocprofiler_sdk
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  variables:
+  - job: rocprofiler_sdk_build_${{ job.target }}
-  - group: common
+    variables:
-  - template: /.azuredevops/variables-global.yml
+    - group: common
-  pool: ${{ variables.LOW_BUILD_POOL }}
+    - template: /.azuredevops/variables-global.yml
-  workspace:
+    pool:
-    clean: all
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-  strategy:
+    workspace:
-    matrix:
+      clean: all
-      gfx942:
+    steps:
-        JOB_GPU_TARGET: gfx942
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  steps:
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+        aptPackages: ${{ parameters.aptPackages }}
-    parameters:
+        pipModules: ${{ parameters.pipModules }}
-      aptPackages: ${{ parameters.aptPackages }}
+        registerROCmPackages: true
-      pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      registerROCmPackages: true
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-    parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+        gpuTarget: ${{ job.target }}
-      dependencyList: ${{ parameters.rocmDependencies }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    - task: Bash@3
-  - task: Bash@3
+      displayName: Add Python site-packages binaries to path
-    displayName: Add Python site-packages binaries to path
+      inputs:
-    inputs:
+        targetType: inline
-      targetType: inline
+        script: |
-      script: |
+          USER_BASE=$(python3 -m site --user-base)
-        USER_BASE=$(python3 -m site --user-base)
+          echo "##vso[task.prependpath]$USER_BASE/bin"
-        echo "##vso[task.prependpath]$USER_BASE/bin"
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+      parameters:
-    parameters:
+        extraBuildFlags: >-
-      extraBuildFlags: >-
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+          -DROCPROFILER_BUILD_TESTS=ON
-        -DROCPROFILER_BUILD_TESTS=ON
+          -DROCPROFILER_BUILD_SAMPLES=ON
-        -DROCPROFILER_BUILD_SAMPLES=ON
+          -DROCPROFILER_BUILD_RELEASE=ON
-        -DROCPROFILER_BUILD_RELEASE=ON
+          -DGPU_TARGETS=${{ job.target }}
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+          -GNinja
-      multithreadFlag: -- -j4
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      parameters:
-    parameters:
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
-    parameters:
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    #   parameters:
-    parameters:
+    #     aptPackages: ${{ parameters.aptPackages }}
-      aptPackages: ${{ parameters.aptPackages }}
+    #     pipModules: ${{ parameters.pipModules }}
-      pipModules: ${{ parameters.pipModules }}
+    #     gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    #     registerROCmPackages: true
-      registerROCmPackages: true
 
-- job: rocprofiler_sdk_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  dependsOn: rocprofiler_sdk
+  - job: rocprofiler_sdk_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: rocprofiler_sdk_build_${{ job.target }}
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool: $(JOB_TEST_POOL)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  workspace:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    clean: all
+      )
-  strategy:
+    variables:
-    matrix:
+    - group: common
-      gfx942:
+    - template: /.azuredevops/variables-global.yml
-        JOB_GPU_TARGET: gfx942
+    pool: ${{ job.target }}_test_pool
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    workspace:
-  steps:
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    steps:
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      pipModules: ${{ parameters.pipModules }}
+        aptPackages: ${{ parameters.aptPackages }}
-      registerROCmPackages: true
+        pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        registerROCmPackages: true
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmDependencies }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmDependencies }}
-  - task: Bash@3
+        gpuTarget: ${{ job.target }}
-    displayName: Add Python site-packages binaries to path
+    - task: Bash@3
-    inputs:
+      displayName: Add Python and ROCm binaries to path
-      targetType: inline
+      inputs:
-      script: |
+        targetType: inline
-        USER_BASE=$(python3 -m site --user-base)
+        script: |
-        echo "##vso[task.prependpath]$USER_BASE/bin"
+          USER_BASE=$(python3 -m site --user-base)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+          echo "##vso[task.prependpath]$USER_BASE/bin"
-    parameters:
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
-      extraBuildFlags: >-
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+      parameters:
-        -DROCPROFILER_BUILD_TESTS=ON
+        extraBuildFlags: >-
-        -DROCPROFILER_BUILD_SAMPLES=ON
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DROCPROFILER_BUILD_RELEASE=ON
+          -DROCPROFILER_BUILD_TESTS=ON
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+          -DROCPROFILER_BUILD_SAMPLES=ON
-      multithreadFlag: -- -j16
+          -DROCPROFILER_BUILD_RELEASE=ON
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DGPU_TARGETS=${{ job.target }}
-    parameters:
+          -GNinja
-      componentName: rocprofiler-sdk
+    - template: ${{ variables.CI_TEMPLATE_PATH}}/steps/gpu-diagnostics.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        componentName: rocprofiler-sdk
-      pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      environment: test
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        aptPackages: ${{ parameters.aptPackages }}
-      registerROCmPackages: true
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}
+        registerROCmPackages: true

.azuredevops/components/rocprofiler-systems.yml

@@ -6,6 +6,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -16,6 +21,7 @@ parameters:
     - bzip2
     - clang
     - cmake
+    - chrpath
     - environment-modules
     - ffmpeg
     - g++-12
@@ -24,6 +30,7 @@ parameters:
     - libavutil-dev
     - libdrm-amdgpu-dev
     - libdrm-dev
+    - libdw-dev
     - libfabric-dev
     - libiberty-dev
     - libpapi-dev
@@ -31,6 +38,7 @@ parameters:
     - libtool
     - libopenmpi-dev
     - m4
+    - ninja-build
     - openmpi-bin
     - pkg-config
     - python3-pip
@@ -59,187 +67,168 @@ parameters:
     - rocprofiler-register
     - rocprofiler-sdk
     - ROCR-Runtime
+    - roctracer
+
+- name: jobMatrix
+  type: object
+  default:
+    buildJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
 
 jobs:
-- job: rocprofiler_systems
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  variables:
+  - job: rocprofiler_systems_build_${{ job.target }}
-  - group: common
+    variables:
-  - template: /.azuredevops/variables-global.yml
+    - group: common
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+    - template: /.azuredevops/variables-global.yml
-  workspace:
+    - name: ROCM_PATH
-    clean: all
+      value: $(Agent.BuildDirectory)/rocm
-  strategy:
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-    matrix:
+    workspace:
-      gfx942:
+      clean: all
-        JOB_GPU_TARGET: gfx942
+    steps:
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
-    parameters:
+        aptPackages: ${{ parameters.aptPackages }}
-      aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
-      pipModules: ${{ parameters.pipModules }}
+        registerROCmPackages: true
-      registerROCmPackages: true
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+      parameters:
-    parameters:
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
-    parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmDependencies }}
-      dependencyList: ${{ parameters.rocmDependencies }}
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-  - task: Bash@3
+    - task: Bash@3
-    displayName: ROCm symbolic link
+      displayName: Add ROCm binaries to PATH
-    inputs:
+      inputs:
-      targetType: inline
+        targetType: inline
-      script: |
+        script: |
-        sudo rm -rf /opt/rocm
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
-  - task: Bash@3
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    displayName: Add ROCm binaries to PATH
+      parameters:
-    inputs:
+  # build flags reference: https://rocm.docs.amd.com/projects/omnitrace/en/latest/install/install.html
-      targetType: inline
+        extraBuildFlags: >-
-      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+          -DROCPROFSYS_BUILD_TESTING=ON
-  - task: Bash@3
+          -DROCPROFSYS_BUILD_DYNINST=ON
-    displayName: Add ROCm compilers to PATH
+          -DROCPROFSYS_BUILD_LIBUNWIND=ON
-    inputs:
+          -DROCPROFSYS_DISABLE_EXAMPLES="openmp-target"
-      targetType: inline
+          -DDYNINST_BUILD_TBB=ON
-      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+          -DDYNINST_BUILD_ELFUTILS=ON
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+          -DDYNINST_BUILD_LIBIBERTY=ON
-    parameters:
+          -DDYNINST_BUILD_BOOST=ON
-# build flags reference: https://rocm.docs.amd.com/projects/omnitrace/en/latest/install/install.html
+          -DROCPROFSYS_USE_PAPI=ON
-      extraBuildFlags: >-
+          -DROCPROFSYS_USE_MPI=ON
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+          -DCMAKE_CXX_FLAGS=-I$(Agent.BuildDirectory)/rocm/include/rocjpeg
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+          -DGPU_TARGETS=${{ job.target }}
-        -DROCPROFSYS_BUILD_TESTING=ON
+          -GNinja
-        -DROCPROFSYS_BUILD_DYNINST=ON
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-        -DROCPROFSYS_BUILD_LIBUNWIND=ON
+      parameters:
-        -DROCPROFSYS_DISABLE_EXAMPLES="openmp-target"
+        gpuTarget: ${{ job.target }}
-        -DDYNINST_BUILD_TBB=ON
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-        -DDYNINST_BUILD_ELFUTILS=ON
+      parameters:
-        -DDYNINST_BUILD_LIBIBERTY=ON
+        gpuTarget: ${{ job.target }}
-        -DDYNINST_BUILD_BOOST=ON
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-        -DROCPROFSYS_USE_PAPI=ON
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-        -DROCPROFSYS_USE_MPI=ON
+      parameters:
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        aptPackages: ${{ parameters.aptPackages }}
-      multithreadFlag: -- -j32
+        pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+        registerROCmPackages: true
-      gpuTarget: $(JOB_GPU_TARGET)
+        extraPaths: /home/user/workspace/rocm/bin:/home/user/workspace/rocm/llvm/bin
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      registerROCmPackages: true
-      optSymLink: true
-      extraPaths: /home/user/workspace/rocm/bin:/home/user/workspace/rocm/llvm/bin
 
-- job: rocprofiler_systems_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  dependsOn: rocprofiler_systems
+  - job: rocprofiler_systems_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: rocprofiler_systems_build_${{ job.target }}
-  timeoutInMinutes: 180
+    condition:
-  variables:
+      and(succeeded(),
-  - group: common
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  - template: /.azuredevops/variables-global.yml
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  pool:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    name: $(JOB_TEST_POOL)
+      )
-    demands: firstRenderDeviceAccess
+    timeoutInMinutes: 180
-  workspace:
+    variables:
-    clean: all
+    - group: common
-  strategy:
+    - template: /.azuredevops/variables-global.yml
-    matrix:
+    - name: ROCM_PATH
-      gfx942:
+      value: $(Agent.BuildDirectory)/rocm
-        JOB_GPU_TARGET: gfx942
+    pool:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+      name: ${{ job.target }}_test_pool
-  steps:
+    workspace:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      clean: all
-    parameters:
+    steps:
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      pipModules: ${{ parameters.pipModules }}
+      parameters:
-      registerROCmPackages: true
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+        registerROCmPackages: true
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-      checkoutRepo: ${{ parameters.checkoutRepo }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      dependencyList: ${{ parameters.rocmDependencies }}
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        checkoutRef: ${{ parameters.checkoutRef }}
-  - task: Bash@3
+        dependencyList: ${{ parameters.rocmDependencies }}
-    displayName: ROCm symbolic link
+        gpuTarget: ${{ job.target }}
-    inputs:
+    - task: Bash@3
-      targetType: inline
+      displayName: Add ROCm binaries to PATH
-      script: |
+      inputs:
-        sudo rm -rf /opt/rocm
+        targetType: inline
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        script: |
-  - task: Bash@3
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
-    displayName: Add ROCm binaries to PATH
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
-    inputs:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      targetType: inline
+      parameters:
-      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  # build flags reference: https://rocm.docs.amd.com/projects/omnitrace/en/latest/install/install.html
-  - task: Bash@3
+        extraBuildFlags: >-
-    displayName: Add ROCm compilers to PATH
+          -DROCPROFSYS_BUILD_TESTING=ON
-    inputs:
+          -DROCPROFSYS_BUILD_DYNINST=ON
-      targetType: inline
+          -DROCPROFSYS_BUILD_LIBUNWIND=ON
-      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+          -DROCPROFSYS_DISABLE_EXAMPLES="openmp-target"
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+          -DDYNINST_BUILD_TBB=ON
-    parameters:
+          -DDYNINST_BUILD_ELFUTILS=ON
-# build flags reference: https://rocm.docs.amd.com/projects/omnitrace/en/latest/install/install.html
+          -DDYNINST_BUILD_LIBIBERTY=ON
-      extraBuildFlags: >-
+          -DDYNINST_BUILD_BOOST=ON
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+          -DROCPROFSYS_USE_PAPI=ON
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+          -DROCPROFSYS_USE_MPI=ON
-        -DROCPROFSYS_BUILD_TESTING=ON
+          -DCMAKE_CXX_FLAGS=-I$(Agent.BuildDirectory)/rocm/include/rocjpeg
-        -DROCPROFSYS_BUILD_DYNINST=ON
+          -DGPU_TARGETS=${{ job.target }}
-        -DROCPROFSYS_BUILD_LIBUNWIND=ON
+          -GNinja
-        -DROCPROFSYS_DISABLE_EXAMPLES="openmp-target"
+    - task: Bash@3
-        -DDYNINST_BUILD_TBB=ON
+      displayName: Set up rocprofiler-systems env
-        -DDYNINST_BUILD_ELFUTILS=ON
+      inputs:
-        -DDYNINST_BUILD_LIBIBERTY=ON
+        targetType: inline
-        -DDYNINST_BUILD_BOOST=ON
+        script: source share/rocprofiler-systems/setup-env.sh
-        -DROCPROFSYS_USE_PAPI=ON
+        workingDirectory: build
-        -DROCPROFSYS_USE_MPI=ON
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+      parameters:
-      multithreadFlag: -- -j32
+        componentName: rocprofiler-systems
-  - task: Bash@3
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    displayName: Set up rocprofiler-systems env
+      parameters:
-    inputs:
+        gpuTarget: ${{ job.target }}
-      targetType: inline
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-      script: source share/rocprofiler-systems/setup-env.sh
+      parameters:
-      workingDirectory: build
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+      parameters:
-      componentName: rocprofiler-systems
+        aptPackages: ${{ parameters.aptPackages }}
-  - task: Bash@3
+        pipModules: ${{ parameters.pipModules }}
-    displayName: Remove ROCm binaries from PATH
+        environment: test
-    condition: always()
+        registerROCmPackages: true
-    inputs:
+        gpuTarget: ${{ job.target }}
-      targetType: inline
+        extraPaths: /home/user/workspace/rocm/bin:/home/user/workspace/rocm/llvm/bin
-      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
-  - task: Bash@3
-    displayName: Remove ROCm compilers from PATH
-    condition: always()
-    inputs:
-      targetType: inline
-      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      environment: test
-      registerROCmPackages: true
-      gpuTarget: $(JOB_GPU_TARGET)
-      optSymLink: true
-      extraPaths: /home/user/workspace/rocm/bin:/home/user/workspace/rocm/llvm/bin

.azuredevops/components/rocprofiler.yml

@@ -5,6 +5,12 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
+
 - name: aptPackages
   type: object
   default:
@@ -41,116 +47,125 @@ parameters:
     - rocprofiler-register
     - roctracer
 
-jobs:
+- name: jobMatrix
-- job: rocprofiler
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  - name: HIP_ROCCLR_HOME
+        target: gfx942
-    value: $(Agent.BuildDirectory)/rocm
+      - gfx90a:
-  - name: ROCM_PATH
+        target: gfx90a
-    value: $(Agent.BuildDirectory)/rocm
+    testJobs:
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+      - gfx942:
-  workspace:
+        target: gfx942
-    clean: all
+      - gfx90a:
-  strategy:
+        target: gfx90a
-    matrix:
-      gfx942:
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_MODULE_PATH=$(Build.SourcesDirectory)/cmake_modules;$(Agent.BuildDirectory)/rocm/lib/cmake;$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DENABLE_LDCONFIG=OFF
-        -DUSE_PROF_API=1
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-      multithreadFlag: -- -j32
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      extraEnvVars:
-        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
-        - ROCM_PATH:::/home/user/workspace/rocm
 
-- job: rocprofiler_testing
+jobs:
-  dependsOn: rocprofiler
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rocprofiler_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    - name: HIP_ROCCLR_HOME
-  workspace:
+      value: $(Agent.BuildDirectory)/rocm
-    clean: all
+    - name: ROCM_PATH
-  strategy:
+      value: $(Agent.BuildDirectory)/rocm
-    matrix:
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-      gfx942:
+    workspace:
-        JOB_GPU_TARGET: gfx942
+      clean: all
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    steps:
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
-    parameters:
+        aptPackages: ${{ parameters.aptPackages }}
-      aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
+      parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
+        dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
-  - task: Bash@3
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-    displayName: Setup test environment
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    inputs:
+      parameters:
-      targetType: inline
+        extraBuildFlags: >-
-      script: |
+          -DCMAKE_MODULE_PATH=$(Build.SourcesDirectory)/cmake_modules;$(Agent.BuildDirectory)/rocm/lib/cmake;$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-        sudo rm -rf /opt/rocm
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+          -DENABLE_LDCONFIG=OFF
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DUSE_PROF_API=1
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DGPU_TARGETS=${{ job.target }}
-    parameters:
+          -DAMDGPU_TARGETS=${{ job.target }}
-      componentName: rocprofilerV1
+        multithreadFlag: -- -j32
-      testExecutable: LD_LIBRARY_PATH="$(Agent.BuildDirectory)/rocm/lib/rocprofiler:$(Agent.BuildDirectory)/rocm/share/rocprofiler/tests-v1/test" ./run.sh
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      testDir: $(Agent.BuildDirectory)/rocm/share/rocprofiler/tests-v1
+      parameters:
-      testParameters: ''
+        gpuTarget: ${{ job.target }}
-      testPublishResults: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
-    parameters:
+        gpuTarget: ${{ job.target }}
-      componentName: rocprofilerV2
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-      testExecutable: LD_LIBRARY_PATH="$(Agent.BuildDirectory)/rocm/lib/rocprofiler:$(Agent.BuildDirectory)/rocm/share/rocprofiler/tests" share/rocprofiler/tests/runUnitTests
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+      parameters:
-      testDir: $(Agent.BuildDirectory)/rocm
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        pipModules: ${{ parameters.pipModules }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      aptPackages: ${{ parameters.aptPackages }}
+        extraEnvVars:
-      pipModules: ${{ parameters.pipModules }}
+          - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
-      environment: test
+          - ROCM_PATH:::/home/user/workspace/rocm
-      gpuTarget: $(JOB_GPU_TARGET)
+
-      optSymLink: true
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rocprofiler_test_${{ job.target }}
+    dependsOn: rocprofiler_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    - name: ROCM_PATH
+      value: $(Agent.BuildDirectory)/rocm
+    - name: LD_LIBRARY_PATH
+      value: $(Agent.BuildDirectory)/rocm/lib/rocprofiler:$(Agent.BuildDirectory)/rocm/share/rocprofiler/tests-v1/test:$(Agent.BuildDirectory)/rocm/share/rocprofiler/tests
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocprofilerV1
+        testDir: $(Agent.BuildDirectory)/rocm/share/rocprofiler/tests-v1
+        testExecutable:  ./run.sh
+        testParameters: ''
+        testPublishResults: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rocprofilerV2
+        testDir: $(Agent.BuildDirectory)/rocm
+        testExecutable:  share/rocprofiler/tests/runUnitTests
+        testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/rocr_debug_agent.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -35,12 +40,22 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
+- name: jobMatrix
+  type: object
+  default:
+    testJobs:
+      - gfx942:
+        target: gfx942
+      - gfx90a:
+        target: gfx90a
+
 jobs:
-- job: rocr_debug_agent
+- job: rocr_debug_agent_build
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LOW_BUILD_POOL }}
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -55,6 +70,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -66,54 +82,54 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+  # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
+  #   parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+  #     aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: rocr_debug_agent_testing
+- ${{ each job in parameters.jobMatrix.testJobs }}:
-  dependsOn: rocr_debug_agent
+  - job: rocr_debug_agent_test_${{ job.target }}
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+    dependsOn: rocr_debug_agent_build
-  variables:
+    condition:
-  - group: common
+      and(succeeded(),
-  - template: /.azuredevops/variables-global.yml
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-  pool: $(JOB_TEST_POOL)
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
-  workspace:
+        eq(${{ parameters.aggregatePipeline }}, False)
-    clean: all
+      )
-  strategy:
+    variables:
-    matrix:
+    - group: common
-      gfx942:
+    - template: /.azuredevops/variables-global.yml
-        JOB_GPU_TARGET: gfx942
+    pool: ${{ job.target }}_test_pool
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    workspace:
-  steps:
+      clean: all
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    steps:
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+        aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      checkoutRef: ${{ parameters.checkoutRef }}
+      parameters:
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        checkoutRef: ${{ parameters.checkoutRef }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        dependencyList: ${{ parameters.rocmTestDependencies }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+        gpuTarget: ${{ job.target }}
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      componentName: rocr_debug_agent-tests
+      parameters:
-      extraBuildFlags: >-
+        componentName: rocr_debug_agent-tests
-        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake;$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+        extraBuildFlags: >-
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+          -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake;$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-      cmakeBuildDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-      cmakeSourceDir: '.'
+        cmakeBuildDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'
-      installEnabled: false
+        cmakeSourceDir: '.'
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+        installEnabled: false
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
-      componentName: rocr_debug_agent
+      parameters:
-      testDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'
+        componentName: rocr_debug_agent
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+        testDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'
-    parameters:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      aptPackages: ${{ parameters.aptPackages }}
+      parameters:
-      environment: test
+        aptPackages: ${{ parameters.aptPackages }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/components/roctracer.yml

@@ -5,12 +5,18 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
     - cmake
     - doxygen
     - graphviz
+    - libdrm-amdgpu-dev
     - ninja-build
     - python3-pip
 - name: pipModules
@@ -35,96 +41,115 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
-jobs:
+- name: jobMatrix
-- job: roctracer
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  - name: HIP_ROCCLR_HOME
+        target: gfx942
-    value: $(Build.BinariesDirectory)/rocm
+      - gfx90a:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx90a
-  workspace:
+    testJobs:
-    clean: all
+      - gfx942:
-  strategy:
+        target: gfx942
-    matrix:
+      - gfx90a:
-      gfx942:
+        target: gfx90a
-        JOB_GPU_TARGET: gfx942
-  steps:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_BUILD_TYPE=release
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: roctracer_testing
+jobs:
-  dependsOn: roctracer
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: roctracer_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+    - name: HIP_ROCCLR_HOME
-  workspace:
+      value: $(Build.BinariesDirectory)/rocm
-    clean: all
+    pool:
-  strategy:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    matrix:
+    workspace:
-      gfx942:
+      clean: all
-        JOB_GPU_TARGET: gfx942
+    steps:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-  steps:
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+        aptPackages: ${{ parameters.aptPackages }}
-    parameters:
+        pipModules: ${{ parameters.pipModules }}
-      aptPackages: ${{ parameters.aptPackages }}
+        registerROCmPackages: true
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
+      parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
-    parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
-      checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmDependencies }}
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
-    parameters:
+        extraBuildFlags: >-
-      componentName: roctracer
+          -DCMAKE_BUILD_TYPE=release
-      testExecutable: $(Agent.BuildDirectory)/rocm/share/roctracer/run_tests.sh
+          -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-      testParameters: ''
+          -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-      testDir: $(Agent.BuildDirectory)
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-      testPublishResults: false
+          -DGPU_TARGETS=${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+          -DAMDGPU_TARGETS=${{ job.target }}
-    parameters:
+          -GNinja
-      aptPackages: ${{ parameters.aptPackages }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-      pipModules: ${{ parameters.pipModules }}
+      parameters:
-      environment: test
+        gpuTarget: ${{ job.target }}
-      gpuTarget: $(JOB_GPU_TARGET)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    #   parameters:
+    #     aptPackages: ${{ parameters.aptPackages }}
+    #     pipModules: ${{ parameters.pipModules }}
+    #     gpuTarget: ${{ job.target }}
+    #     registerROCmPackages: true
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: roctracer_test_${{ job.target }}
+    dependsOn: roctracer_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        registerROCmPackages: true
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: roctracer
+        testExecutable: $(Agent.BuildDirectory)/rocm/share/roctracer/run_tests.sh
+        testParameters: ''
+        testDir: $(Agent.BuildDirectory)
+        testPublishResults: false
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}
+        registerROCmPackages: true

.azuredevops/components/rpp.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -45,141 +50,151 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
 
-jobs:
+- name: jobMatrix
-- job: rpp
+  type: object
-  variables:
+  default:
-  - group: common
+    buildJobs:
-  - template: /.azuredevops/variables-global.yml
+      - gfx942:
-  pool: ${{ variables.LOW_BUILD_POOL }}
+        target: gfx942
-  workspace:
+      - gfx90a:
-    clean: all
+        target: gfx90a
-  strategy:
+    testJobs:
-    matrix:
+      - gfx942:
-      gfx942:
+        target: gfx942
-        JOB_GPU_TARGET: gfx942
+      - gfx90a:
-  steps:
+        target: gfx90a
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
-      dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DHALF_INCLUDE_DIRS=$(Agent.BuildDirectory)/rocm/include
-        -DCMAKE_BUILD_TYPE=Release
-        -DGPU_TARGETS=$(JOB_GPU_TARGET)
-        -DROCM_PLATFORM_VERSION=$(NEXT_RELEASE_VERSION)
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-      pipModules: ${{ parameters.pipModules }}
-      gpuTarget: $(JOB_GPU_TARGET)
 
-- job: rpp_testing
+jobs:
-  dependsOn: rpp
+- ${{ each job in parameters.jobMatrix.buildJobs }}:
-  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  - job: rpp_build_${{ job.target }}
-  variables:
+    variables:
-  - group: common
+    - group: common
-  - template: /.azuredevops/variables-global.yml
+    - template: /.azuredevops/variables-global.yml
-  - name: LD_LIBRARY_PATH
+    - name: ROCM_PATH
-    value: $(Agent.BuildDirectory)/rocm/lib;$(Agent.BuildDirectory)/rocm/llvm/lib
+      value: $(Agent.BuildDirectory)/rocm
-  pool: $(JOB_TEST_POOL)
+    pool:
-  workspace:
+      vmImage: ${{ variables.BASE_BUILD_POOL }}
-    clean: all
+    workspace:
-  strategy:
+      clean: all
-    matrix:
+    steps:
-      gfx942:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-        JOB_GPU_TARGET: gfx942
+      parameters:
-        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+        aptPackages: ${{ parameters.aptPackages }}
-  steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        checkoutRepo: ${{ parameters.checkoutRepo }}
-      pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+      parameters:
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+        checkoutRef: ${{ parameters.checkoutRef }}
-    parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
+      parameters:
-      checkoutRef: ${{ parameters.checkoutRef }}
+        extraBuildFlags: >-
-      dependencyList: ${{ parameters.rocmTestDependencies }}
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-      gpuTarget: $(JOB_GPU_TARGET)
-  # Dependencies from: https://github.com/ROCm/rpp/blob/develop/utilities/test_suite/README.md
-  - task: Bash@3
-    displayName: Build and install Turbo JPEG
-    inputs:
-      targetType: 'inline'
-      script: |
-        sudo apt-get install nasm
-        sudo apt-get install wget
-        git clone -b 3.0.2 https://github.com/libjpeg-turbo/libjpeg-turbo.git
-        cd libjpeg-turbo
-        mkdir build
-        cd build
-        cmake -DCMAKE_INSTALL_PREFIX=/usr \
-              -DCMAKE_BUILD_TYPE=RELEASE  \
-              -DENABLE_STATIC=FALSE       \
-              -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib  \
-              -DWITH_JPEG8=TRUE           \
-              ..
-        make -j$nproc
-        sudo make install
-  - task: Bash@3
-    displayName: Build and install Nifti
-    inputs:
-      targetType: 'inline'
-      script: |
-        git clone -b v3.0.1 https://github.com/NIFTI-Imaging/nifti_clib.git
-        cd nifti_clib
-        mkdir build
-        cd build
-        cmake ..
-        sudo make -j$nproc install
-  - task: Bash@3
-    displayName: Build rpp tests
-    inputs:
-      targetType: 'inline'
-      script: |
-        sudo rm -rf /opt/rocm
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-        mkdir rpp-tests
-        cd rpp-tests
-        cmake /opt/rocm/share/rpp/test \
-          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ \
           -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+          -DHALF_INCLUDE_DIRS=$(Agent.BuildDirectory)/rocm/include
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+          -DCMAKE_BUILD_TYPE=Release
-    parameters:
+          -DGPU_TARGETS=${{ job.target }}
-      componentName: rpp
+          -DROCM_PLATFORM_VERSION=$(NEXT_RELEASE_VERSION)
-      testExecutable: 'export PATH=$(Agent.BuildDirectory)/rocm/llvm/bin:$PATH; CC=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang CMAKE_VERBOSE_MAKEFILE=ON VERBOSE=1 ctest'
+          -GNinja
-      testDir: 'rpp-tests'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-  - script: sudo rm /opt/rocm
+      parameters:
-    condition: always()
+        gpuTarget: ${{ job.target }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
-    parameters:
+      parameters:
-      aptPackages: ${{ parameters.aptPackages }}
+        gpuTarget: ${{ job.target }}
-      pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-      environment: test
+    # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
-      gpuTarget: $(JOB_GPU_TARGET)
+    #   parameters:
+    #     aptPackages: ${{ parameters.aptPackages }}
+    #     pipModules: ${{ parameters.pipModules }}
+    #     gpuTarget: ${{ job.target }}
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: rpp_test_${{ job.target }}
+    dependsOn: rpp_build_${{ job.target }}
+    condition:
+      and(succeeded(),
+        eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
+      )
+    variables:
+    - group: common
+    - template: /.azuredevops/variables-global.yml
+    - name: ROCM_PATH
+      value: $(Agent.BuildDirectory)/rocm
+    pool: ${{ job.target }}_test_pool
+    workspace:
+      clean: all
+    steps:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        checkoutRef: ${{ parameters.checkoutRef }}
+        dependencyList: ${{ parameters.rocmTestDependencies }}
+        gpuTarget: ${{ job.target }}
+    # Dependencies from: https://github.com/ROCm/rpp/blob/develop/utilities/test_suite/README.md
+    - task: Bash@3
+      displayName: Build and install Turbo JPEG
+      inputs:
+        targetType: 'inline'
+        script: |
+          sudo apt-get install nasm
+          sudo apt-get install wget
+          git clone -b 3.0.2 https://github.com/libjpeg-turbo/libjpeg-turbo.git
+          cd libjpeg-turbo
+          mkdir build
+          cd build
+          cmake -DCMAKE_INSTALL_PREFIX=/usr \
+                -DCMAKE_BUILD_TYPE=RELEASE  \
+                -DENABLE_STATIC=FALSE       \
+                -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib  \
+                -DWITH_JPEG8=TRUE           \
+                ..
+          make -j$nproc
+          sudo make install
+    - task: Bash@3
+      displayName: Build and install Nifti
+      inputs:
+        targetType: 'inline'
+        script: |
+          git clone -b v3.0.1 https://github.com/NIFTI-Imaging/nifti_clib.git
+          cd nifti_clib
+          mkdir build
+          cd build
+          cmake ..
+          sudo make -j$nproc install
+    - task: Bash@3
+      displayName: Build rpp tests
+      inputs:
+        targetType: 'inline'
+        script: |
+          mkdir rpp-tests
+          cd rpp-tests
+          cmake $(Agent.BuildDirectory)/rocm/share/rpp/test \
+            -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ \
+            -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+      parameters:
+        componentName: rpp
+        testExecutable: 'CMAKE_VERBOSE_MAKEFILE=ON VERBOSE=1 ctest'
+        testDir: 'rpp-tests'
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+        environment: test
+        gpuTarget: ${{ job.target }}

.azuredevops/dependencies/grpc.yml

@@ -38,6 +38,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       cmakeBuildDir: $(Agent.BuildDirectory)/grpc/build
+      cmakeSourceDir: $(Agent.BuildDirectory)/grpc
       extraBuildFlags: >-
         -DgRPC_INSTALL=ON
         -DgRPC_BUILD_TESTS=OFF

.azuredevops/dependencies/gtest.yml

@@ -38,6 +38,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       cmakeBuildDir: $(Agent.BuildDirectory)/googletest/build
+      cmakeSourceDir: $(Agent.BuildDirectory)/googletest
       extraBuildFlags: >-
         -DGTEST_FORCE_SHARED_CRT=ON
         -DCMAKE_DEBUG_POSTFIX=d

.azuredevops/nightly/pytorch.yml

@@ -68,16 +68,16 @@ parameters:
   default:
     - cmake
     - astunparse
-    - expecttest>=0.2.1
+    - "expecttest>=0.2.1"
     - hypothesis
     - numpy
     - psutil
     - pyyaml
     - requests
-    - setuptools
+    - setuptools==75.8.0
     - types-dataclasses
-    - typing-extensions>=4.8.0
+    - "typing-extensions>=4.8.0"
-    - sympy>=1.13.0
+    - "sympy>=1.13.0"
     - filelock
     - networkx
     - jinja2
@@ -85,8 +85,8 @@ parameters:
     - lintrunner
     - ninja
     - packaging
-    - optree>=0.13.0
+    - "optree>=0.13.0"
-    - click>=8.0.3
+    - "click>=8.0.3"
   # list for vision
     - auditwheel
     - future
@@ -170,13 +170,13 @@ jobs:
   - template: /.azuredevops/variables-global.yml
 # various flags/parameters expected by bash scripts in pytorch repo's .ci directory
   - name: ROCM_VERSION
-    value: 6.3.0
+    value: 6.5.0
   - name: ROCM_PATH
     value: /opt/rocm
   - name: DESIRED_CUDA
-    value: 6.3.0
+    value: 6.5.0
-  - name: MKLROOT
+  - name: MAGMA_ROCM
-    value: /opt/intel
+    value: 6.3
   - name: DESIRED_PYTHON
     value: 3.10
   - name: PYTORCH_ROOT
@@ -218,6 +218,7 @@ jobs:
       dependencyList: ${{ parameters.rocmDependencies }}
       dependencySource: staging
       gpuTarget: $(JOB_GPU_TARGET)
+      setupHIPLibrarySymlinks: true
   - task: Bash@3
     displayName: ROCm symbolic link
     inputs:
@@ -252,13 +253,6 @@ jobs:
       script: |
         sudo bash pytorch/.ci/docker/common/install_patchelf.sh
       workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: Install mkl dependency for magma
-    inputs:
-      targetType: inline
-      script: |
-        sudo bash pytorch/.ci/docker/common/install_mkl.sh
-      workingDirectory: $(Build.SourcesDirectory)
   - task: Bash@3
     displayName: Install rocm drm
     inputs:
@@ -271,7 +265,7 @@ jobs:
     inputs:
       targetType: inline
       script: |
-        sudo PYTORCH_ROCM_ARCH=$(JOB_GPU_TARGET) MKLROOT=$(MKLROOT) bash pytorch/.ci/docker/common/install_rocm_magma.sh
+        sudo bash pytorch/.ci/docker/common/install_rocm_magma.sh $(MAGMA_ROCM)
       workingDirectory: $(Build.SourcesDirectory)
   - task: Bash@3
     displayName: Run ROCm Build Script
@@ -279,6 +273,7 @@ jobs:
       targetType: inline
       script: >-
         sudo
+        PYTHONPATH=/home/AzDevOps/.local/lib/python3.10/site-packages:/usr/lib/python3/dist-packages
         DESIRED_CUDA=$(DESIRED_CUDA)
         PYTORCH_ROCM_ARCH=$(JOB_GPU_TARGET)
         GPU_TARGET=$(JOB_GPU_TARGET)

.azuredevops/nightly/rocm-nightly.yml

@@ -1,10 +1,14 @@
 parameters:
-- name: dependencySource
+# using a parameter instead of a strategy matrix allows job variables to be eval'd at compile-time rather than runtime
-  type: string
+- name: jobList
-  default: staging
+  type: object
-  values: 
+  default:
-    - staging
+    - gfx942-staging:
-    - mainline
+      target: gfx942
+      source: staging
+    - gfx90a-staging:
+      target: gfx90a
+      source: staging
 - name: rocmDependencies
   type: object
   default:
@@ -78,62 +82,59 @@ schedules:
   always: true
 
 jobs:
-- job: rocm_nightly
+- ${{ each job in parameters.jobList }}:
-  variables:
+  - job: rocm_nightly_${{ job.target }}_${{ job.source }}
-  - group: common
+    variables:
-  - template: /.azuredevops/variables-global.yml
+    - group: common
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+    - template: /.azuredevops/variables-global.yml
-  workspace:
+    pool: ${{ variables.MEDIUM_BUILD_POOL }}
-    clean: all
+    workspace:
-  strategy:
+      clean: all
-    matrix:
+    steps:
-      gfx942:
+    - task: DeleteFiles@1
-        JOB_GPU_TARGET: gfx942
+      displayName: 'Cleanup checkout space'
-  steps:
+      inputs:
-  - task: DeleteFiles@1
+        SourceFolder: '$(Agent.BuildDirectory)/s'
-    displayName: 'Cleanup checkout space'
+        Contents: '**/*'
-    inputs:
+    - task: DeleteFiles@1
-      SourceFolder: '$(Agent.BuildDirectory)/s'
+      displayName: 'Cleanup Staging Area'
-      Contents: '**/*'
+      inputs:
-  - task: DeleteFiles@1
+        SourceFolder: '$(Build.ArtifactStagingDirectory)'
-    displayName: 'Cleanup Staging Area'
+        Contents: '/**/*'
-    inputs:
+        RemoveDotFiles: true
-      SourceFolder: '$(Build.ArtifactStagingDirectory)'
+    - script: df -h
-      Contents: '/**/*'
+      displayName: System disk space before ROCm
-      RemoveDotFiles: true
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-  - script: df -h
+      parameters:
-    displayName: System disk space before ROCm
+        dependencySource: ${{ job.source }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+        dependencyList: ${{ parameters.rocmDependencies }}
-    parameters:
+        gpuTarget: ${{ job.target }}
-      dependencySource: ${{ parameters.dependencySource }}
+        skipLibraryLinking: true
-      dependencyList: ${{ parameters.rocmDependencies }}
+        skipLlvmSymlink: true
-      gpuTarget: $(JOB_GPU_TARGET)
+    - script: df -h
-      skipLibraryLinking: true
+      displayName: System disk space after ROCm
-      skipLlvmSymlink: true
+    - script: du -sh $(Agent.BuildDirectory)/rocm
-  - script: df -h
+      displayName: Uncompressed ROCm size
-    displayName: System disk space after ROCm
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-  - script: du -sh $(Agent.BuildDirectory)/rocm
+    - task: ArchiveFiles@2
-    displayName: Uncompressed ROCm size
+      displayName: Compress rocm-nightly
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+      inputs:
-  - task: ArchiveFiles@2
+        rootFolderOrFile: $(Agent.BuildDirectory)/rocm
-    displayName: Compress rocm-nightly
+        includeRootFolder: false
-    inputs:
+        archiveType: tar
-      rootFolderOrFile: $(Agent.BuildDirectory)/rocm
+        tarCompression: gz
-      includeRootFolder: false
+        archiveFile: $(Build.ArtifactStagingDirectory)/$(Build.DefinitionName)_$(Build.BuildNumber)_ubuntu2204_${{ job.target }}.tar.gz
-      archiveType: tar
+    - script: du -sh $(Build.ArtifactStagingDirectory)
-      tarCompression: gz
+      displayName: Compressed ROCm size
-      archiveFile: $(Build.ArtifactStagingDirectory)/$(Build.DefinitionName)_$(Build.BuildNumber)_ubuntu2204_$(JOB_GPU_TARGET).tar.gz
+    - task: PublishPipelineArtifact@1
-  - script: du -sh $(Build.ArtifactStagingDirectory)
+      displayName: 'Publish ROCm Nightly Artifact'
-    displayName: Compressed ROCm size
+      retryCountOnTaskFailure: 3
-  - task: PublishPipelineArtifact@1
+      inputs:
-    displayName: 'Publish ROCm Nightly Artifact'
+        targetPath: '$(Build.ArtifactStagingDirectory)'
-    retryCountOnTaskFailure: 3
+    - task: Bash@3
-    inputs:
+      displayName: Save pipeline artifact file name
-      targetPath: '$(Build.ArtifactStagingDirectory)'
+      inputs:
-  - task: Bash@3
+        workingDirectory: $(Pipeline.Workspace)
-    displayName: Save pipeline artifact file name
+        targetType: inline
-    inputs:
+        script: echo "$(Build.DefinitionName)_$(Build.BuildNumber)_ubuntu2204_${{ job.target }}.tar.gz" >> pipelineArtifacts.txt
-      workingDirectory: $(Pipeline.Workspace)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
-      targetType: inline
-      script: echo "$(Build.DefinitionName)_$(Build.BuildNumber)_ubuntu2204_$(JOB_GPU_TARGET).tar.gz" >> pipelineArtifacts.txt
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml

.azuredevops/scripts/download-manifest-artifacts/download-manifest-artifacts.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+import json
+import requests
+import argparse
+from pathlib import Path
+
+def get_builds(entries, gpu_target, output):
+    already_downloaded = {}
+    for entry in entries:
+        already_downloaded = _get_builds(entry, gpu_target, already_downloaded, output)
+
+def _get_builds(entry, gpu_target, already_downloaded, output):
+    print()
+    print(f"{entry['buildNumber']} - {entry['buildId']} - {entry['repoName']}")
+    if already_downloaded.get(entry['buildId']):
+        print('Skipping, already downloaded from build ' + entry['buildId'])
+        return already_downloaded
+
+    artifacts_url = f"https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds/{entry['buildId']}/artifacts?api-version=7.1"
+    artifacts = requests.get(artifacts_url).json()
+    for artifact in artifacts['value']:
+        if 'gfx' in artifact['name'] and gpu_target not in artifact['name']:
+            continue
+
+        print('Artifact name: ' + artifact['name'])
+        print('File size: ~' +
+              str(round(int(artifact['resource']['properties']['artifactsize'])/1000000, 2)) + ' MB')
+        download_url = f"{artifact['resource']['downloadUrl']}"
+        download = requests.get(download_url)
+
+        zip_file = Path(output) / f"{artifact['name']}.zip"
+        with open(zip_file, 'wb') as f:
+            f.write(download.content)
+        already_downloaded[entry['buildId']] = True
+
+    return already_downloaded
+
+def main():
+    parser = argparse.ArgumentParser(description="Command line tool for downloading external ci artifacts")
+    parser.add_argument('--target', type=str, dest="target", choices=["gfx90a", "gfx942"], help="Target gfx")
+    parser.add_argument('--manifest', type=str, dest="manifest", help='JSON manifest url or path to local manifest')
+    parser.add_argument('--output_dir', type=str, dest="output", help='Path to download directory')
+    args = parser.parse_args()
+
+    manifest = args.manifest
+    gpu_target = args.target
+
+    if not gpu_target:
+        print("Enter the GPU target (gfx942, gfx90a)")
+        gpu_target = input()
+
+    if not manifest:
+        print("Enter the manifest file (URL or local path)")
+        manifest = input()
+
+    if 'http' in manifest:
+        data = requests.get(manifest).json()
+    else:
+        with open(manifest, 'r') as f:
+            data = json.load(f)
+
+    entries = [e for e in data['current']]
+    entries.extend([e for e in data['dependencies']])
+    get_builds(entries, gpu_target, args.output)
+
+if __name__ == "__main__":
+    main()

.azuredevops/scripts/download-manifest-artifacts/requirements.txt

@@ -0,0 +1 @@
+requests

.azuredevops/tag-builds/aomp.yml

@@ -3,7 +3,16 @@ variables:
 - template: /.azuredevops/variables-global.yml
 
 parameters:
-- name: checkoutRef
+- name: aompRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+- name: extrasRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+- name: flangRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+- name: llvmRef
   type: string
   default: refs/tags/$(LATEST_RELEASE_TAG)
 
@@ -13,22 +22,22 @@ resources:
     type: github
     endpoint: ROCm
     name: ROCm/aomp
-    ref: ${{ parameters.checkoutRef }}
+    ref: ${{ parameters.aompRef }}
   - repository: aomp-extras_repo
     type: github
     endpoint: ROCm
     name: ROCm/aomp-extras
-    ref: ${{ parameters.checkoutRef }}
+    ref: ${{ parameters.extrasRef }}
   - repository: flang_repo
     type: github
     endpoint: ROCm
     name: ROCm/flang
-    ref: ${{ parameters.checkoutRef }}
+    ref: ${{ parameters.flangRef }}
   - repository: llvm-project_repo
     type: github
     endpoint: ROCm
     name: ROCm/llvm-project
-    ref: amd-staging
+    ref: ${{ parameters.llvmRef }}
 
 trigger: none
 pr: none
@@ -37,4 +46,4 @@ jobs:
   - template: ${{ variables.CI_COMPONENT_PATH }}/aomp.yml
     parameters:
       checkoutRepo: aomp_repo
-      checkoutRef: ${{ parameters.checkoutRef }}
+      checkoutRef: ${{ parameters.aompRef }}

.azuredevops/templates/steps/artifact-download.yml

@@ -12,6 +12,11 @@ parameters:
 - name: fileFilter
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 
 steps:
 - task: Bash@3
@@ -27,16 +32,23 @@ steps:
 - task: DownloadPipelineArtifact@2
   displayName: Download ${{ parameters.componentName }}
   inputs:
-    buildType: 'specific'
+    ${{ if eq(parameters.aggregatePipeline, false) }}:
-    project: ROCm-CI
+      buildType: 'specific'
-    definition: ${{ parameters.pipelineId }}
+      project: ROCm-CI
-    specificBuildWithTriggering: true
+      definition: ${{ parameters.pipelineId }}
-    itemPattern: '**/*${{ parameters.fileFilter }}*'
+      specificBuildWithTriggering: true
-    ${{ if notIn(parameters.componentName, 'aomp') }}: # remove this once these pipelines are functional + up-to-date
+      itemPattern: '**/*${{ parameters.fileFilter }}*'
-      buildVersionToDownload: latestFromBranch # default is 'latest'
+      # aomp is a special case, since the trigger file is under ROCm/ROCm instead of the component repo
-    branchName: refs/heads/${{ parameters.branchName }}
+      ${{ if notIn(parameters.componentName, 'aomp') }}:
-    allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
+        buildVersionToDownload: latestFromBranch # default is 'latest'
-    targetPath: '$(Pipeline.Workspace)/d'
+      branchName: refs/heads/${{ parameters.branchName }}
+      allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
+      targetPath: '$(Pipeline.Workspace)/d'
+    ${{ else }}:
+      buildType: 'current'
+      itemPattern: '**/${{ parameters.componentName }}*${{ parameters.fileFilter }}*'
+      allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
+      targetPath: '$(Pipeline.Workspace)/d'
 - task: ExtractFiles@1
   displayName: Extract ${{ parameters.componentName }}
   inputs:

.azuredevops/templates/steps/artifact-upload.yml

@@ -20,7 +20,7 @@ steps:
     includeRootFolder: false
     archiveType: 'tar'
     tarCompression: 'gz'
-    archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.tar.gz'
+    archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.DefinitionName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.gpuTarget }}_${{ parameters.artifactName }}.tar.gz'
 - task: DeleteFiles@1
   displayName: 'Cleanup Staging Area'
   inputs:
@@ -32,10 +32,11 @@ steps:
   inputs:
     workingDirectory: $(Pipeline.Workspace)
     targetType: inline
-    script: echo "$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.tar.gz" >> pipelineArtifacts.txt
+    script: echo "$(Build.DefinitionName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.gpuTarget }}_${{ parameters.artifactName }}.tar.gz" >> pipelineArtifacts.txt
 # then publish it
 - ${{ if parameters.publish }}:
   - task: PublishPipelineArtifact@1
+    condition: always()
     displayName: '${{ parameters.artifactName }} Publish'
     retryCountOnTaskFailure: 3
     inputs:

.azuredevops/templates/steps/build-cmake.yml

@@ -10,10 +10,10 @@ parameters:
   default: ''
 - name: cmakeBuildDir
   type: string
-  default: 'build'
+  default: $(Agent.BuildDirectory)/s/build
 - name: cmakeSourceDir
   type: string
-  default: '..'
+  default: $(Agent.BuildDirectory)/s
 - name: customBuildTarget
   type: string
   default: ''
@@ -32,6 +32,9 @@ parameters:
 - name: installEnabled
   type: boolean
   default: true
+- name: printDiskSpace
+  type: boolean
+  default: true
 
 steps:
 # create workingDirectory if it does not exist and change into it
@@ -43,9 +46,10 @@ steps:
     ${{ if eq(parameters.customInstallPath, true) }}:
       cmakeArgs: -DCMAKE_INSTALL_PREFIX=${{ parameters.installDir }} ${{ parameters.extraBuildFlags }} ${{ parameters.cmakeSourceDir }}
     ${{ else }}:
-      cmakeArgs: ${{ parameters.extraBuildFlags }} ..
+      cmakeArgs: ${{ parameters.extraBuildFlags }} ${{ parameters.cmakeSourceDir }}
-- script: df -h
+- ${{ if parameters.printDiskSpace }}:
-  displayName: Disk space before build
+  - script: df -h
+    displayName: Disk space before build
 # equivalent to running make $cmakeTargetDir from $cmakeBuildDir
 # i.e., cd $cmakeBuildDir; make $cmakeTargetDir
 - task: CMake@1
@@ -57,8 +61,9 @@ steps:
     ${{ else }}:
       cmakeArgs: '--build ${{ parameters.cmakeTargetDir }} --target ${{ parameters.customBuildTarget }} ${{ parameters.multithreadFlag }}'
     retryCountOnTaskFailure: 10
-- script: df -h
+- ${{ if parameters.printDiskSpace }}:
-  displayName: Disk space after build
+  - script: df -h
+    displayName: Disk space after build
 # equivalent to running make $cmakeTarget from $cmakeBuildDir
 # e.g., make install
 - ${{ if eq(parameters.installEnabled, true) }}:

.azuredevops/templates/steps/checkout.yml

@@ -4,6 +4,12 @@ parameters:
 - name: checkoutRepo
   type: string
   default: 'self'
+- name: sparseCheckout
+  type: boolean
+  default: false
+- name: sparseCheckoutDir
+  type: string
+  default: ''
 # submodule download behaviour
 # change to 'recursive' for repos with submodules
 - name: submoduleBehaviour
@@ -15,3 +21,13 @@ steps:
     clean: true
     submodules: ${{ parameters.submoduleBehaviour }}
     retryCountOnTaskFailure: 3
+    fetchFilter: blob:none
+    ${{ if eq(parameters.sparseCheckout, true) }}:
+      sparseCheckoutDirectories: ${{ parameters.sparseCheckoutDir }}
+      path: sparse
+  - ${{ if eq(parameters.sparseCheckout, true) }}:
+    - task: Bash@3
+      displayName: Symlink sparse checkout
+      inputs:
+        targetType: inline
+        script: ln -s $(Agent.BuildDirectory)/sparse/${{ parameters.sparseCheckoutDir }} $(Agent.BuildDirectory)/s

.azuredevops/templates/steps/dependencies-cmake-latest.yml

@@ -6,5 +6,5 @@ steps:
     targetType: inline
     script: |
       sudo apt purge cmake -y
-      sudo snap install cmake --classic
+      sudo snap install cmake --classic --channel=3.31/stable
       hash -r

.azuredevops/templates/steps/dependencies-other.yml

@@ -34,7 +34,12 @@ steps:
   displayName: 'sudo apt-get update'
   inputs:
     targetType: inline
-    script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes update
+    script: |
+      echo "deb http://archive.ubuntu.com/ubuntu/ jammy main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/default.list
+      echo "deb http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/default.list
+      echo "deb http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/default.list
+      echo "deb http://archive.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/default.list
+      sudo DEBIAN_FRONTEND=noninteractive apt-get --yes update
 - task: Bash@3
   displayName: 'sudo apt-get upgrade'
   inputs:
@@ -50,10 +55,10 @@ steps:
     displayName: 'sudo apt-get install ...'
     inputs:
       targetType: inline
-      script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install ${{ join(' ', parameters.aptPackages) }}
+      script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes --fix-missing install ${{ join(' ', parameters.aptPackages) }}
 - ${{ if gt(length(parameters.pipModules), 0) }}:
   - task: Bash@3
     displayName: 'pip install  ...'
     inputs:
       targetType: inline
-      script: pip install -v ${{ join(' ', parameters.pipModules) }}
+      script: pip install -v --force-reinstall ${{ join(' ', parameters.pipModules) }}

.azuredevops/templates/steps/dependencies-rocm.yml

@@ -31,6 +31,11 @@ parameters:
 - name: setupHIPLibrarySymlinks
   type: boolean
   default: false
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 
 - name: componentVarList
   type: object
@@ -38,7 +43,7 @@ parameters:
     AMDMIGraphX:
       pipelineId: $(AMDMIGRAPHX_PIPELINE_ID)
       stagingBranch: develop
-      mainlineBranch: mainline
+      mainlineBranch: master
       hasGpuTarget: true
     amdsmi:
       pipelineId: $(AMDSMI_PIPELINE_ID)
@@ -53,7 +58,7 @@ parameters:
     aomp:
       pipelineId: $(AOMP_PIPELINE_ID)
       stagingBranch: aomp-dev
-      mainlineBranch: amd-mainline-open
+      mainlineBranch: amd-mainline
       hasGpuTarget: false
     clr:
       pipelineId: $(CLR_PIPELINE_ID)
@@ -143,17 +148,17 @@ parameters:
     llvm-project:
       pipelineId: $(LLVM_PROJECT_PIPELINE_ID)
       stagingBranch: amd-staging
-      mainlineBranch: amd-mainline-open
+      mainlineBranch: amd-mainline
       hasGpuTarget: false
     MIOpen:
       pipelineId: $(MIOpen_PIPELINE_ID)
       stagingBranch: develop
-      mainlineBranch: mainline
+      mainlineBranch: amd-master
       hasGpuTarget: true
     MIVisionX:
       pipelineId: $(MIVISIONX_PIPELINE_ID)
       stagingBranch: develop
-      mainlineBranch: mainline
+      mainlineBranch: master
       hasGpuTarget: true
     omnitrace: # deprecated
       pipelineId: $(OMNITRACE_PIPELINE_ID)
@@ -173,7 +178,7 @@ parameters:
     rocAL:
       pipelineId: $(ROCAL_PIPELINE_ID)
       stagingBranch: develop
-      mainlineBranch: master
+      mainlineBranch: mainline
       hasGpuTarget: true
     rocALUTION:
       pipelineId: $(ROCALUTION_PIPELINE_ID)
@@ -222,13 +227,13 @@ parameters:
       hasGpuTarget: false
     rocm-examples:
       pipelineId: $(ROCM_EXAMPLES_PIPELINE_ID)
-      stagingBranch: develop
+      stagingBranch: amd-staging
-      mainlineBranch: develop
+      mainlineBranch: amd-mainline
       hasGpuTarget: true
     rocminfo:
       pipelineId: $(ROCMINFO_PIPELINE_ID)
       stagingBranch: amd-staging
-      mainlineBranch: amd-master
+      mainlineBranch: amd-mainline
       hasGpuTarget: false
     rocMLIR:
       pipelineId: $(ROCMLIR_PIPELINE_ID)
@@ -238,7 +243,7 @@ parameters:
     ROCmValidationSuite:
       pipelineId: $(ROCMVALIDATIONSUITE_PIPELINE_ID)
       stagingBranch: master
-      mainlineBranch: mainline
+      mainlineBranch: master
       hasGpuTarget: true
     rocm_bandwidth_test:
       pipelineId: $(ROCM_BANDWIDTH_TEST_PIPELINE_ID)
@@ -262,7 +267,7 @@ parameters:
       hasGpuTarget: true
     rocprofiler-compute:
       pipelineId: $(ROCPROFILER_COMPUTE_PIPELINE_ID)
-      stagingBranch: amd-staging
+      stagingBranch: develop
       mainlineBranch: amd-mainline
       hasGpuTarget: true
     rocprofiler-register:
@@ -288,7 +293,7 @@ parameters:
     ROCR-Runtime:
       pipelineId: $(ROCR_RUNTIME_PIPELINE_ID)
       stagingBranch: amd-staging
-      mainlineBranch: amd-master
+      mainlineBranch: amd-mainline
       hasGpuTarget: false
     rocRAND:
       pipelineId: $(ROCRAND_PIPELINE_ID)
@@ -323,7 +328,7 @@ parameters:
     roctracer:
       pipelineId: $(ROCTRACER_PIPELINE_ID)
       stagingBranch: amd-staging
-      mainlineBranch: amd-master
+      mainlineBranch: amd-mainline
       hasGpuTarget: true
     rocWMMA:
       pipelineId: $(ROCWMMA_PIPELINE_ID)
@@ -354,6 +359,7 @@ steps:
       parameters:
         componentName: ${{ split(dependency, ':')[0] }}
         pipelineId: ${{ parameters.componentVarList[split(dependency, ':')[0]].pipelineId }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
         ${{ if parameters.componentVarList[split(dependency, ':')[0]].hasGpuTarget }}:
           fileFilter: "${{ split(dependency, ':')[1] }}*${{ parameters.gpuTarget }}"
         # dependencySource = staging
@@ -383,6 +389,7 @@ steps:
       parameters:
         componentName: ${{ dependency }}
         pipelineId: ${{ parameters.componentVarList[dependency].pipelineId }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
         ${{ if parameters.componentVarList[dependency].hasGpuTarget }}:
           fileFilter: ${{ parameters.gpuTarget }}
         # dependencySource = staging
@@ -456,7 +463,7 @@ steps:
   displayName: 'List downloaded ROCm files'
   inputs:
     targetType: inline
-    script: ls -1R $(Agent.BuildDirectory)/rocm
+    script: ls -la1R $(Agent.BuildDirectory)/rocm
 - ${{ if eq(parameters.skipLibraryLinking, false) }}:
   - task: Bash@3
     displayName: 'Link ROCm shared libraries'

.azuredevops/templates/steps/docker-container-commit.yml

@@ -0,0 +1,82 @@
+# This template creates and uploads a Docker image from the current environment
+# It uses `docker commit` to do so, which can provide more accurate images than the standard template
+# It requires the following conditions:
+# - Job must be run inside a Docker container
+# - The container's external name must be the same as the container's internal hostname
+# - Docker must be installed inside said container and given sufficient permissions
+# Currently, it is only usable for test jobs run on our self-managed systems
+# Jobs run on Azure VMs will not be able to use this template (most if not all build jobs)
+
+parameters:
+- name: gpuTarget
+  type: string
+  default: ''
+- name: environment
+  type: string
+  default: build
+  values:
+    - build
+    - test
+    - combined
+    - amd
+    - nvidia
+- name: extraPaths
+  type: string
+  default: ''
+- name: extraEnvVars
+  type: object
+  default: []
+- name: forceDockerCreation
+  type: boolean
+  default: false
+
+steps:
+  - task: Bash@3
+    displayName: Commit container and initialize Dockerfile
+    condition: or(and(failed(), not(contains(variables['DOCKER_SKIP_GFX'], variables['JOB_GPU_TARGET']))), ${{ eq(parameters.forceDockerCreation, true) }})
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: |
+        docker commit $(hostname) az-ci-temp-image:latest
+        echo "FROM az-ci-temp-image:latest" > Dockerfile
+        echo "RUN sudo groupmod -g $(getent group render | awk -F: '{print $3}') render" >> Dockerfile
+        echo "RUN sudo groupmod -g $(getent group docker | awk -F: '{print $3}') docker" >> Dockerfile
+        echo "ENV PATH='$PATH:${{ parameters.extraPaths }}'" >> Dockerfile
+        echo "ENTRYPOINT [\"/bin/bash\"]" >> Dockerfile
+  - ${{ each extraEnvVar in parameters.extraEnvVars }}:
+    - task: Bash@3
+      displayName: Add extra environment variables
+      condition: or(and(failed(), not(contains(variables['DOCKER_SKIP_GFX'], variables['JOB_GPU_TARGET']))), ${{ eq(parameters.forceDockerCreation, true) }})
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: echo "ENV ${{ split(extraEnvVar, ':::')[0] }}='${{ split(extraEnvVar, ':::')[1] }}'" >> Dockerfile
+  - task: Bash@3
+    displayName: Print Dockerfile
+    condition: or(and(failed(), not(contains(variables['DOCKER_SKIP_GFX'], variables['JOB_GPU_TARGET']))), ${{ eq(parameters.forceDockerCreation, true) }})
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: cat Dockerfile
+  - task: Docker@2
+    displayName: Build and upload Docker image
+    condition: or(and(failed(), not(contains(variables['DOCKER_SKIP_GFX'], variables['JOB_GPU_TARGET']))), ${{ eq(parameters.forceDockerCreation, true) }})
+    inputs:
+      containerRegistry: 'ContainerService3'
+      ${{ if ne(parameters.gpuTarget, '') }}:
+        repository: '$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}'
+      ${{ else }}:
+        repository: '$(Build.DefinitionName)-${{ parameters.environment }}'
+      Dockerfile: '$(Pipeline.Workspace)/Dockerfile'
+      buildContext: '$(Pipeline.Workspace)'
+  - task: Bash@3
+    condition: or(and(failed(), not(contains(variables['DOCKER_SKIP_GFX'], variables['JOB_GPU_TARGET']))), ${{ eq(parameters.forceDockerCreation, true) }})
+    displayName: "!! Docker Image URL !!"
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      ${{ if ne(parameters.gpuTarget, '') }}:
+        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}:$(Build.BuildId)" | tr '[:upper:]' '[:lower:]'
+      ${{ else }}:
+        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}:$(Build.BuildId)" | tr '[:upper:]' '[:lower:]'

.azuredevops/templates/steps/docker-container.yml

@@ -102,12 +102,18 @@ parameters:
   type: boolean
   default: false
 
+- name: dockerSkipGfx
+  type: object
+  default:
+    - gfx90a
+    - gfx942
+
 steps:
 # these steps should only be run if there was a failure or warning
 # dynamically write to a Dockerfile
 # first is to do base setup of users, groups
   - task: Bash@3
-    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
     displayName: Create start of Dockerfile
     inputs:
       workingDirectory: $(Pipeline.Workspace)
@@ -126,7 +132,7 @@ steps:
 # for test jobs, setup GPU-related users and group
   - ${{ if eq(parameters.environment, 'test') }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: GPU setup of Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -137,7 +143,7 @@ steps:
 # now install a common set of packages through apt
   - ${{ if gt(length(parameters.baseAptPackages), 0) }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: Base Apt Packages to Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -146,7 +152,7 @@ steps:
 # iterate through possible apt repos that might need to be added to the docker container
   - ${{ if eq(parameters.registerROCmPackages, true) }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: Register ROCm packages to Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -160,7 +166,7 @@ steps:
           echo "RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update" >> Dockerfile
   - ${{ if eq(parameters.registerCUDAPackages, true) }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: Register CUDA packages to Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -172,7 +178,7 @@ steps:
           echo 'RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update' >> Dockerfile
   - ${{ if eq(parameters.registerJPEGPackages, true) }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: Register libjpeg-turbo packages to Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -185,7 +191,7 @@ steps:
 # install AOCL to docker container, if needed
   - ${{ if eq(parameters.installAOCL, true) }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: aocl install to Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -197,7 +203,7 @@ steps:
 # since apt repo list is updated, install the extra apt packages
   - ${{ if gt(length(parameters.aptPackages), 0) }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: Extra Apt Packages to Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -206,7 +212,7 @@ steps:
 # install latest cmake to docker container, if needed
   - ${{ if eq(parameters.installLatestCMake, true) }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: latest cmake install to Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -216,7 +222,7 @@ steps:
             echo "RUN pip install cmake --upgrade" >> Dockerfile
 # setup workspace where binaries, sources, and dependencies from the job will be copied to
   - task: Bash@3
-    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
     displayName: Workspace setup of Dockerfile
     inputs:
       workingDirectory: $(Pipeline.Workspace)
@@ -228,7 +234,7 @@ steps:
 # pip install is done here as non-root
   - ${{ if gt(length(parameters.pipModules), 0) }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: Extra Python Modules to Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -236,7 +242,7 @@ steps:
         script: echo "RUN pip install -v ${{ join(' ', parameters.pipModules) }}" >> Dockerfile
 # copy common directories
   - task: Bash@3
-    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
     displayName: Copy base directories to Dockerfile
     inputs:
       workingDirectory: $(Pipeline.Workspace)
@@ -254,7 +260,7 @@ steps:
 # copy extra directories, if applicable to the job
   - ${{ each extraCopyDirectory in parameters.extraCopyDirectories }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: Copy ${{ extraCopyDirectory }} to Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -265,7 +271,7 @@ steps:
           fi
 # setup ldconfig
   - task: Bash@3
-    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
     displayName: ldconfig to Dockerfile
     inputs:
       workingDirectory: $(Pipeline.Workspace)
@@ -279,7 +285,7 @@ steps:
 # create /opt/rocm symbolic link, if needed
   - ${{ if eq(parameters.optSymLink, true) }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: /opt/rocm symbolic link to Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -290,7 +296,7 @@ steps:
 # set environment variables needed for some python-based components
   - ${{ if eq(parameters.pythonEnvVars, true) }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: python environment variables
       inputs:
         workingDirectory: $(Pipeline.Workspace)
@@ -303,33 +309,33 @@ steps:
 # add to PATH environment variable
   - ${{ if ne(parameters.extraPaths, '') }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: Add to PATH in Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
         targetType: inline
-        script: echo "ENV PATH='${{ parameters.extraPaths }}:\$PATH'" >> Dockerfile
+        script: echo "ENV PATH='$PATH:${{ parameters.extraPaths }}'" >> Dockerfile
 # set extra environment variables, if applicable to the job
 # use ::: as delimiter to allow for colons to be in the environment variable values
   - ${{ each extraEnvVar in parameters.extraEnvVars }}:
     - task: Bash@3
-      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
       displayName: Set ${{ extraEnvVar }} to Dockerfile
       inputs:
         workingDirectory: $(Pipeline.Workspace)
         targetType: inline
         script: echo "ENV ${{ split(extraEnvVar, ':::')[0] }}='${{ split(extraEnvVar, ':::')[1] }}'" >> Dockerfile
   - task: Bash@3
-    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
     displayName: Print Dockerfile
     inputs:
       workingDirectory: $(Pipeline.Workspace)
       targetType: inline
       script: cat Dockerfile
   - task: Docker@2
-    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
     inputs:
-      containerRegistry: 'ContainerService'
+      containerRegistry: 'ContainerService3'
       ${{ if ne(parameters.gpuTarget, '') }}:
         repository: '$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}'
       ${{ else }}:
@@ -337,12 +343,12 @@ steps:
       Dockerfile: '$(Pipeline.Workspace)/Dockerfile'
       buildContext: '$(Pipeline.Workspace)'
   - task: Bash@3
-    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
     displayName: "!! Docker Image URL !!"
     inputs:
       workingDirectory: $(Pipeline.Workspace)
       targetType: inline
       ${{ if ne(parameters.gpuTarget, '') }}:
-        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}:$(Build.BuildId)"
+        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}:$(Build.BuildId)" | tr '[:upper:]' '[:lower:]'
       ${{ else }}:
-        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}:$(Build.BuildId)"
+        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}:$(Build.BuildId)" | tr '[:upper:]' '[:lower:]'

.azuredevops/templates/steps/gh-actions-post.yml

@@ -0,0 +1,34 @@
+parameters:
+- name: repoName
+  type: string
+  default: ''
+- name: eventType
+  type: string
+  default: ''
+- name: azureBuildNumber
+  type: string
+  default: ''
+- name:  githubActionsToken
+  type: string
+  default: ''
+
+steps:
+- task: Bash@3
+  name: ghActionsPost
+  displayName: Post a request to a github repository with a repository_dispatch workflow
+  continueOnError: true
+  inputs:
+    targetType: inline
+    workingDirectory: $(Build.SourcesDirectory) 
+    script: |
+      curl -L -X POST \
+        -H "Accept: application/vnd.github+json" \
+        -H "Authorization: Bearer ${{ parameters.githubActionsToken }}" \
+        -H "X-GitHub-Api-Version: 2022-11-28" \
+        https://api.github.com/repos/${{ parameters.repoName }}/dispatches \
+        -d '{
+          "event_type": "${{ parameters.eventType }}",
+          "client_payload": {
+            "azure-job-id": "${{ parameters.azureBuildNumber }}"
+          }
+        }'

.azuredevops/templates/steps/local-artifact-download.yml

@@ -1,7 +1,16 @@
+# The default behaviour of this template is to download artifacts from previous jobs in the current pipeline
+# It can be overridden to download any artifact from any pipeline, given the appropriate build/pipeline IDs
+
 parameters:
   - name: gpuTarget
     type: string
     default: ''
+  - name: preTargetFilter
+    type: string
+    default: ''
+  - name: postTargetFilter
+    type: string
+    default: ''
   # Set buildType to specific to download artifacts from previous builds, useful for saving time when debugging
   - name: buildType
     type: string
@@ -9,8 +18,8 @@ parameters:
     values:
       - current
       - specific
-  # Must be set if buildType == specific
+  # One of the below params must be set if buildType == specific
-  # Set definitionId to the pipeline ID and buildId to the specific build ID
+  # Set definitionId to the pipeline ID or buildId to the specific build ID
   - name: definitionId
     type: string
     default: 0
@@ -28,7 +37,7 @@ steps:
         project: ROCm-CI
         definition: ${{ parameters.definitionId }}
         buildId: ${{ parameters.buildId }}
-      itemPattern: '**/*${{ parameters.gpuTarget }}*'
+      itemPattern: '**/*${{ parameters.preTargetFilter }}*${{ parameters.gpuTarget }}*${{ parameters.postTargetFilter }}*'
       targetPath: $(Pipeline.Workspace)/d
   - task: ExtractFiles@1
     displayName: 'Extract Pipeline Build'

.azuredevops/templates/steps/manifest.yml

@@ -8,7 +8,7 @@ parameters:
 
 steps:
 - task: Bash@3
-  displayName: Set up current_repo values
+  displayName: Create manifest.json
   condition: always()
   continueOnError: true
   inputs:
@@ -24,64 +24,76 @@ steps:
       cat resources.repositories
 
       IS_TAG_BUILD=$(jq 'has("release_repo")' resources.repositories)
-      if [ "$IS_TAG_BUILD" = "true" ]; then
+      IS_AOMP_BUILD=$(jq 'has("aomp_repo")' resources.repositories)
-        REPO_TYPE="release_repo" # Triggered by a ROCm/ROCm tag-builds file
+
+      if [ "$IS_TAG_BUILD" = "true" ] || [ "$IS_AOMP_BUILD" = "true" ]; then
+        exclude_keys=("pipelines_repo" "self") # Triggered by a file under ROCm/ROCm
       else
-        REPO_TYPE="self" # Triggered by component repo's rocm-ci.yml file
+        exclude_keys=("pipelines_repo") # Triggered by a file under a component repo
       fi
 
-      echo "##vso[task.setvariable variable=current_repo.id;]$(jq .$REPO_TYPE.id resources.repositories | tr -d '"')"
+      exclude_keys_string=$(printf '"%s", ' "${exclude_keys[@]}")
-      echo "##vso[task.setvariable variable=current_repo.name;]$(jq .$REPO_TYPE.name resources.repositories | tr -d '"')"
+      exclude_keys_string=${exclude_keys_string%, }
-      echo "##vso[task.setvariable variable=current_repo.ref;]$(jq .$REPO_TYPE.ref resources.repositories | tr -d '"')"
+
-      echo "##vso[task.setvariable variable=current_repo.url;]$(jq .$REPO_TYPE.url resources.repositories | tr -d '"')"
+      current=$(jq --argjson exclude "[$exclude_keys_string]" '
-      echo "##vso[task.setvariable variable=current_repo.version;]$(jq .$REPO_TYPE.version resources.repositories | tr -d '"')"
+        reduce to_entries[] as $entry (
-- task: Bash@3
+          [];
-  displayName: Create manifest.json
+          if ($exclude | index($entry.key) | not)
-  condition: always()
+          then . + [
-  continueOnError: true
+            {
-  inputs:
+              buildNumber: "$(Build.BuildNumber)",
-    targetType: inline
+              buildId: "$(Build.BuildId)",
-    script: |
+              repoId: $entry.value.id,
-      manifest_json=$(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.json
+              repoName: $entry.value.name,
+              repoRef: $entry.value.ref,
+              repoUrl: $entry.value.url,
+              repoVersion: $entry.value.version
+            }
+          ]
+          else .
+          end
+        )
+      ' resources.repositories)
+
+      manifest_json=$(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.gpuTarget }}_${{ parameters.artifactName }}.json
 
       dependencies=()
       for manifest_file in $(Pipeline.Workspace)/d/**/manifest_*.json; do
         echo "Processing $manifest_file"
-        cat $manifest_file
+        file=$(jq -c '.current | if type=="array" then .[] else . end' "$manifest_file")
-        current=$(jq '.current' "$manifest_file")
+        while IFS= read -r line; do
-        dependencies+=("$current")
+          dependencies+=("$line")
+        done <<< "$file"
       done
       dependencies_json=$(printf '%s\n' "${dependencies[@]}" | jq -s '.')
 
       jq -n \
-        --arg buildNumber "$(Build.BuildNumber)" \
+        --argjson current "$current" \
-        --arg buildId "$(Build.BuildId)" \
-        --arg repoId "$(current_repo.id)" \
-        --arg repoName "$(current_repo.name)" \
-        --arg repoRef "$(current_repo.ref)" \
-        --arg repoUrl "$(current_repo.url)" \
-        --arg repoVersion "$(current_repo.version)" \
         --argjson dependencies "$dependencies_json" \
         '{
-          current: {
+          current: $current,
-            buildNumber: $buildNumber,
-            buildId: $buildId,
-            repoId: $repoId,
-            repoName: $repoName,
-            repoRef: $repoRef,
-            repoUrl: $repoUrl,
-            repoVersion: $repoVersion
-          },
           dependencies: $dependencies
         }' > $manifest_json
 
+      current_rows=$(cat $manifest_json | \
+        jq -r '
+          .current[] |
+          "<tr><td>" + .buildNumber + "</td>" +
+          "<td><a href=\"https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=" + .buildId + "\">" + .buildId + "</a></td>" +
+          "<td><a href=\"" + .repoUrl + "\">" + .repoName + "</a></td>" +
+          "<td><a href=\"" + .repoUrl + "/tree/" + .repoRef + "\">" + .repoRef + "</a></td>" +
+          "<td><a href=\"" + .repoUrl + "/commit/" + .repoVersion + "\">" + .repoVersion + "</a></td></tr>"
+        ')
+      current_rows=$(echo $current_rows)
+      echo "##vso[task.setvariable variable=current_rows;]$current_rows"
+
       dependencies_rows=$(cat $manifest_json | \
         jq -r '
-          .dependencies[] | 
+          .dependencies[] |
-          "<tr><td>" + .buildNumber + "</td>" + 
+          "<tr><td>" + .buildNumber + "</td>" +
-          "<td><a href=\"https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=" + .buildId + "\">" + .buildId + "</a></td>" + 
+          "<td><a href=\"https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=" + .buildId + "\">" + .buildId + "</a></td>" +
-          "<td><a href=\"" + .repoUrl + "\">" + .repoName + "</a></td>" + 
+          "<td><a href=\"" + .repoUrl + "\">" + .repoName + "</a></td>" +
-          "<td><a href=\"" + .repoUrl + "/tree/" + .repoRef + "\">" + .repoRef + "</a></td>" + 
+          "<td><a href=\"" + .repoUrl + "/tree/" + .repoRef + "\">" + .repoRef + "</a></td>" +
           "<td><a href=\"" + .repoUrl + "/commit/" + .repoVersion + "\">" + .repoVersion + "</a></td></tr>"
         ')
       dependencies_rows=$(echo $dependencies_rows)
@@ -95,7 +107,7 @@ steps:
   inputs:
     targetType: inline
     script: |
-      manifest_html=$(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.html
+      manifest_html=$(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.gpuTarget }}_${{ parameters.artifactName }}.html
       cat <<EOF > $manifest_html
       <html>
       <h1>Manifest</h1>
@@ -108,13 +120,7 @@ steps:
         <th>Repo Ref</th>
         <th>Repo Version</th>
       </tr>
-      <tr>
+      $(current_rows)
-        <td>$(Build.BuildNumber)</td>
-        <td><a href="https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$(Build.BuildId)">$(Build.BuildId)</a></td>
-        <td><a href="$(current_repo.url)">$(current_repo.name)</a></td>
-        <td><a href="$(current_repo.url)/tree/$(current_repo.ref)">$(current_repo.ref)</a></td>
-        <td><a href="$(current_repo.url)/commit/$(current_repo.version)">$(current_repo.version)</a></td>
-      </tr>
       </table>
       <h2>Dependencies</h2>
       <table border="1">
@@ -130,6 +136,11 @@ steps:
       </html>
       EOF
 
+      sed -i -e 's|</tr> <tr>|</tr>\n<tr>|g' \
+        -e 's|</td><td>|</td>\n  <td>|g' \
+        -e 's|<tr><td>|<tr>\n  <td>|g' \
+        -e 's|</td></tr>|</td>\n</tr>|g' $manifest_html
+
       cat $manifest_html
 - task: PublishHtmlReport@1
   displayName: Publish manifest.html
@@ -137,7 +148,7 @@ steps:
   continueOnError: true
   inputs:
     tabName: Manifest
-    reportDir: $(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.html
+    reportDir: $(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.gpuTarget }}_${{ parameters.artifactName }}.html
 - task: Bash@3
   displayName: Save manifest artifact file name
   condition: always()
@@ -146,5 +157,5 @@ steps:
     workingDirectory: $(Pipeline.Workspace)
     targetType: inline
     script: |
-      echo "manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.html" >> pipelineArtifacts.txt
+      echo "manifest_$(Build.DefinitionName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.gpuTarget }}_${{ parameters.artifactName }}.html" >> pipelineArtifacts.txt
-      echo "manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.json" >> pipelineArtifacts.txt
+      echo "manifest_$(Build.DefinitionName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.gpuTarget }}_${{ parameters.artifactName }}.json" >> pipelineArtifacts.txt

.azuredevops/templates/steps/miopen-get-ck-build.yml

@@ -17,7 +17,7 @@ steps:
     script: |
       AZ_API="https://dev.azure.com/ROCm-CI/ROCm-CI/_apis"
       GH_API="https://api.github.com/repos/ROCm"
-      ARTIFACT_NAME="composablekernel.${{ parameters.gpuTarget }}"
+      ARTIFACT_NAME="composablekernelbuild${{ parameters.gpuTarget }}"
       EXIT_CODE=0
 
       # Try to find an Azure build for the specific CK commit called out in MIOpen's requirements.txt
@@ -54,7 +54,7 @@ steps:
       fi
 
       echo "Downloading CK artifact from $ARTIFACT_URL"
-      wget -nv $ARTIFACT_URL -O $(System.ArtifactsDirectory)/ck.zip
+      wget --tries=5 --waitretry=10 --retry-connrefused -nv $ARTIFACT_URL -O $(System.ArtifactsDirectory)/ck.zip
       unzip $(System.ArtifactsDirectory)/ck.zip -d $(System.ArtifactsDirectory)
       mkdir -p $(Agent.BuildDirectory)/rocm
       tar -zxvf $(System.ArtifactsDirectory)/$ARTIFACT_NAME/*.tar.gz -C $(Agent.BuildDirectory)/rocm

.azuredevops/templates/steps/preamble.yml

@@ -3,17 +3,39 @@
 # also display installed components and packages
 steps:
 - task: Bash@3
-  displayName: 'apt installed list'
+  displayName: List apt packages
   inputs:
     targetType: inline
     script: apt list --installed
 - task: Bash@3
-  displayName: 'python version'
+  displayName: Print Python version
   inputs:
     targetType: inline
     script: python3 --version
-- script: pip list -v
+- task: Bash@3
-  displayName: 'list python packages'
+  displayName: List Python packages
+  inputs:
+    targetType: inline
+    script: pip list -v
+# The "Azure Pipelines" agents install CMake in multiple ways, including a standalone install into /usr/local/bin:
+# https://github.com/actions/runner-images/blob/6d939a3ab352a54a021dd67b071577287b6f14a5/images/ubuntu/scripts/build/install-cmake.sh#L27
+# This standalone CMake does not have a fixed version, and is not the same version as the one installed by the package manager
+# We want to use the CMake installed by the package manager, so just remove any bins from the standalone install
+- task: Bash@3
+  displayName: Remove CMake binaries from /usr/local/bin
+  inputs:
+    targetType: inline
+    script: |
+      sudo rm -f /usr/local/bin/ccmake
+      sudo rm -f /usr/local/bin/cmake
+      sudo rm -f /usr/local/bin/cmake-gui
+- task: Bash@3
+  displayName: Print CMake info
+  inputs:
+    targetType: inline
+    script: |
+      cmake --version
+      which cmake
 - task: DeleteFiles@1
   displayName: 'Cleanup checkout space'
   inputs:

.azuredevops/templates/steps/test.yml

@@ -10,7 +10,7 @@ parameters:
   default: 'ctest'
 - name: testParameters
   type: string
-  default: '-VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml'
+  default: '--output-on-failure --force-new-ctest-process --output-junit test_output.xml'
 - name: testOutputFile
   type: string
   default: test_output.xml
@@ -33,7 +33,6 @@ parameters:
     - aomp
     - HIPIFY
     - MIVisionX
-    - rocm-cmake
     - rocm_smi_lib
     - rocprofiler-sdk
     - roctracer

.azuredevops/variables-global.yml

@@ -27,14 +27,18 @@ variables:
   value: rocm-ci_larger_base_disk_pool
 - name: GFX942_TEST_POOL
   value: gfx942_test_pool
+- name: GFX90A_TEST_POOL
+  value: gfx90a_test_pool
 - name: LATEST_RELEASE_VERSION
-  value: 6.3.2
-- name: REPO_RADEON_VERSION
-  value: 6.3.2
-- name: NEXT_RELEASE_VERSION
   value: 6.4.0
+- name: REPO_RADEON_VERSION
+  value: 6.4
+- name: NEXT_RELEASE_VERSION
+  value: 6.5.0
 - name: LATEST_RELEASE_TAG
-  value: rocm-6.3.2
+  value: rocm-6.4.0
+- name: DOCKER_SKIP_GFX
+  value: gfx90a
 - name: AMDMIGRAPHX_GFX942_TEST_PIPELINE_ID
   value: 197
 - name: AMDMIGRAPHX_PIPELINE_ID

.gitignore

@@ -5,15 +5,16 @@ build
 # documentation artifacts
 _build/
 _images/
+__pycache__/
 _static/
 _templates/
 _toc.yml
 docBin/
 _doxygen/
 _readthedocs/
+__pycache__/
 
 # avoid duplicating contributing.md due to conf.py
-docs/CHANGELOG.md
 docs/contribute/index.md
 docs/about/release-notes.md
-docs/about/changelog.md
+docs/release/changelog.md

.markdownlint-cli2.yaml

@@ -12,6 +12,5 @@ config:
   MD041: false
   MD051: false
 ignores:
-  - CHANGELOG.md
+  - "{,docs/}{RELEASE,release,CHANGELOG,changelog}.md"
-  - "{,docs/}{RELEASE,release}.md"
   - tools/autotag/templates/**/*.md

.wordlist.txt

@@ -18,12 +18,14 @@ APBDIS
 APIC
 APIs
 APU
+APUs
 ASIC
 ASICs
 ASan
 ASAN
 ASm
 ATI
+atomicRMW
 AddressSanitizer
 AlexNet
 Andrej
@@ -32,10 +34,12 @@ Autocast
 BARs
 BLAS
 BMC
+BabelStream
 Blit
 Blockwise
 Bluefield
 Bootloader
+CAS
 CCD
 CDNA
 CHTML
@@ -73,11 +77,14 @@ Concretized
 Conda
 ConnectX
 CuPy
+da
 Dashboarding
 DBRX
 DDR
 DF
 DGEMM
+dGPU
+dGPUs
 DIMM
 DKMS
 DL
@@ -98,6 +105,7 @@ DeepSpeed
 Dependabot
 Deprecations
 DevCap
+DirectX
 Dockerfile
 Doxygen
 ELMo
@@ -117,6 +125,7 @@ FX
 Filesystem
 FindDb
 Flang
+FluxBenchmark
 Fortran
 Fuyu
 GALB
@@ -130,11 +139,14 @@ GDR
 GDS
 GEMM
 GEMMs
+GFLOPS
 GFortran
+GFXIP
 Gemma
 GiB
 GIM
 GL
+Glibc
 GLXT
 Gloo
 GMI
@@ -175,6 +187,7 @@ IDE
 IDEs
 IFWI
 IMDb
+IncDec
 IOMMU
 IOP
 IOPM
@@ -215,6 +228,7 @@ LM
 LSAN
 LSan
 LTS
+LanguageCrossEntropy
 LoRA
 MEM
 MERCHANTABILITY
@@ -232,6 +246,7 @@ MMIOH
 MMU
 MNIST
 MPI
+MPT
 MSVC
 MVAPICH
 MVFFR
@@ -239,6 +254,7 @@ Makefile
 Makefiles
 Matplotlib
 Matrox
+MaxText
 Megatrends
 Megatron
 Mellanox
@@ -247,6 +263,7 @@ Meta's
 Miniconda
 MirroredStrategy
 Mixtral
+MosaicML
 Multicore
 Multithreaded
 MyEnvironment
@@ -270,6 +287,7 @@ NaN
 Nano
 Navi
 Noncoherently
+NoReturn
 NousResearch's
 NumPy
 OAM
@@ -316,6 +334,8 @@ PipelineParallel
 PnP
 PowerEdge
 PowerShell
+Pretrained
+Pretraining
 Profiler's
 PyPi
 Pytest
@@ -377,6 +397,7 @@ SIMDs
 SKU
 SKUs
 SLES
+SLURM
 SMEM
 SMI
 SMT
@@ -436,6 +457,7 @@ UMC
 USM
 UTCL
 UTIL
+UltraChat
 Uncached
 Unittests
 Unhandled
@@ -478,6 +500,7 @@ ZenDNN
 accuracies
 activations
 addr
+ai
 alloc
 allocatable
 allocator
@@ -498,15 +521,19 @@ bfloat
 bilinear
 bitcode
 bitsandbytes
+bitwise
+Bitwise
 blit
 bootloader
 boson
 bosons
 br
+BrainFloat
 buildable
 bursty
 bzip
 cacheable
+carveout
 cd
 centos
 centric
@@ -543,6 +570,7 @@ cTDP
 dataset
 datasets
 dataspace
+datatemplate
 datatype
 datatypes
 dbgapi
@@ -567,10 +595,12 @@ distro
 distros
 dkms
 dtype
+eb
 el
 embeddings
 enablement
 encodings
+endfor
 endpgm
 enqueue
 env
@@ -583,6 +613,7 @@ ffmpeg
 filesystem
 fortran
 fp
+framebuffer
 gRPC
 galb
 gcc
@@ -612,6 +643,7 @@ hipSPARSELt
 hipTensor
 hipamd
 hipblas
+hipcc
 hipcub
 hipfft
 hipfort
@@ -619,6 +651,7 @@ hipify
 hipsolver
 hipsparse
 hlist
+hostname
 hotspotting
 hpc
 hpp
@@ -665,6 +698,7 @@ macOS
 matchers
 microarchitecture
 migraphx
+migratable
 miopen
 miopengemm
 mivisionx
@@ -677,6 +711,9 @@ mvffr
 namespace
 namespaces
 nanoGPT
+NCS
+NOP
+NVLink
 num
 numref
 ocl
@@ -691,6 +728,7 @@ pageable
 pallas
 parallelization
 parallelizing
+param
 parameterization
 passthrough
 perfcounter
@@ -715,11 +753,13 @@ preprocessing
 preprocessor
 prequantized
 prerequisites
+pretraining
 profiler
 profilers
 protobuf
 pseudorandom
 py
+pytorch
 recommender
 recommenders
 quantile
@@ -807,6 +847,7 @@ supercomputing
 symlink
 symlinks
 sys
+tabindex
 td
 tensorfloat
 th
@@ -825,6 +866,7 @@ tracebacks
 txt
 uarch
 uncached
+uncacheable
 uncorrectable
 underoptimized
 unhandled
@@ -852,6 +894,7 @@ vectorizes
 virtualize
 virtualized
 vjxb
+vllm
 voxel
 walkthrough
 walkthroughs
@@ -868,6 +911,7 @@ wrreq
 wzo
 xargs
 xGMI
+xPacked
 xz
 yaml
 ysvmadyb

CONTRIBUTING.md

@@ -66,11 +66,10 @@ project-specific steps. Refer to each repository's PR process for any additional
   during our release cycle, as coordinated by the maintainer
 * We'll inform you once your change is committed
 
-:::{important}
+> [!IMPORTANT]
-By creating a PR, you agree to allow your contribution to be licensed under the
+> By creating a PR, you agree to allow your contribution to be licensed under the
-terms of the LICENSE.txt file in the corresponding repository. Different repositories may use different
+> terms of the LICENSE.txt file in the corresponding repository. Different repositories may use different
-licenses.
+> licenses.
-:::
 
 You can look up each license on the [ROCm licensing](https://rocm.docs.amd.com/en/latest/about/license.html) page.
 

README.md

@@ -50,8 +50,8 @@ The following example shows how to use the repo tool to download the ROCm source
 ```bash
 mkdir -p ~/ROCm/
 cd ~/ROCm/
-export ROCM_VERSION=6.3.3
+export ROCM_VERSION=6.4.0
-~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.3.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
+~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.4.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
 ~/bin/repo sync
 ```
 
@@ -77,8 +77,8 @@ The Build time will reduce significantly if we limit the GPU Architecture/s agai
 
 mkdir -p ~/WORKSPACE/      # Or any folder name other than WORKSPACE
 cd ~/WORKSPACE/
-export ROCM_VERSION=6.3.3
+export ROCM_VERSION=6.4.0
-~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.3.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
+~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.4.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
 ~/bin/repo sync
 
 # --------------------------------------
@@ -87,12 +87,10 @@ export ROCM_VERSION=6.3.3
 
 # Option 1: Start a docker container
 # Pulling required base docker images:
-# Ubuntu20.04 built from ROCm/tools/rocm-build/docker/ubuntu20/Dockerfile
-docker pull rocm/rocm-build-ubuntu-20.04:6.3
 # Ubuntu22.04 built from ROCm/tools/rocm-build/docker/ubuntu22/Dockerfile
-docker pull rocm/rocm-build-ubuntu-22.04:6.3
+docker pull rocm/rocm-build-ubuntu-22.04:6.4
 # Ubuntu24.04 built from ROCm/tools/rocm-build/docker/ubuntu24/Dockerfile
-docker pull rocm/rocm-build-ubuntu-24.04:6.3
+docker pull rocm/rocm-build-ubuntu-24.04:6.4
 
 # Start docker container and mount the source code folder:
 docker run -ti \
@@ -109,14 +107,10 @@ docker run -ti \
     <replace_with_required_ubuntu_base_docker_image> bash
 
 # Option 2: Install required packages into the host machine
-# For ubuntu20.04 system
-cd ROCm/tools/rocm-build/docker/ubuntu20
-cp * /tmp && cd /tmp
-bash install-prerequisites.sh
 # For ubuntu22.04 system
 cd ROCm/tools/rocm-build/docker/ubuntu22
 cp * /tmp && cd /tmp
-bash install-prerequisities.sh
+bash install-prerequisites.sh
 # For ubuntu24.04 system
 cd ROCm/tools/rocm-build/docker/ubuntu24
 cp * /tmp && cd /tmp
@@ -143,15 +137,11 @@ make -f ROCm/tools/rocm-build/ROCm.mk list_components
 # Build a single ROCm packages
 make -f ROCm/tools/rocm-build/ROCm.mk T_rocblas
 
-# Find built packages in ubuntu20.04:
-out/ubuntu-20.04/20.04/deb/
 # Find built packages in ubuntu22.04:
 out/ubuntu-22.04/22.04/deb/
 # Find built packages in ubuntu24.04:
 out/ubuntu-24.04/24.04/deb/
 
-# Find built logs in ubuntu20.04:
-out/ubuntu-20.04/20.04/logs/
 # Find built logs in ubuntu22.04:
 out/ubuntu-22.04/22.04/logs/
 # Find built logs in ubuntu24.04:

default.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <manifest>
     <remote name="rocm-org" fetch="https://github.com/ROCm/" />
-    <default revision="refs/tags/rocm-6.3.3"
+    <default revision="refs/tags/rocm-6.4.0"
      remote="rocm-org"
      sync-c="true"
      sync-j="4" />
@@ -30,6 +30,7 @@
 <!-- The following projects are all associated with the AMDGPU LLVM compiler -->
     <project name="half" />
     <project name="llvm-project" />
+    <project name="spirv-llvm-translator" />
 <!-- gdb projects -->
     <project name="ROCdbgapi" />
     <project name="ROCgdb" />
@@ -62,6 +63,7 @@
     <project groups="mathlibs" name="rocFFT" />
     <project groups="mathlibs" name="rocPRIM" />
     <project groups="mathlibs" name="rocRAND" />
+    <project groups="mathlibs" name="rocSHMEM" />
     <project groups="mathlibs" name="rocSOLVER" />
     <project groups="mathlibs" name="rocSPARSE" />
     <project groups="mathlibs" name="rocThrust" />

docs/about/license.md

@@ -25,7 +25,7 @@ additional licenses. Please review individual repositories for more information.
 <!-- spellcheck-disable -->
 | Component | License |
 |:---------------------|:-------------------------|
-| [AMD Compute Language Runtime (CLR)](https://github.com/ROCm/clr) | [MIT](https://github.com/ROCm/clr/blob/amd-staging/LICENCE) |
+| [AMD Compute Language Runtime (CLR)](https://github.com/ROCm/clr) | [MIT](https://github.com/ROCm/clr/blob/amd-staging/LICENSE.txt) |
 | [AMD SMI](https://github.com/ROCm/amdsmi) | [MIT](https://github.com/ROCm/amdsmi/blob/amd-staging/LICENSE) |
 | [aomp](https://github.com/ROCm/aomp/) | [Apache 2.0](https://github.com/ROCm/aomp/blob/aomp-dev/LICENSE) |
 | [aomp-extras](https://github.com/ROCm/aomp-extras/) | [MIT](https://github.com/ROCm/aomp-extras/blob/aomp-dev/LICENSE) |
@@ -77,10 +77,11 @@ additional licenses. Please review individual repositories for more information.
 | [rocPRIM](https://github.com/ROCm/rocPRIM/) | [MIT](https://github.com/ROCm/rocPRIM/blob/develop/LICENSE.txt) |
 | [ROCProfiler](https://github.com/ROCm/rocprofiler/) | [MIT](https://github.com/ROCm/rocprofiler/blob/amd-staging/LICENSE) |
 | [ROCprofiler-SDK](https://github.com/ROCm/rocprofiler-sdk) | [MIT](https://github.com/ROCm/rocprofiler-sdk/blob/amd-mainline/LICENSE) |
-| [rocPyDecode](https://github.com/ROCm/rocPyDecode) | [MIT](https://github.com/ROCm/rocPyDecode/blob/develop/LICENSE) |
+| [rocPyDecode](https://github.com/ROCm/rocPyDecode) | [MIT](https://github.com/ROCm/rocPyDecode/blob/develop/LICENSE.txt) |
 | [rocRAND](https://github.com/ROCm/rocRAND/) | [MIT](https://github.com/ROCm/rocRAND/blob/develop/LICENSE.txt) |
 | [ROCr Debug Agent](https://github.com/ROCm/rocr_debug_agent/) | [The University of Illinois/NCSA](https://github.com/ROCm/rocr_debug_agent/blob/amd-staging/LICENSE.txt) |
 | [ROCR-Runtime](https://github.com/ROCm/ROCR-Runtime/) | [The University of Illinois/NCSA](https://github.com/ROCm/ROCR-Runtime/blob/amd-staging/LICENSE.txt) |
+| [rocSHMEM](https://github.com/ROCm/rocSHMEM/) | [MIT](https://github.com/ROCm/rocSHMEM/blob/develop/LICENSE.md) |
 | [rocSOLVER](https://github.com/ROCm/rocSOLVER/) | [BSD-2-Clause](https://github.com/ROCm/rocSOLVER/blob/develop/LICENSE.md) |
 | [rocSPARSE](https://github.com/ROCm/rocSPARSE/) | [MIT](https://github.com/ROCm/rocSPARSE/blob/develop/LICENSE.md) |
 | [rocThrust](https://github.com/ROCm/rocThrust/) | [Apache 2.0](https://github.com/ROCm/rocThrust/blob/develop/LICENSE) |

docs/compatibility/compatibility-matrix-historical-6.0.csv

@@ -1,120 +1,121 @@
-ROCm Version,6.3.3,6.3.2,6.3.1,6.3.0,6.2.4,6.2.2,6.2.1,6.2.0, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
+ROCm Version,6.4.0,6.3.3,6.3.2,6.3.1,6.3.0,6.2.4,6.2.2,6.2.1,6.2.0, 6.1.5, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
-      :ref:`Operating systems & kernels <OS-kernel-versions>`,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04.2,"Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04",Ubuntu 24.04,,,,,
+      :ref:`Operating systems & kernels <OS-kernel-versions>`,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04.2,"Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04",Ubuntu 24.04,,,,,,
-      ,Ubuntu 22.04.5,Ubuntu 22.04.5,Ubuntu 22.04.5,Ubuntu 22.04.5,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3, 22.04.2","Ubuntu 22.04.4, 22.04.3, 22.04.2"
+      ,Ubuntu 22.04.5,Ubuntu 22.04.5,Ubuntu 22.04.5,Ubuntu 22.04.5,Ubuntu 22.04.5,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3, 22.04.2","Ubuntu 22.04.4, 22.04.3, 22.04.2"
-      ,,,,,,,,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
+      ,,,,,,,,,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
-      ,"RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
+      ,"RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
-      ,RHEL 8.10,RHEL 8.10,RHEL 8.10,RHEL 8.10,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
+      ,RHEL 8.10,RHEL 8.10,RHEL 8.10,RHEL 8.10,RHEL 8.10,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
-      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
+      ,SLES 15 SP6,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
-      ,,,,,,,,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
+      ,,,,,,,,,,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
-      ,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,,,
+      ,"Oracle Linux 9, 8 [#mi300x-past-60]_",Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,,,
-,Debian 12 [#single-node-past-60]_,Debian 12 [#single-node-past-60]_,Debian 12 [#single-node-past-60]_,,,,,,,,,,
+,Debian 12 [#single-node-past-60]_,Debian 12 [#single-node-past-60]_,Debian 12 [#single-node-past-60]_,Debian 12 [#single-node-past-60]_,,,,,,,,,,,
-,Azure Linux 3.0 [#mi300x-past-60]_,Azure Linux 3.0 [#mi300x-past-60]_,,,,,,,,,,,
+,Azure Linux 3.0 [#mi300x-past-60]_,Azure Linux 3.0 [#mi300x-past-60]_,Azure Linux 3.0 [#mi300x-past-60]_,,,,,,,,,,,,
-      ,.. _architecture-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      ,.. _architecture-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
+      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
-      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
+      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
-      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
+      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
-      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
+      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
-      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
+      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
-      ,.. _gpu-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      ,.. _gpu-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
+      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
-      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
+      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
-      ,gfx942,gfx942,gfx942,gfx942,gfx942 [#mi300_624-past-60]_,gfx942 [#mi300_622-past-60]_,gfx942 [#mi300_621-past-60]_,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
+      ,gfx942,gfx942,gfx942,gfx942,gfx942,gfx942 [#mi300_624-past-60]_,gfx942 [#mi300_622-past-60]_,gfx942 [#mi300_621-past-60]_,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
-      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
+      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
-      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
+      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
-,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,
-      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`PyTorch <../compatibility/ml-compatibility/pytorch-compatibility>`,"2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
+      :doc:`PyTorch <../compatibility/ml-compatibility/pytorch-compatibility>`,"2.6, 2.5, 2.4, 2.3","2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <../compatibility/ml-compatibility/tensorflow-compatibility>`,"2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1","2.14.0, 2.13.1, 2.12.1"
+      :doc:`TensorFlow <../compatibility/ml-compatibility/tensorflow-compatibility>`,"2.18.1, 2.17.1, 2.16.2","2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1","2.14.0, 2.13.1, 2.12.1"
-      :doc:`JAX <../compatibility/ml-compatibility/jax-compatibility>`,0.4.31,0.4.31,0.4.31,0.4.31,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
+      :doc:`JAX <../compatibility/ml-compatibility/jax-compatibility>`,0.4.35,0.4.31,0.4.31,0.4.31,0.4.31,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
-      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
+      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.2,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
-      ,,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
-      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
+      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
-      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
+      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
-      ,,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
-      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      Thrust,2.3.2,2.3.2,2.3.2,2.3.2,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+      Thrust,2.5.0,2.3.2,2.3.2,2.3.2,2.3.2,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-      CUB,2.3.2,2.3.2,2.3.2,2.3.2,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+      CUB,2.5.0,2.3.2,2.3.2,2.3.2,2.3.2,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,
-      KMD & USER SPACE [#kfd_support-past-60]_,.. _kfd-userspace-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      KMD & USER SPACE [#kfd_support-past-60]_,.. _kfd-userspace-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      Tested user space versions,"6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.2.x, 6.1.x, 6.0.x, 5.7.x, 5.6.x","6.2.x, 6.1.x, 6.0.x, 5.7.x, 5.6.x"
+      KMD versions,"6.4.x, 6.3.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.2.x, 6.1.x, 6.0.x, 5.7.x, 5.6.x","6.2.x, 6.1.x, 6.0.x, 5.7.x, 5.6.x"
-      ,,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
-      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
+      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
-      :doc:`MIGraphX <amdmigraphx:index>`,2.11.0,2.11.0,2.11.0,2.11.0,2.10.0,2.10.0,2.10.0,2.10.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
+      :doc:`MIGraphX <amdmigraphx:index>`,2.12.0,2.11.0,2.11.0,2.11.0,2.11.0,2.10.0,2.10.0,2.10.0,2.10.0,2.9.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
-      :doc:`MIOpen <miopen:index>`,3.3.0,3.3.0,3.3.0,3.3.0,3.2.0,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`MIOpen <miopen:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.0,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`MIVisionX <mivisionx:index>`,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0,3.0.0,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
+      :doc:`MIVisionX <mivisionx:index>`,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0,3.0.0,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
-      :doc:`rocAL <rocal:index>`,2.1.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0,2.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`rocAL <rocal:index>`,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0,2.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`rocDecode <rocdecode:index>`,0.8.0,0.8.0,0.8.0,0.8.0,0.6.0,0.6.0,0.6.0,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
+      :doc:`rocDecode <rocdecode:index>`,0.10.0,0.8.0,0.8.0,0.8.0,0.8.0,0.6.0,0.6.0,0.6.0,0.6.0,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
-      :doc:`rocJPEG <rocjpeg:index>`,0.6.0,0.6.0,0.6.0,0.6.0,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A
+      :doc:`rocJPEG <rocjpeg:index>`,0.8.0,0.6.0,0.6.0,0.6.0,0.6.0,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`rocPyDecode <rocpydecode:index>`,0.2.0,0.2.0,0.2.0,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0,N/A,N/A,N/A,N/A,N/A
+      :doc:`rocPyDecode <rocpydecode:index>`,0.3.1,0.2.0,0.2.0,0.2.0,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`RPP <rpp:index>`,1.9.1,1.9.1,1.9.1,1.9.1,1.8.0,1.8.0,1.8.0,1.8.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
+      :doc:`RPP <rpp:index>`,1.9.10,1.9.1,1.9.1,1.9.1,1.9.1,1.8.0,1.8.0,1.8.0,1.8.0,1.5.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
-      ,,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
-      COMMUNICATION,.. _commlibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      COMMUNICATION,.. _commlibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`RCCL <rccl:index>`,2.21.5,2.21.5,2.21.5,2.21.5,2.20.5,2.20.5,2.20.5,2.20.5,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
+      :doc:`RCCL <rccl:index>`,2.22.3,2.21.5,2.21.5,2.21.5,2.21.5,2.20.5,2.20.5,2.20.5,2.20.5,2.18.6,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
-      ,,,,,,,,,,,,,
+`rocSHMEM <https://github.com/ROCm/rocSHMEM>`_,2.0.0,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A
-      MATH LIBS,.. _mathlibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
-      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
+      MATH LIBS,.. _mathlibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`hipBLAS <hipblas:index>`,2.3.0,2.3.0,2.3.0,2.3.0,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
+      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
-      :doc:`hipBLASLt <hipblaslt:index>`,0.10.0,0.10.0,0.10.0,0.10.0,0.8.0,0.8.0,0.8.0,0.8.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
+      :doc:`hipBLAS <hipblas:index>`,2.4.0,2.3.0,2.3.0,2.3.0,2.3.0,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
-      :doc:`hipFFT <hipfft:index>`,1.0.17,1.0.17,1.0.17,1.0.17,1.0.16,1.0.15,1.0.15,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
+      :doc:`hipBLASLt <hipblaslt:index>`,0.12.0,0.10.0,0.10.0,0.10.0,0.10.0,0.8.0,0.8.0,0.8.0,0.8.0,0.7.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
-      :doc:`hipfort <hipfort:index>`,0.5.1,0.5.1,0.5.0,0.5.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
+      :doc:`hipFFT <hipfft:index>`,1.0.18,1.0.17,1.0.17,1.0.17,1.0.17,1.0.16,1.0.15,1.0.15,1.0.14,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
-      :doc:`hipRAND <hiprand:index>`,2.11.1,2.11.1,2.11.1,2.11.0,2.11.1,2.11.0,2.11.0,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
+      :doc:`hipfort <hipfort:index>`,0.6.0,0.5.1,0.5.1,0.5.0,0.5.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
-      :doc:`hipSOLVER <hipsolver:index>`,2.3.0,2.3.0,2.3.0,2.3.0,2.2.0,2.2.0,2.2.0,2.2.0,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
+      :doc:`hipRAND <hiprand:index>`,2.12.0,2.11.1,2.11.1,2.11.1,2.11.0,2.11.1,2.11.0,2.11.0,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
-      :doc:`hipSPARSE <hipsparse:index>`,3.1.2,3.1.2,3.1.2,3.1.2,3.1.1,3.1.1,3.1.1,3.1.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      :doc:`hipSOLVER <hipsolver:index>`,2.4.0,2.3.0,2.3.0,2.3.0,2.3.0,2.2.0,2.2.0,2.2.0,2.2.0,2.1.1,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
-      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.2,0.2.2,0.2.2,0.2.2,0.2.1,0.2.1,0.2.1,0.2.1,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
+      :doc:`hipSPARSE <hipsparse:index>`,3.2.0,3.1.2,3.1.2,3.1.2,3.1.2,3.1.1,3.1.1,3.1.1,3.1.1,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      :doc:`rocALUTION <rocalution:index>`,3.2.1,3.2.1,3.2.1,3.2.1,3.2.1,3.2.0,3.2.0,3.2.0,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
+      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.3,0.2.2,0.2.2,0.2.2,0.2.2,0.2.1,0.2.1,0.2.1,0.2.1,0.2.0,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
-      :doc:`rocBLAS <rocblas:index>`,4.3.0,4.3.0,4.3.0,4.3.0,4.2.4,4.2.1,4.2.1,4.2.0,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
+      :doc:`rocALUTION <rocalution:index>`,3.2.2,3.2.1,3.2.1,3.2.1,3.2.1,3.2.1,3.2.0,3.2.0,3.2.0,3.1.1,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
-      :doc:`rocFFT <rocfft:index>`,1.0.31,1.0.31,1.0.31,1.0.31,1.0.30,1.0.29,1.0.29,1.0.28,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
+      :doc:`rocBLAS <rocblas:index>`,4.4.0,4.3.0,4.3.0,4.3.0,4.3.0,4.2.4,4.2.1,4.2.1,4.2.0,4.1.2,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
-      :doc:`rocRAND <rocrand:index>`,3.2.0,3.2.0,3.2.0,3.2.0,3.1.1,3.1.0,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
+      :doc:`rocFFT <rocfft:index>`,1.0.32,1.0.31,1.0.31,1.0.31,1.0.31,1.0.30,1.0.29,1.0.29,1.0.28,1.0.27,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
-      :doc:`rocSOLVER <rocsolver:index>`,3.27.0,3.27.0,3.27.0,3.27.0,3.26.2,3.26.0,3.26.0,3.26.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
+      :doc:`rocRAND <rocrand:index>`,3.3.0,3.2.0,3.2.0,3.2.0,3.2.0,3.1.1,3.1.0,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
-      :doc:`rocSPARSE <rocsparse:index>`,3.3.0,3.3.0,3.3.0,3.3.0,3.2.1,3.2.0,3.2.0,3.2.0,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
+      :doc:`rocSOLVER <rocsolver:index>`,3.28.0,3.27.0,3.27.0,3.27.0,3.27.0,3.26.2,3.26.0,3.26.0,3.26.0,3.25.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
-      :doc:`rocWMMA <rocwmma:index>`,1.6.0,1.6.0,1.6.0,1.6.0,1.5.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
+      :doc:`rocSPARSE <rocsparse:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.1,3.2.0,3.2.0,3.2.0,3.1.2,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
-      :doc:`Tensile <tensile:src/index>`,4.42.0,4.42.0,4.42.0,4.42.0,4.41.0,4.41.0,4.41.0,4.41.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
+      :doc:`rocWMMA <rocwmma:index>`,1.7.0,1.6.0,1.6.0,1.6.0,1.6.0,1.5.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
-      ,,,,,,,,,,,,,
+      :doc:`Tensile <tensile:src/index>`,4.43.0,4.42.0,4.42.0,4.42.0,4.42.0,4.41.0,4.41.0,4.41.0,4.41.0,4.40.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
-      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
-      :doc:`hipCUB <hipcub:index>`,3.3.0,3.3.0,3.3.0,3.3.0,3.2.1,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`hipTensor <hiptensor:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0,1.3.0,1.3.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
+      :doc:`hipCUB <hipcub:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.1,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`rocPRIM <rocprim:index>`,3.3.0,3.3.0,3.3.0,3.3.0,3.2.2,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`hipTensor <hiptensor:index>`,1.5.0,1.4.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0,1.3.0,1.3.0,1.2.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
-      :doc:`rocThrust <rocthrust:index>`,3.3.0,3.3.0,3.3.0,3.3.0,3.1.1,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      :doc:`rocPRIM <rocprim:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.2,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      ,,,,,,,,,,,,,
+      :doc:`rocThrust <rocthrust:index>`,3.3.0,3.3.0,3.3.0,3.3.0,3.3.0,3.1.1,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      SUPPORT LIBS,,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
-      `hipother <https://github.com/ROCm/hipother>`_,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      SUPPORT LIBS,,,,,,,,,,,,,,,
-      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.3.3,6.3.2,6.3.1,6.3.0,6.2.4,6.2.2,6.2.1,6.2.0,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
+      `hipother <https://github.com/ROCm/hipother>`_,6.4.43482,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,20240607.5.7,20240607.5.7,20240607.4.05,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
+      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.4.0,6.3.3,6.3.2,6.3.1,6.3.0,6.2.4,6.2.2,6.2.1,6.2.0,6.1.5,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
-      ,,,,,,,,,,,,,
+      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,20240607.5.7,20240607.5.7,20240607.4.05,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
-      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
-      :doc:`AMD SMI <amdsmi:index>`,24.7.1,24.7.1,24.7.1,24.7.1,24.6.3,24.6.3,24.6.3,24.6.2,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
+      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`ROCm Data Center Tool <rdc:index>`,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
+      :doc:`AMD SMI <amdsmi:index>`,25.3.0,24.7.1,24.7.1,24.7.1,24.7.1,24.6.3,24.6.3,24.6.3,24.6.2,24.5.1,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
-      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`ROCm Data Center Tool <rdc:index>`,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
-      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.4.0,7.4.0,7.4.0,7.4.0,7.3.0,7.3.0,7.3.0,7.3.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
+      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.0.60204,1.0.60202,1.0.60201,1.0.60200,1.0.60102,1.0.60101,1.0.60100,1.0.60002,1.0.60000
+      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.5.0,7.4.0,7.4.0,7.4.0,7.4.0,7.3.0,7.3.0,7.3.0,7.3.0,7.2.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
-      ,,,,,,,,,,,,,
+      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.0.60204,1.0.60202,1.0.60201,1.0.60200,1.0.60105,1.0.60102,1.0.60101,1.0.60100,1.0.60002,1.0.60000
-      PERFORMANCE TOOLS,,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
-      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
+      PERFORMANCE TOOLS,,,,,,,,,,,,,,,
-      :doc:`ROCm Compute Profiler <rocprofiler-compute:index>`,3.0.0,3.0.0,3.0.0,3.0.0,2.0.1,2.0.1,2.0.1,2.0.1,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
-      :doc:`ROCm Systems Profiler <rocprofiler-systems:index>`,0.1.2,0.1.1,0.1.0,0.1.0,1.11.2,1.11.2,1.11.2,1.11.2,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCm Compute Profiler <rocprofiler-compute:index>`,3.1.0,3.0.0,3.0.0,3.0.0,3.0.0,2.0.1,2.0.1,2.0.1,2.0.1,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60303,2.0.60302,2.0.60301,2.0.60300,2.0.60204,2.0.60202,2.0.60201,2.0.60200,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
+      :doc:`ROCm Systems Profiler <rocprofiler-systems:index>`,1.0.0,0.1.2,0.1.1,0.1.0,0.1.0,1.11.2,1.11.2,1.11.2,1.11.2,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.5.0,0.5.0,0.5.0,0.5.0,0.4.0,0.4.0,0.4.0,0.4.0,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60400,2.0.60303,2.0.60302,2.0.60301,2.0.60300,2.0.60204,2.0.60202,2.0.60201,2.0.60200,2.0.60105,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
-      :doc:`ROCTracer <roctracer:index>`,4.1.60303,4.1.60302,4.1.60301,4.1.60300,4.1.60204,4.1.60202,4.1.60201,4.1.60200,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
+      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.6.0,0.5.0,0.5.0,0.5.0,0.5.0,0.4.0,0.4.0,0.4.0,0.4.0,N/A,N/A,N/A,N/A,N/A,N/A
-      ,,,,,,,,,,,,,
+      :doc:`ROCTracer <roctracer:index>`,4.1.60400,4.1.60303,4.1.60302,4.1.60301,4.1.60300,4.1.60204,4.1.60202,4.1.60201,4.1.60200,4.1.60105,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
-      DEVELOPMENT TOOLS,,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
-      :doc:`HIPIFY <hipify:index>`,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24455,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      DEVELOPMENT TOOLS,,,,,,,,,,,,,,,
-      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.14.0,0.14.0,0.14.0,0.14.0,0.13.0,0.13.0,0.13.0,0.13.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
+      :doc:`HIPIFY <hipify:index>`,19.0.0.25104,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24455,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      :doc:`ROCdbgapi <rocdbgapi:index>`,0.77.0,0.77.0,0.77.0,0.77.0,0.76.0,0.76.0,0.76.0,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
+      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.14.0,0.14.0,0.14.0,0.14.0,0.14.0,0.13.0,0.13.0,0.13.0,0.13.0,0.12.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
-      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,15.2.0,15.2.0,15.2.0,15.2.0,14.2.0,14.2.0,14.2.0,14.2.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
+      :doc:`ROCdbgapi <rocdbgapi:index>`,0.77.2,0.77.0,0.77.0,0.77.0,0.77.0,0.76.0,0.76.0,0.76.0,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
-      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.3.0,0.3.0,0.3.0,N/A,N/A
+      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,15.2.0,15.2.0,15.2.0,15.2.0,15.2.0,14.2.0,14.2.0,14.2.0,14.2.0,14.1.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
-      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
+      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.3.0,0.3.0,0.3.0,0.3.0,N/A,N/A
-      ,,,,,,,,,,,,,
+      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.4,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
-      COMPILERS,.. _compilers-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
-      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
+      COMPILERS,.. _compilers-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
-      `Flang <https://github.com/ROCm/flang>`_,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24455,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`llvm-project <llvm-project:index>`,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24491,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      `Flang <https://github.com/ROCm/flang>`_,19.0.0.25133,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24455,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24491,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`llvm-project <llvm-project:index>`,19.0.0.25133,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24491,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-,,,,,,,,,,,,,
+      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,19.0.0.25133,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24491,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      RUNTIMES,.. _runtime-support-compatibility-matrix-past-60:,,,,,,,,,,,,
+,,,,,,,,,,,,,,,
-      :doc:`AMD CLR <hip:understand/amd_clr>`,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      RUNTIMES,.. _runtime-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`HIP <hip:index>`,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.4.43482,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
+      :doc:`HIP <hip:index>`,6.4.43482,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      :doc:`ROCr Runtime <rocr-runtime:index>`,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0
+      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
+      :doc:`ROCr Runtime <rocr-runtime:index>`,1.15.0,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.13.0,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0

docs/compatibility/compatibility-matrix.rst

@@ -23,15 +23,15 @@ compatibility and system requirements.
 .. container:: format-big-table
 
   .. csv-table::
-      :header: "ROCm Version", "6.3.3", "6.3.2", "6.2.0"
+      :header: "ROCm Version", "6.4.0", "6.3.3", "6.2.0"
       :stub-columns: 1
 
       :ref:`Operating systems & kernels <OS-kernel-versions>`,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04
       ,Ubuntu 22.04.5,Ubuntu 22.04.5,"Ubuntu 22.04.5, 22.04.4"
       ,"RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.4, 9.3"
       ,RHEL 8.10,RHEL 8.10,"RHEL 8.10, 8.9"
-      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5"
+      ,"SLES 15 SP6","SLES 15 SP6, SP5","SLES 15 SP6, SP5"
-      ,Oracle Linux 8.10 [#mi300x]_,Oracle Linux 8.10 [#mi300x]_,Oracle Linux 8.9 [#mi300x]_
+      ,"Oracle Linux 9, 8 [#mi300x]_",Oracle Linux 8.10 [#mi300x]_,Oracle Linux 8.9 [#mi300x]_
       ,Debian 12 [#single-node]_,Debian 12 [#single-node]_,
       ,Azure Linux 3.0 [#mi300x]_,Azure Linux 3.0 [#mi300x]_,
       ,.. _architecture-support-compatibility-matrix:,,
@@ -48,103 +48,102 @@ compatibility and system requirements.
       ,gfx908,gfx908,gfx908
       ,,,
       FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix:,,
-      :doc:`PyTorch <../compatibility/ml-compatibility/pytorch-compatibility>`,"2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 1.13","2.3, 2.2, 2.1, 2.0, 1.13"
+      :doc:`PyTorch <../compatibility/ml-compatibility/pytorch-compatibility>`,"2.6, 2.5, 2.4, 2.3","2.4, 2.3, 2.2, 1.13","2.3, 2.2, 2.1, 2.0, 1.13"
-      :doc:`TensorFlow <../compatibility/ml-compatibility/tensorflow-compatibility>`,"2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.16.1, 2.15.1, 2.14.1"
+      :doc:`TensorFlow <../compatibility/ml-compatibility/tensorflow-compatibility>`,"2.18.1, 2.17.1, 2.16.2","2.17.0, 2.16.2, 2.15.1","2.16.1, 2.15.1, 2.14.1"
-      :doc:`JAX <../compatibility/ml-compatibility/jax-compatibility>`,0.4.31,0.4.31,0.4.26
+      :doc:`JAX <../compatibility/ml-compatibility/jax-compatibility>`,0.4.35,0.4.31,0.4.26
-      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3
+      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.20,1.17.3,1.17.3
       ,,,
       THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix:,,
       `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0
       `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.15.0
       ,,,
       THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix:,,
-      Thrust,2.3.2,2.3.2,2.2.0
+      Thrust,2.5.0,2.3.2,2.2.0
-      CUB,2.3.2,2.3.2,2.2.0
+      CUB,2.5.0,2.3.2,2.2.0
       ,,,
       KMD & USER SPACE [#kfd_support]_,.. _kfd-userspace-support-compatibility-matrix:,,
-      Tested user space versions,"6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x, 6.0.x"
+      KMD versions,"6.4.x, 6.3.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x"
       ,,,
       ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix:,,
       :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0
-      :doc:`MIGraphX <amdmigraphx:index>`,2.11.0,2.11.0,2.10.0
+      :doc:`MIGraphX <amdmigraphx:index>`,2.12.0,2.11.0,2.10.0
-      :doc:`MIOpen <miopen:index>`,3.3.0,3.3.0,3.2.0
+      :doc:`MIOpen <miopen:index>`,3.4.0,3.3.0,3.2.0
-      :doc:`MIVisionX <mivisionx:index>`,3.1.0,3.1.0,3.0.0
+      :doc:`MIVisionX <mivisionx:index>`,3.2.0,3.1.0,3.0.0
-      :doc:`rocAL <rocal:index>`,2.1.0,2.1.0,1.0.0
+      :doc:`rocAL <rocal:index>`,2.2.0,2.1.0,1.0.0
-      :doc:`rocDecode <rocdecode:index>`,0.8.0,0.8.0,0.6.0
+      :doc:`rocDecode <rocdecode:index>`,0.10.0,0.8.0,0.6.0
-      :doc:`rocJPEG <rocjpeg:index>`,0.6.0,0.6.0,N/A
+      :doc:`rocJPEG <rocjpeg:index>`,0.8.0,0.6.0,N/A
-      :doc:`rocPyDecode <rocpydecode:index>`,0.2.0,0.2.0,0.1.0
+      :doc:`rocPyDecode <rocpydecode:index>`,0.3.1,0.2.0,0.1.0
-      :doc:`RPP <rpp:index>`,1.9.1,1.9.1,1.8.0
+      :doc:`RPP <rpp:index>`,1.9.10,1.9.1,1.8.0
       ,,,
       COMMUNICATION,.. _commlibs-support-compatibility-matrix:,,
-      :doc:`RCCL <rccl:index>`,2.21.5,2.21.5,2.20.5
+      :doc:`RCCL <rccl:index>`,2.22.3,2.21.5,2.20.5
+      `rocSHMEM <https://github.com/ROCm/rocSHMEM>`_ ,2.0.0,N/A,N/A
       ,,,
       MATH LIBS,.. _mathlibs-support-compatibility-matrix:,,
       `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0
-      :doc:`hipBLAS <hipblas:index>`,2.3.0,2.3.0,2.2.0
+      :doc:`hipBLAS <hipblas:index>`,2.4.0,2.3.0,2.2.0
-      :doc:`hipBLASLt <hipblaslt:index>`,0.10.0,0.10.0,0.8.0
+      :doc:`hipBLASLt <hipblaslt:index>`,0.12.0,0.10.0,0.8.0
-      :doc:`hipFFT <hipfft:index>`,1.0.17,1.0.17,1.0.14
+      :doc:`hipFFT <hipfft:index>`,1.0.18,1.0.17,1.0.14
-      :doc:`hipfort <hipfort:index>`,0.5.1,0.5.1,0.4.0
+      :doc:`hipfort <hipfort:index>`,0.6.0,0.5.1,0.4.0
-      :doc:`hipRAND <hiprand:index>`,2.11.1,2.11.1,2.11.0
+      :doc:`hipRAND <hiprand:index>`,2.12.0,2.11.1,2.11.0
-      :doc:`hipSOLVER <hipsolver:index>`,2.3.0,2.3.0,2.2.0
+      :doc:`hipSOLVER <hipsolver:index>`,2.4.0,2.3.0,2.2.0
-      :doc:`hipSPARSE <hipsparse:index>`,3.1.2,3.1.2,3.1.1
+      :doc:`hipSPARSE <hipsparse:index>`,3.2.0,3.1.2,3.1.1
-      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.2,0.2.2,0.2.1
+      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.3,0.2.2,0.2.1
-      :doc:`rocALUTION <rocalution:index>`,3.2.1,3.2.1,3.2.0
+      :doc:`rocALUTION <rocalution:index>`,3.2.2,3.2.1,3.2.0
-      :doc:`rocBLAS <rocblas:index>`,4.3.0,4.3.0,4.2.0
+      :doc:`rocBLAS <rocblas:index>`,4.4.0,4.3.0,4.2.0
-      :doc:`rocFFT <rocfft:index>`,1.0.31,1.0.31,1.0.28
+      :doc:`rocFFT <rocfft:index>`,1.0.32,1.0.31,1.0.28
-      :doc:`rocRAND <rocrand:index>`,3.2.0,3.2.0,3.1.0
+      :doc:`rocRAND <rocrand:index>`,3.3.0,3.2.0,3.1.0
-      :doc:`rocSOLVER <rocsolver:index>`,3.27.0,3.27.0,3.26.0
+      :doc:`rocSOLVER <rocsolver:index>`,3.28.0,3.27.0,3.26.0
-      :doc:`rocSPARSE <rocsparse:index>`,3.3.0,3.3.0,3.2.0
+      :doc:`rocSPARSE <rocsparse:index>`,3.4.0,3.3.0,3.2.0
-      :doc:`rocWMMA <rocwmma:index>`,1.6.0,1.6.0,1.5.0
+      :doc:`rocWMMA <rocwmma:index>`,1.7.0,1.6.0,1.5.0
-      :doc:`Tensile <tensile:src/index>`,4.42.0,4.42.0,4.41.0
+      :doc:`Tensile <tensile:src/index>`,4.43.0,4.42.0,4.41.0
       ,,,
       PRIMITIVES,.. _primitivelibs-support-compatibility-matrix:,,
-      :doc:`hipCUB <hipcub:index>`,3.3.0,3.3.0,3.2.0
+      :doc:`hipCUB <hipcub:index>`,3.4.0,3.3.0,3.2.0
-      :doc:`hipTensor <hiptensor:index>`,1.4.0,1.4.0,1.3.0
+      :doc:`hipTensor <hiptensor:index>`,1.5.0,1.4.0,1.3.0
-      :doc:`rocPRIM <rocprim:index>`,3.3.0,3.3.0,3.2.0
+      :doc:`rocPRIM <rocprim:index>`,3.4.0,3.3.0,3.2.0
       :doc:`rocThrust <rocthrust:index>`,3.3.0,3.3.0,3.0.1
       ,,,
       SUPPORT LIBS,,,
-      `hipother <https://github.com/ROCm/hipother>`_,6.3.42134,6.3.42134,6.2.41133
+      `hipother <https://github.com/ROCm/hipother>`_,6.4.43482,6.3.42134,6.2.41133
-      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.3.3,6.3.2,6.2.0
+      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.4.0,6.3.3,6.2.0
       `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,N/A [#ROCT-rocr]_,N/A [#ROCT-rocr]_,20240607.1.4246
       ,,,
       SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix:,,
-      :doc:`AMD SMI <amdsmi:index>`,24.7.1,24.7.1,24.6.2
+      :doc:`AMD SMI <amdsmi:index>`,25.3.0,24.7.1,24.6.2
       :doc:`ROCm Data Center Tool <rdc:index>`,0.3.0,0.3.0,0.3.0
       :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0
-      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.4.0,7.4.0,7.3.0
+      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.5.0,7.4.0,7.3.0
       :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,1.1.0,1.1.0,1.0.60200
       ,,,
       PERFORMANCE TOOLS,,,
       :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0
-      :doc:`ROCm Compute Profiler <rocprofiler-compute:index>`,3.0.0,3.0.0,2.0.1
+      :doc:`ROCm Compute Profiler <rocprofiler-compute:index>`,3.1.0,3.0.0,2.0.1
-      :doc:`ROCm Systems Profiler <rocprofiler-systems:index>`,0.1.2,0.1.1,1.11.2
+      :doc:`ROCm Systems Profiler <rocprofiler-systems:index>`,1.0.0,0.1.2,1.11.2
-      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60303,2.0.60302,2.0.60200
+      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60400,2.0.60303,2.0.60200
-      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.5.0,0.5.0,0.4.0
+      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.6.0,0.5.0,0.4.0
-      :doc:`ROCTracer <roctracer:index>`,4.1.60303,4.1.60302,4.1.60200
+      :doc:`ROCTracer <roctracer:index>`,4.1.60400,4.1.60303,4.1.60200
       ,,,
       DEVELOPMENT TOOLS,,,
-      :doc:`HIPIFY <hipify:index>`,18.0.0.25012,18.0.0.25012,18.0.0.24232
+      :doc:`HIPIFY <hipify:index>`,19.0.0.25133,18.0.0.25012,18.0.0.24232
       :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.14.0,0.14.0,0.13.0
-      :doc:`ROCdbgapi <rocdbgapi:index>`,0.77.0,0.77.0,0.76.0
+      :doc:`ROCdbgapi <rocdbgapi:index>`,0.77.2,0.77.0,0.76.0
       :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,15.2.0,15.2.0,14.2.0
       `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.4.0
-      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3
+      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.4,2.0.3,2.0.3
       ,,,
       COMPILERS,.. _compilers-support-compatibility-matrix:,,
       `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,N/A
       :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.1.1
-      `Flang <https://github.com/ROCm/flang>`_,18.0.0.25012,18.0.0.25012,18.0.0.24232
+      `Flang <https://github.com/ROCm/flang>`_,19.0.0.25133,18.0.0.25012,18.0.0.24232
-      :doc:`llvm-project <llvm-project:index>`,18.0.0.25012,18.0.0.25012,18.0.0.24232
+      :doc:`llvm-project <llvm-project:index>`,19.0.0.25133,18.0.0.25012,18.0.0.24232
-      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.25012,18.0.0.25012,18.0.0.24232
+      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,19.0.0.25133,18.0.0.25012,18.0.0.24232
       ,,,
       RUNTIMES,.. _runtime-support-compatibility-matrix:,,
-      :doc:`AMD CLR <hip:understand/amd_clr>`,6.3.42134,6.3.42134,6.2.41133
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.4.43482,6.3.42134,6.2.41133
-      :doc:`HIP <hip:index>`,6.3.42134,6.3.42134,6.2.41133
+      :doc:`HIP <hip:index>`,6.4.43482,6.3.42134,6.2.41133
       `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0
-      :doc:`ROCr Runtime <rocr-runtime:index>`,1.14.0,1.14.0,1.13.0
+      :doc:`ROCr Runtime <rocr-runtime:index>`,1.15.0,1.14.0,1.13.0
-
-
 
 
 .. rubric:: Footnotes
@@ -152,45 +151,47 @@ compatibility and system requirements.
 .. [#mi300x] Oracle Linux and Azure Linux are supported only on AMD Instinct MI300X.
 .. [#single-node] Debian 12 is supported only on AMD Instinct MI300X for single-node functionality. 
 .. [#mi300_620] **For ROCm 6.2.0** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
-.. [#kfd_support] ROCm provides forward and backward compatibility between the AMD Kernel-mode GPU Driver (KMD) and its user space software for +/- 2 releases. These are the compatibility combinations that are currently supported.
+.. [#kfd_support] Starting from ROCm 6.4.0, forward and backward compatibility between the AMD Kernel-mode GPU Driver (KMD) and its user space software is provided up to a year apart (assuming hardware support is available in both). For earlier ROCm releases, the compatibility is provided for +/- 2 releases. These are the compatibility combinations that are currently supported.
 .. [#ROCT-rocr] Starting from ROCm 6.3.0, the ROCT Thunk Interface is included as part of the ROCr runtime package.
 
 .. _OS-kernel-versions:
 
-Operating systems and kernel versions
+Operating systems, kernel and Glibc versions
-*************************************
+*********************************************
 
 Use this lookup table to confirm which operating system and kernel versions are supported with ROCm.
 
 .. csv-table::
-   :header: "OS", "Version", "Kernel"
+   :header: "OS", "Version", "Kernel", "Glibc"
-   :widths: 40, 20, 40
+   :widths: 40, 20, 30, 20
    :stub-columns: 1
 
-   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 24.04.2, "6.8 GA, 6.11 HWE"
+   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 24.04.2, "6.8 GA, 6.11 HWE", 2.39
-   , 24.04, "6.8 GA"
    ,,
-   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 22.04.5, "5.15 GA, 6.8 HWE"
+   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 22.04.5, "5.15 GA, 6.8 HWE", 2.35
-   , 22.04.4, "5.15 GA, 6.5 HWE"
    ,,
-   `Red Hat Enterprise Linux (RHEL 9) <https://access.redhat.com/articles/3078#RHEL9>`_, 9.5, 5.14.0
+   `Red Hat Enterprise Linux (RHEL 9) <https://access.redhat.com/articles/3078#RHEL9>`_, 9.5, 5.14+, 2.34
-   ,9.4, 5.14.0
+   ,9.4, 5.14+, 2.34
-   ,9.3, 5.14.0
+   ,9.3, 5.14+, 2.34
    ,,
-   `Red Hat Enterprise Linux (RHEL 8) <https://access.redhat.com/articles/3078#RHEL8>`_, 8.10, 4.18.0
+   `Red Hat Enterprise Linux (RHEL 8) <https://access.redhat.com/articles/3078#RHEL8>`_, 8.10, 4.18.0+, 2.28
-   ,8.9, 4.18.0
+   ,8.9, 4.18.0, 2.28
    ,,
-   `SUSE Linux Enterprise Server (SLES) <https://www.suse.com/support/kb/doc/?id=000019587#SLE15SP4>`_, 15 SP6, 6.4.0
+   `SUSE Linux Enterprise Server (SLES) <https://www.suse.com/support/kb/doc/?id=000019587#SLE15SP4>`_, 15 SP6, "6.5.0+, 6.4.0", 2.38
-   ,15 SP5, 5.14.21
+   ,15 SP5, 5.14.21, 2.31
    ,,
-   `Oracle Linux <https://blogs.oracle.com/scoter/post/oracle-linux-and-unbreakable-enterprise-kernel-uek-releases>`_, 8.10, 5.15.0
+   `Oracle Linux <https://blogs.oracle.com/scoter/post/oracle-linux-and-unbreakable-enterprise-kernel-uek-releases>`_, 9, 5.15.0 (UEK), 2.35
-   ,8.9, 5.15.0
+   ,8, 5.15.0 (UEK), 2.28
    ,,
-   `Debian <https://www.debian.org/download>`_,12, 6.1
+   `Debian <https://www.debian.org/download>`_,12, 6.1, 2.36
    ,,
-   `Azure Linux <https://techcommunity.microsoft.com/blog/linuxandopensourceblog/azure-linux-3-0-now-in-preview-on-azure-kubernetes-service-v1-31/4287229>`_,3.0, 6.6
+   `Azure Linux <https://techcommunity.microsoft.com/blog/linuxandopensourceblog/azure-linux-3-0-now-in-preview-on-azure-kubernetes-service-v1-31/4287229>`_,3.0, 6.6.60, 2.38
    ,,
 
+.. note::
+
+  * See `Red Hat Enterprise Linux Release Dates <https://access.redhat.com/articles/3078>`_ to learn about the specific kernel versions supported on Red Hat Enterprise Linux (RHEL).
+  * See `List of SUSE Linux Enterprise Server kernel <https://www.suse.com/support/kb/doc/?id=000019587>`_ to learn about the specific kernel version supported on SUSE Linux Enterprise Server (SLES).
 ..
    Footnotes and ref anchors in below historical tables should be appended with "-past-60", to differentiate from the
    footnote references in the above, latest, compatibility matrix.  It also allows to easily find & replace.
@@ -228,5 +229,5 @@ Expand for full historical view of:
    .. [#mi300_610-past-60] **For ROCm 6.1.0** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4.
    .. [#mi300_602-past-60] **For ROCm 6.0.2** - MI300A (gfx942) is supported on Ubuntu 22.04.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.3.
    .. [#mi300_600-past-60] **For ROCm 6.0.0** - MI300A (gfx942) is supported on Ubuntu 22.04.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.3.
-   .. [#kfd_support-past-60] ROCm provides forward and backward compatibility between the AMD Kernel-mode GPU Driver (KMD) and its user space software for +/- 2 releases. These are the compatibility combinations that are currently supported.
+   .. [#kfd_support-past-60] Starting from ROCm 6.4.0, forward and backward compatibility between the AMD Kernel-mode GPU Driver (KMD) and its user space software is provided up to a year apart (assuming hardware support is available in both). For earlier ROCm releases, the compatibility is provided for +/- 2 releases. These are the compatibility combinations that are currently supported.
    .. [#ROCT-rocr-past-60] Starting from ROCm 6.3.0, the ROCT Thunk Interface is included as part of the ROCr runtime package.

docs/compatibility/ml-compatibility/jax-compatibility.rst

@@ -4,6 +4,8 @@
    :description: JAX compatibility
    :keywords: GPU, JAX compatibility
 
+.. version-set:: rocm_version latest
+
 *******************************************************************************
 JAX compatibility
 *******************************************************************************
@@ -12,17 +14,18 @@ JAX provides a NumPy-like API, which combines automatic differentiation and the
 Accelerated Linear Algebra (XLA) compiler to achieve high-performance machine
 learning at scale.
 
-JAX uses composable transformations of Python and NumPy through just-in-time (JIT) compilation,
+JAX uses composable transformations of Python and NumPy through just-in-time
-automatic vectorization, and parallelization. To learn about JAX, including profiling and
+(JIT) compilation, automatic vectorization, and parallelization. To learn about
-optimizations, see the official `JAX documentation
+JAX, including profiling and optimizations, see the official `JAX documentation
 <https://jax.readthedocs.io/en/latest/notebooks/quickstart.html>`_.
 
-ROCm support for JAX is upstreamed and users can build the official source code with ROCm
+ROCm support for JAX is upstreamed, and users can build the official source code
-support:
+with ROCm support:
 
 - ROCm JAX release:
 
-  - Offers AMD-validated and community :ref:`Docker images <jax-docker-compat>` with ROCm and JAX pre-installed.
+  - Offers AMD-validated and community :ref:`Docker images <jax-docker-compat>`
+    with ROCm and JAX preinstalled.
 
   - ROCm JAX repository: `ROCm/jax <https://github.com/ROCm/jax>`_
 
@@ -34,8 +37,8 @@ support:
   - Official JAX repository: `jax-ml/jax <https://github.com/jax-ml/jax>`_
 
   - See the `AMD GPU (Linux) installation section
-    <https://jax.readthedocs.io/en/latest/installation.html#amd-gpu-linux>`_ in the JAX
+    <https://jax.readthedocs.io/en/latest/installation.html#amd-gpu-linux>`_ in
-    documentation.
+    the JAX documentation.
 
 .. note::
 
@@ -44,6 +47,44 @@ support:
    `Community ROCm JAX Docker images <https://hub.docker.com/r/rocm/jax-community>`_
    follow upstream JAX releases and use the latest available ROCm version.
 
+Use cases and recommendations
+================================================================================
+
+* The `nanoGPT in JAX <https://rocm.blogs.amd.com/artificial-intelligence/nanoGPT-JAX/README.html>`_
+  blog explores the implementation and training of a Generative Pre-trained
+  Transformer (GPT) model in JAX, inspired by Andrej Karpathy’s JAX-based
+  nanoGPT. Comparing how essential GPT components—such as self-attention 
+  mechanisms and optimizers—are realized in JAX and JAX, also highlights
+  JAX’s unique features.
+
+* The `Optimize GPT Training: Enabling Mixed Precision Training in JAX using
+  ROCm on AMD GPUs <https://rocm.blogs.amd.com/artificial-intelligence/jax-mixed-precision/README.html>`_
+  blog post provides a comprehensive guide on enhancing the training efficiency
+  of GPT models by implementing mixed precision techniques in JAX, specifically
+  tailored for AMD GPUs utilizing the ROCm platform.
+
+* The `Supercharging JAX with Triton Kernels on AMD GPUs <https://rocm.blogs.amd.com/artificial-intelligence/jax-triton/README.html>`_
+  blog demonstrates how to develop a custom fused dropout-activation kernel for
+  matrices using Triton, integrate it with JAX, and benchmark its performance
+  using ROCm.
+
+* The `Distributed fine-tuning with JAX on AMD GPUs <https://rocm.blogs.amd.com/artificial-intelligence/distributed-sft-jax/README.html>`_
+  outlines the process of fine-tuning a Bidirectional Encoder Representations
+  from Transformers (BERT)-based large language model (LLM) using JAX for a text
+  classification task. The blog post discuss techniques for parallelizing the
+  fine-tuning across multiple AMD GPUs and assess the model's performance on a
+  holdout dataset. During the fine-tuning, a BERT-base-cased transformer model
+  and the General Language Understanding Evaluation (GLUE) benchmark dataset was
+  used on a multi-GPU setup.
+
+* The `MI300X workload optimization guide <https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/workload.html>`_
+  provides detailed guidance on optimizing workloads for the AMD Instinct MI300X
+  accelerator using ROCm. The page is aimed at helping users achieve optimal
+  performance for deep learning and other high-performance computing tasks on
+  the MI300X GPU.
+
+For more use cases and recommendations, see `ROCm JAX blog posts <https://rocm.blogs.amd.com/blog/tag/jax.html>`_.
+
 .. _jax-docker-compat:
 
 Docker image compatibility
@@ -55,8 +96,8 @@ Docker image compatibility
 
 AMD validates and publishes ready-made `ROCm JAX Docker images <https://hub.docker.com/r/rocm/jax>`_
 with ROCm backends on Docker Hub. The following Docker image tags and
-associated inventories are validated for
+associated inventories represent the latest JAX version from the official Docker Hub and are validated for
-`ROCm 6.3.1 <https://repo.radeon.com/rocm/apt/6.3.1/>`_. Click the |docker-icon|
+`ROCm 6.4.0 <https://repo.radeon.com/rocm/apt/6.4/>`_. Click the |docker-icon|
 icon to view the image on Docker Hub.
 
 .. list-table:: JAX Docker image components
@@ -66,24 +107,26 @@ icon to view the image on Docker Hub.
       - JAX
       - Linux
       - Python
+
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/jax/rocm6.3.1-jax0.4.31-py3.12/images/sha256-085a0cd5207110922f1fca684933a9359c66d42db6c5aba4760ed5214fdabde0"><i class="fab fa-docker fa-lg"></i> rocm/jax</a>
+           <a href="https://hub.docker.com/layers/rocm/jax/rocm6.4-jax0.4.35-py3.12/images/sha256-4069398229078f3311128b6d276c6af377c7e97d3363d020b0bf7154fae619ca"><i class="fab fa-docker fa-lg"></i> rocm/jax</a>
 
-      - `0.4.31 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.31>`_
+      - `0.4.35 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.35>`_
       - Ubuntu 24.04
       - `3.12.7 <https://www.python.org/downloads/release/python-3127/>`_
+
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/jax/rocm6.3.1-jax0.4.31-py3.10/images/sha256-f88eddad8f47856d8640b694da4da347ffc1750d7363175ab7dc872e82b43324"><i class="fab fa-docker fa-lg"></i> rocm/jax</a>
+           <a href="https://hub.docker.com/layers/rocm/jax/rocm6.4-jax0.4.35-py3.10/images/sha256-a137f901f91ce6c13b424c40a6cf535248d4d20fd36d5daf5eee0570190a4a11"><i class="fab fa-docker fa-lg"></i> rocm/jax</a>
 
-      - `0.4.31 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.31>`_
+      - `0.4.35 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.35>`_
       - Ubuntu 22.04
       - `3.10.14 <https://www.python.org/downloads/release/python-31014/>`_
 
 AMD publishes `Community ROCm JAX Docker images <https://hub.docker.com/r/rocm/jax-community>`_
 with ROCm backends on Docker Hub. The following Docker image tags and
-associated inventories are tested for `ROCm 6.2.4 <https://repo.radeon.com/rocm/apt/6.2.4/>`_.
+associated inventories are tested for `ROCm 6.3.2 <https://repo.radeon.com/rocm/apt/6.3.2/>`_.
 
 .. list-table:: JAX community Docker image components
     :header-rows: 1
@@ -92,34 +135,37 @@ associated inventories are tested for `ROCm 6.2.4 <https://repo.radeon.com/rocm/
       - JAX
       - Linux
       - Python
+
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/jax-community/rocm6.2.4-jax0.4.35-py3.12.7/images/sha256-a6032d89c07573b84c44e42c637bf9752b1b7cd2a222d39344e603d8f4c63beb?context=explore"><i class="fab fa-docker fa-lg"></i> rocm/jax-community</a>
+           <a href="https://hub.docker.com/layers/rocm/jax-community/rocm6.3.2-jax0.5.0-py3.12.8/images/sha256-25dfaa0183e274bd0a3554a309af3249c6f16a1793226cb5373f418e39d3146a"><i class="fab fa-docker fa-lg"></i> rocm/jax-community</a>
 
-      - `0.4.35 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.35>`_
+      - `0.5.0 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.5.0>`_
       - Ubuntu 22.04
-      - `3.12.7 <https://www.python.org/downloads/release/python-3127/>`_
+      - `3.12.8 <https://www.python.org/downloads/release/python-3128/>`_
+
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/jax-community/rocm6.2.4-jax0.4.35-py3.11.10/images/sha256-d462f7e445545fba2f3b92234a21beaa52fe6c5f550faabcfdcd1bf53486d991?context=explore"><i class="fab fa-docker fa-lg"></i> rocm/jax-community</a>
+           <a href="https://hub.docker.com/layers/rocm/jax-community/rocm6.3.2-jax0.5.0-py3.11.11/images/sha256-ff9baeca9067d13e6c279c911e5a9e5beed0817d24fafd424367cc3d5bd381d7"><i class="fab fa-docker fa-lg"></i> rocm/jax-community</a>
 
-      - `0.4.35 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.35>`_
+      - `0.5.0 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.5.0>`_
       - Ubuntu 22.04
-      - `3.11.10 <https://www.python.org/downloads/release/python-31110/>`_
+      - `3.11.11 <https://www.python.org/downloads/release/python-31111/>`_
+
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/jax-community/rocm6.2.4-jax0.4.35-py3.10.15/images/sha256-6f2d4d0f529378d9572f0e8cfdcbc101d1e1d335bd626bb3336fff87814e9d60?context=explore"><i class="fab fa-docker fa-lg"></i> rocm/jax-community</a>
+           <a href="https://hub.docker.com/layers/rocm/jax-community/rocm6.3.2-jax0.5.0-py3.10.16/images/sha256-8bab484be1713655f74da51a191ed824bb9d03db1104fd63530a1ac3c37cf7b1"><i class="fab fa-docker fa-lg"></i> rocm/jax-community</a>
 
-      - `0.4.35 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.35>`_
+      - `0.5.0 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.5.0>`_
       - Ubuntu 22.04
-      - `3.10.15 <https://www.python.org/downloads/release/python-31015/>`_
+      - `3.10.16 <https://www.python.org/downloads/release/python-31016/>`_
 
-Critical ROCm libraries for JAX
+Key ROCm libraries for JAX
 ================================================================================
 
-The functionality of JAX with ROCm is determined by its underlying library
+JAX functionality on ROCm is determined by its underlying library
-dependencies. These critical ROCm components affect the capabilities,
+dependencies. These ROCm components affect the capabilities, performance, and
-performance, and feature set available to developers.
+feature set available to developers.
 
 .. list-table::
     :header-rows: 1
@@ -129,7 +175,7 @@ performance, and feature set available to developers.
       - Purpose
       - Used in
     * - `hipBLAS <https://github.com/ROCm/hipBLAS>`_
-      - 2.3.0
+      - :version-ref:`hipBLAS rocm_version`
       - Provides GPU-accelerated Basic Linear Algebra Subprograms (BLAS) for
         matrix and vector operations.
       - Matrix multiplication in ``jax.numpy.matmul``, ``jax.lax.dot`` and
@@ -138,7 +184,7 @@ performance, and feature set available to developers.
         ``jax.numpy.einsum`` with matrix-multiplication patterns algebra
         operations.
     * - `hipBLASLt <https://github.com/ROCm/hipBLASLt>`_
-      - 0.10.0
+      - :version-ref:`hipBLASLt rocm_version`
       - hipBLASLt is an extension of hipBLAS, providing additional
         features like epilogues fused into the matrix multiplication kernel or
         use of integer tensor cores.
@@ -147,7 +193,7 @@ performance, and feature set available to developers.
         operations, mixed-precision support, and hardware-specific
         optimizations.
     * - `hipCUB <https://github.com/ROCm/hipCUB>`_
-      - 3.3.0
+      - :version-ref:`hipCUB rocm_version`
       - Provides a C++ template library for parallel algorithms for reduction,
         scan, sort and select.
       - Reduction functions (``jax.numpy.sum``, ``jax.numpy.mean``,
@@ -155,23 +201,23 @@ performance, and feature set available to developers.
         (``jax.numpy.cumsum``, ``jax.numpy.cumprod``) and sorting
         (``jax.numpy.sort``, ``jax.numpy.argsort``).
     * - `hipFFT <https://github.com/ROCm/hipFFT>`_
-      - 1.0.17
+      - :version-ref:`hipFFT rocm_version`
       - Provides GPU-accelerated Fast Fourier Transform (FFT) operations.
       - Used in functions like ``jax.numpy.fft``.
     * - `hipRAND <https://github.com/ROCm/hipRAND>`_
-      - 2.11.0
+      - :version-ref:`hipRAND rocm_version`
       - Provides fast random number generation for GPUs.
       - The ``jax.random.uniform``, ``jax.random.normal``,
         ``jax.random.randint`` and ``jax.random.split``.
     * - `hipSOLVER <https://github.com/ROCm/hipSOLVER>`_
-      - 2.3.0
+      - :version-ref:`hipSOLVER rocm_version`
       - Provides GPU-accelerated solvers for linear systems, eigenvalues, and
         singular value decompositions (SVD).
       - Solving linear systems (``jax.numpy.linalg.solve``), matrix
         factorizations, SVD (``jax.numpy.linalg.svd``) and eigenvalue problems
         (``jax.numpy.linalg.eig``).
     * - `hipSPARSE <https://github.com/ROCm/hipSPARSE>`_
-      - 3.1.2
+      - :version-ref:`hipSPARSE rocm_version`
       - Accelerates operations on sparse matrices, such as sparse matrix-vector
         or matrix-matrix products.
       - Sparse matrix multiplication (``jax.numpy.matmul``), sparse
@@ -179,38 +225,38 @@ performance, and feature set available to developers.
         (``jax.experimental.sparse.dot``), sparse linear system solvers and
         sparse data handling.
     * - `hipSPARSELt <https://github.com/ROCm/hipSPARSELt>`_
-      - 0.2.2
+      - :version-ref:`hipSPARSELt rocm_version`
       - Accelerates operations on sparse matrices, such as sparse matrix-vector
         or matrix-matrix products.
       - Sparse matrix multiplication (``jax.numpy.matmul``), sparse
         matrix-vector and matrix-matrix products
         (``jax.experimental.sparse.dot``) and sparse linear system solvers.
     * - `MIOpen <https://github.com/ROCm/MIOpen>`_
-      - 3.3.0
+      - :version-ref:`MIOpen rocm_version`
       - Optimized for deep learning primitives such as convolutions, pooling,
         normalization, and activation functions.
       - Speeds up convolutional neural networks (CNNs), recurrent neural
         networks (RNNs), and other layers. Used in operations like
         ``jax.nn.conv``, ``jax.nn.relu``, and ``jax.nn.batch_norm``.
     * - `RCCL <https://github.com/ROCm/rccl>`_
-      - 2.21.5
+      - :version-ref:`RCCL rocm_version`
       - Optimized for multi-GPU communication for operations like  all-reduce,
         broadcast, and scatter.
       - Distribute computations across multiple GPU with ``pmap`` and
         ``jax.distributed``. XLA automatically uses rccl when executing
         operations across multiple GPUs on AMD hardware.
     * - `rocThrust <https://github.com/ROCm/rocThrust>`_
-      - 3.3.0
+      - :version-ref:`rocThrust rocm_version`
       - Provides a C++ template library for parallel algorithms like sorting,
         reduction, and scanning.
       - Reduction operations like ``jax.numpy.sum``, ``jax.pmap`` for
         distributed training, which involves parallel reductions or
         operations like ``jax.numpy.cumsum`` can use rocThrust.
 
-Supported and unsupported features
+Supported features
 ===============================================================================
 
-The following table maps GPU-accelerated JAX modules to their supported
+The following table maps the public JAX API modules to their supported
 ROCm and JAX versions.
 
 .. list-table::
@@ -218,8 +264,8 @@ ROCm and JAX versions.
 
     * - Module
       - Description
-      - Since JAX
+      - As of JAX
-      - Since ROCm
+      - As of ROCm
     * - ``jax.numpy``
       - Implements the NumPy API, using the primitives in ``jax.lax``.
       - 0.1.56
@@ -247,21 +293,11 @@ ROCm and JAX versions.
         devices.
       - 0.3.20
       - 5.1.0
-    * - ``jax.dlpack``
-      - For exchanging tensor data between JAX and other libraries that support the
-        DLPack standard.
-      - 0.1.57
-      - 5.0.0
     * - ``jax.distributed``
       - Enables the scaling of computations across multiple devices on a single
         machine or across multiple machines.
       - 0.1.74
       - 5.0.0
-    * - ``jax.dtypes``
-      - Provides utilities for working with and managing data types in JAX
-        arrays and computations.
-      - 0.1.66
-      - 5.0.0
     * - ``jax.image``
       - Contains image manipulation functions like resize, scale and translation.
       - 0.1.57
@@ -275,27 +311,10 @@ ROCm and JAX versions.
         array.
       - 0.1.57
       - 5.0.0
-    * - ``jax.profiler``
-      - Contains JAX’s tracing and time profiling features.
-      - 0.1.57
-      - 5.0.0
     * - ``jax.stages``
       - Contains interfaces to stages of the compiled execution process.
       - 0.3.4
       - 5.0.0
-    * - ``jax.tree``
-      - Provides utilities for working with tree-like container data structures.
-      - 0.4.26
-      - 5.6.0
-    * - ``jax.tree_util``
-      - Provides utilities for working with nested data structures, or
-        ``pytrees``.
-      - 0.1.65
-      - 5.0.0
-    * - ``jax.typing``
-      - Provides JAX-specific static type annotations.
-      - 0.3.18
-      - 5.1.0
     * - ``jax.extend``
       - Provides modules for access to JAX internal machinery module. The
         ``jax.extend`` module defines a library view of some of JAX’s internal
@@ -331,8 +350,8 @@ A SciPy-like API for scientific computing.
     :header-rows: 1
 
     * - Module
-      - Since JAX
+      - As of JAX
-      - Since ROCm
+      - As of ROCm
     * - ``jax.scipy.cluster``
       - 0.3.11
       - 5.1.0
@@ -377,8 +396,8 @@ jax.scipy.stats module
    :header-rows: 1
 
    * - Module
-     - Since JAX
+     - As of JAX
-     - Since ROCm
+     - As of ROCm
    * - ``jax.scipy.stats.bernouli``
      - 0.1.56
      - 5.0.0
@@ -461,8 +480,8 @@ Modules for JAX extensions.
     :header-rows: 1
 
     * - Module
-      - Since JAX
+      - As of JAX
-      - Since ROCm
+      - As of ROCm
     * - ``jax.extend.ffi``
       - 0.4.30
       - 6.0.0
@@ -476,190 +495,25 @@ Modules for JAX extensions.
       - 0.4.15
       - 5.5.0
 
-jax.experimental module
--------------------------------------------------------------------------------
-
-Experimental modules and APIs.
-
-.. list-table::
-    :header-rows: 1
-
-    * - Module
-      - Since JAX
-      - Since ROCm
-    * - ``jax.experimental.checkify``
-      - 0.1.75
-      - 5.0.0
-    * - ``jax.experimental.compilation_cache.compilation_cache``
-      - 0.1.68
-      - 5.0.0
-    * - ``jax.experimental.custom_partitioning``
-      - 0.4.0
-      - 5.3.0
-    * - ``jax.experimental.jet``
-      - 0.1.56
-      - 5.0.0
-    * - ``jax.experimental.key_reuse``
-      - 0.4.26
-      - 5.6.0
-    * - ``jax.experimental.mesh_utils``
-      - 0.1.76
-      - 5.0.0
-    * - ``jax.experimental.multihost_utils``
-      - 0.3.2
-      - 5.0.0
-    * - ``jax.experimental.pallas``
-      - 0.4.15
-      - 5.5.0
-    * - ``jax.experimental.pjit``
-      - 0.1.61
-      - 5.0.0
-    * - ``jax.experimental.serialize_executable``
-      - 0.4.0
-      - 5.3.0
-    * - ``jax.experimental.shard_map``
-      - 0.4.3
-      - 5.3.0
-    * - ``jax.experimental.sparse``
-      - 0.1.75
-      - 5.0.0
-
-.. list-table::
-    :header-rows: 1
-
-    * - API
-      - Since JAX
-      - Since ROCm
-    * - ``jax.experimental.enable_x64``
-      - 0.1.60
-      - 5.0.0
-    * - ``jax.experimental.disable_x64``
-      - 0.1.60
-      - 5.0.0
-
-jax.experimental.pallas module
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Module for Pallas, a JAX extension for custom kernels.
-
-.. list-table::
-    :header-rows: 1
-
-    * - Module
-      - Since JAX
-      - Since ROCm
-    * - ``jax.experimental.pallas.mosaic_gpu``
-      - 0.4.31
-      - 6.1.3
-    * - ``jax.experimental.pallas.tpu``
-      - 0.4.15
-      - 5.5.0
-    * - ``jax.experimental.pallas.triton``
-      - 0.4.32
-      - 6.1.3
-
-jax.experimental.sparse module
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Experimental support for sparse matrix operations.
-
-.. list-table::
-    :header-rows: 1
-
-    * - Module
-      - Since JAX
-      - Since ROCm
-    * - ``jax.experimental.sparse.linalg``
-      - 0.3.15
-      - 5.2.0
-    * - ``jax.experimental.sparse.sparsify``
-      - 0.3.25
-      - ❌
-
-.. list-table::
-    :header-rows: 1
-
-    * - ``sparse`` data structure API
-      - Since JAX
-      - Since ROCm
-    * - ``jax.experimental.sparse.BCOO``
-      - 0.1.72
-      - 5.0.0
-    * - ``jax.experimental.sparse.BCSR``
-      - 0.3.20
-      - 5.1.0
-    * - ``jax.experimental.sparse.CSR``
-      - 0.1.75
-      - 5.0.0
-    * - ``jax.experimental.sparse.NM``
-      - 0.4.27
-      - 5.6.0
-    * - ``jax.experimental.sparse.COO``
-      - 0.1.75
-      - 5.0.0
-
 Unsupported JAX features
-------------------------
+===============================================================================
 
-The following are GPU-accelerated JAX features not currently supported by
+The following GPU-accelerated JAX features are not supported by ROCm for
-ROCm.
+the listed supported JAX versions.
 
 .. list-table::
     :header-rows: 1
 
     * - Feature
       - Description
-      - Since JAX
+
     * - Mixed Precision with TF32
       - Mixed precision with TF32 is used for matrix multiplications,
         convolutions, and other linear algebra operations, particularly in
         deep learning workloads like CNNs and transformers.
-      - 0.2.25
+
-    * - RNN support
-      - Currently only LSTM with double bias is supported with float32 input
-        and weight.
-      - 0.3.25
     * - XLA int4 support
       - 4-bit integer (int4) precision in the XLA compiler.
-      - 0.4.0
-    * - ``jax.experimental.sparsify``
-      - Converts a dense matrix to a sparse matrix representation.
-      - Experimental
 
-Use cases and recommendations
+    * - MOSAIC (GPU)
-================================================================================
+      - Mosaic is a library of kernel-building abstractions for JAX's Pallas system
-
-* The `nanoGPT in JAX <https://rocm.blogs.amd.com/artificial-intelligence/nanoGPT-JAX/README.html>`_
-  blog explores the implementation and training of a Generative Pre-trained
-  Transformer (GPT) model in JAX, inspired by Andrej Karpathy’s PyTorch-based
-  nanoGPT. By comparing how essential GPT components—such as self-attention
-  mechanisms and optimizers—are realized in PyTorch and JAX, also highlight
-  JAX’s unique features.
-
-* The `Optimize GPT Training: Enabling Mixed Precision Training in JAX using
-  ROCm on AMD GPUs <https://rocm.blogs.amd.com/artificial-intelligence/jax-mixed-precision/README.html>`_
-  blog post provides a comprehensive guide on enhancing the training efficiency
-  of GPT models by implementing mixed precision techniques in JAX, specifically
-  tailored for AMD GPUs utilizing the ROCm platform.
-
-* The `Supercharging JAX with Triton Kernels on AMD GPUs <https://rocm.blogs.amd.com/artificial-intelligence/jax-triton/README.html>`_
-  blog demonstrates how to develop a custom fused dropout-activation kernel for
-  matrices using Triton, integrate it with JAX, and benchmark its performance
-  using ROCm.
-
-* The `Distributed fine-tuning with JAX on AMD GPUs <https://rocm.blogs.amd.com/artificial-intelligence/distributed-sft-jax/README.html>`_
-  outlines the process of fine-tuning a Bidirectional Encoder Representations
-  from Transformers (BERT)-based large language model (LLM) using JAX for a text
-  classification task. The blog post discuss techniques for parallelizing the
-  fine-tuning across multiple AMD GPUs and assess the model's performance on a
-  holdout dataset. During the fine-tuning, a BERT-base-cased transformer model
-  and the General Language Understanding Evaluation (GLUE) benchmark dataset was
-  used on a multi-GPU setup.
-
-* The `MI300X workload optimization guide <https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/workload.html>`_
-  provides detailed guidance on optimizing workloads for the AMD Instinct MI300X
-  accelerator using ROCm. The page is aimed at helping users achieve optimal
-  performance for deep learning and other high-performance computing tasks on
-  the MI300X GPU.
-
-For more use cases and recommendations, see `ROCm JAX blog posts <https://rocm.blogs.amd.com/blog/tag/jax.html>`_.

docs/compatibility/ml-compatibility/pytorch-compatibility.rst

@@ -4,6 +4,8 @@
     :description: PyTorch compatibility
     :keywords: GPU, PyTorch compatibility
 
+.. version-set:: rocm_version latest
+
 ********************************************************************************
 PyTorch compatibility
 ********************************************************************************
@@ -19,31 +21,68 @@ release cycles for PyTorch on ROCm:
 
 - ROCm PyTorch release:
 
-  - Provides the latest version of ROCm but doesn't immediately support the latest stable PyTorch
+  - Provides the latest version of ROCm but might not necessarily support the
-    version.
+    latest stable PyTorch version.
 
   - Offers :ref:`Docker images <pytorch-docker-compat>` with ROCm and PyTorch
-    pre-installed.
+    preinstalled.
 
   - ROCm PyTorch repository: `<https://github.com/ROCm/pytorch>`_
 
-  - See the :doc:`ROCm PyTorch installation guide <rocm-install-on-linux:install/3rd-party/pytorch-install>` to get started.
+  - See the :doc:`ROCm PyTorch installation guide <rocm-install-on-linux:install/3rd-party/pytorch-install>`
+    to get started.
 
 - Official PyTorch release:
 
-  - Provides the latest stable version of PyTorch but doesn't immediately support the latest ROCm version.
+  - Provides the latest stable version of PyTorch  but might not necessarily
+    support the latest ROCm version.
 
   - Official PyTorch repository: `<https://github.com/pytorch/pytorch>`_
 
   - See the `Nightly and latest stable version installation guide <https://pytorch.org/get-started/locally/>`_
-    or `Previous versions <https://pytorch.org/get-started/previous-versions/>`_ to get started.
+    or `Previous versions <https://pytorch.org/get-started/previous-versions/>`_
+    to get started.
 
-The upstream PyTorch includes an automatic HIPification solution that automatically generates HIP
+PyTorch includes tooling that generates HIP source code from the CUDA backend.
-source code from the CUDA backend. This approach allows PyTorch to support ROCm without requiring
+This approach allows PyTorch to support ROCm without requiring manual code
-manual code modifications.
+modifications. For more information, see :doc:`HIPIFY <hipify:index>`.
 
-Development of ROCm is aligned with the stable release of PyTorch while upstream PyTorch testing uses
+ROCm development is aligned with the stable release of PyTorch, while upstream
-the stable release of ROCm to maintain consistency.
+PyTorch testing uses the stable release of ROCm to maintain consistency.
+
+.. _pytorch-recommendations:
+
+Use cases and recommendations
+================================================================================
+
+* :doc:`Using ROCm for AI: training a model </how-to/rocm-for-ai/training/benchmark-docker/pytorch-training>`
+  guides how to leverage the ROCm platform for training AI models. It covers the
+  steps, tools, and best practices for optimizing training workflows on AMD GPUs
+  using PyTorch features.
+
+* :doc:`Single-GPU fine-tuning and inference </how-to/rocm-for-ai/fine-tuning/single-gpu-fine-tuning-and-inference>`
+  describes and demonstrates how to use the ROCm platform for the fine-tuning
+  and inference of machine learning models, particularly large language models
+  (LLMs), on systems with a single GPU. This topic provides a detailed guide for
+  setting up, optimizing, and executing fine-tuning and inference workflows in
+  such environments.
+
+* :doc:`Multi-GPU fine-tuning and inference optimization </how-to/rocm-for-ai/fine-tuning/multi-gpu-fine-tuning-and-inference>`
+  describes and demonstrates the fine-tuning and inference of machine learning
+  models on systems with multiple GPUs.
+
+* The :doc:`Instinct MI300X workload optimization guide </how-to/rocm-for-ai/inference-optimization/workload>`
+  provides detailed guidance on optimizing workloads for the AMD Instinct MI300X
+  accelerator using ROCm. This guide helps users achieve optimal performance for
+  deep learning and other high-performance computing tasks on the MI300X
+  accelerator.
+
+* The :doc:`Inception with PyTorch documentation </conceptual/ai-pytorch-inception>`
+  describes how PyTorch integrates with ROCm for AI workloads It outlines the
+  use of PyTorch on the ROCm platform and focuses on efficiently leveraging AMD
+  GPU hardware for training and inference tasks in AI applications.
+
+For more use cases and recommendations, see `ROCm PyTorch blog posts <https://rocm.blogs.amd.com/blog/tag/pytorch.html>`_.
 
 .. _pytorch-docker-compat:
 
@@ -54,10 +93,10 @@ Docker image compatibility
 
    <i class="fab fa-docker"></i>
 
-AMD validates and publishes ready-made `PyTorch images <https://hub.docker.com/r/rocm/pytorch>`_
+AMD validates and publishes `PyTorch images <https://hub.docker.com/r/rocm/pytorch>`_
-with ROCm backends on Docker Hub. The following Docker image tags and
+with ROCm backends on Docker Hub. The following Docker image tags and associated
-associated inventories are validated for `ROCm 6.3.0 <https://repo.radeon.com/rocm/apt/6.3/>`_.
+inventories were tested on `ROCm 6.4.0 <https://repo.radeon.com/rocm/apt/6.4/>`_.
-Click the |docker-icon| icon to view the image on Docker Hub.
+Click |docker-icon| to view the image on Docker Hub.
 
 .. list-table:: PyTorch Docker image components
     :header-rows: 1
@@ -77,28 +116,28 @@ Click the |docker-icon| icon to view the image on Docker Hub.
 
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu24.04_py3.12_pytorch_release_2.4.0/images/sha256-98ddf20333bd01ff749b8092b1190ee369a75d3b8c71c2fac80ffdcb1a98d529?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.4_ubuntu24.04_py3.12_pytorch_release_2.6.0/images/sha256-ab1d350b818b90123cfda31363019d11c0d41a8f12a19e3cb2cb40cf0261137d"><i class="fab fa-docker fa-lg"></i></a>
 
-      - `2.4.0 <https://github.com/ROCm/pytorch/tree/release/2.4>`_
+      - `2.6.0 <https://github.com/ROCm/pytorch/tree/release/2.6>`_
       - 24.04
-      - `3.12 <https://www.python.org/downloads/release/python-3128/>`_
+      - `3.12.9 <https://www.python.org/downloads/release/python-3129/>`_
-      - `1.4.0 <https://github.com/ROCm/apex/tree/release/1.4.0>`_
+      - `1.6.0 <https://github.com/ROCm/apex/tree/release/1.6.0>`_
-      - `0.19.0 <https://github.com/pytorch/vision/tree/v0.19.0>`_
+      - `0.21.0 <https://github.com/pytorch/vision/tree/v0.21.0>`_
       - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13.0>`_
       - `master <https://bitbucket.org/icl/magma/src/master/>`_
       - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
-      - `4.0.7 <https://github.com/open-mpi/ompi/tree/v4.0.7>`_
+      - `4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
       - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
 
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu22.04_py3.10_pytorch_release_2.4.0/images/sha256-402c9b4f1a6b5a81c634a1932b56cbe01abb699cfcc7463d226276997c6cf8ea?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.4_ubuntu22.04_py3.10_pytorch_release_2.6.0/images/sha256-130536fdfceb374626a7bcb8d00b9d796ddfc3115677d51229e5b852d96b5ef4"><i class="fab fa-docker fa-lg"></i></a>
 
-      - `2.4.0 <https://github.com/ROCm/pytorch/tree/release/2.4>`_
+      - `2.6.0 <https://github.com/ROCm/pytorch/tree/release/2.6>`_
       - 22.04
-      - `3.10 <https://www.python.org/downloads/release/python-31016/>`_
+      - `3.10.16 <https://www.python.org/downloads/release/python-31016/>`_
-      - `1.4.0 <https://github.com/ROCm/apex/tree/release/1.4.0>`_
+      - `1.6.0 <https://github.com/ROCm/apex/tree/release/1.6.0>`_
-      - `0.19.0 <https://github.com/pytorch/vision/tree/v0.19.0>`_
+      - `0.21.0 <https://github.com/pytorch/vision/tree/v0.21.0>`_
       - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13.0>`_
       - `master <https://bitbucket.org/icl/magma/src/master/>`_
       - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
@@ -107,100 +146,115 @@ Click the |docker-icon| icon to view the image on Docker Hub.
 
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu22.04_py3.9_pytorch_release_2.4.0/images/sha256-e0608b55d408c3bfe5c19fdd57a4ced3e0eb3a495b74c309980b60b156c526dd?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.4_ubuntu24.04_py3.12_pytorch_release_2.5.1/images/sha256-20a2e24b4738dc1f1a44a04f23827918b56c99f7e697e6fccb90e9c4fae8ca9b"><i class="fab fa-docker fa-lg"></i></a>
 
-      - `2.4.0 <https://github.com/ROCm/pytorch/tree/release/2.4>`_
+      - `2.5.1 <https://github.com/ROCm/pytorch/tree/release/2.5>`_
+      - 24.04
+      - `3.12.9 <https://www.python.org/downloads/release/python-3129/>`_
+      - `1.5.0 <https://github.com/ROCm/apex/tree/release/1.5.0>`_
+      - `0.20.1 <https://github.com/pytorch/vision/tree/v0.20.1>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13.0>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+      - `4.0.7 <https://github.com/open-mpi/ompi/tree/v4.0.7>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.4_ubuntu22.04_py3.11_pytorch_release_2.5.1/images/sha256-f09cb8ca39cc39222fb554060711f5c19130f7b4047aaf41fad4ba3ec470ca03"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `2.5.1 <https://github.com/ROCm/pytorch/tree/release/2.5>`_
       - 22.04
-      - `3.9.18 <https://www.python.org/downloads/release/python-3918/>`_
+      - `3.11.9 <https://www.python.org/downloads/release/python-3119/>`_
+      - `1.5.0 <https://github.com/ROCm/apex/tree/release/1.5.0>`_
+      - `0.20.1 <https://github.com/pytorch/vision/tree/v0.20.1>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13.0>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.14.1 <https://github.com/openucx/ucx/tree/v1.14.1>`_
+      - `4.1.5 <https://github.com/open-mpi/ompi/tree/v4.1.5>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.4_ubuntu22.04_py3.10_pytorch_release_2.5.1/images/sha256-a91c100d1fe608dae3eb7f60a751630363d4027ac3d077d428e92945204c338e"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `2.5.1 <https://github.com/ROCm/pytorch/tree/release/2.5>`_
+      - 22.04
+      - `3.10.16 <https://www.python.org/downloads/release/python-31016/>`_
+      - `1.5.0 <https://github.com/ROCm/apex/tree/release/1.5.0>`_
+      - `0.20.1 <https://github.com/pytorch/vision/tree/v0.20.1>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13.0>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.14.1 <https://github.com/openucx/ucx/tree/v1.14.1>`_
+      - `4.1.5 <https://github.com/open-mpi/ompi/tree/v4.1.5>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.4_ubuntu24.04_py3.12_pytorch_release_2.4.1/images/sha256-66a89ce6485bb887af74bb9bd76bb613ab9834a6b1374649ea7ae379883454a4"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `2.4.1 <https://github.com/ROCm/pytorch/tree/release/2.4>`_
+      - 24.04
+      - `3.12.9 <https://www.python.org/downloads/release/python-3129/>`_
       - `1.4.0 <https://github.com/ROCm/apex/tree/release/1.4.0>`_
       - `0.19.0 <https://github.com/pytorch/vision/tree/v0.19.0>`_
       - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13.0>`_
       - `master <https://bitbucket.org/icl/magma/src/master/>`_
       - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
-      - `4.0.7 <https://github.com/open-mpi/ompi/tree/v4.0.7>`_
+      - `4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
       - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
 
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu22.04_py3.10_pytorch_release_2.3.0/images/sha256-652cf25263d05b1de548222970aeb76e60b12de101de66751264709c0d0ff9d8?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.4_ubuntu22.04_py3.10_pytorch_release_2.4.1/images/sha256-c716cf167e6e49893f11de03606ed37044153aca089e74ca615065c06877f86b"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `2.4.1 <https://github.com/ROCm/pytorch/tree/release/2.4>`_
+      - 22.04
+      - `3.10.16 <https://www.python.org/downloads/release/python-31016/>`_
+      - `1.4.0 <https://github.com/ROCm/apex/tree/release/1.4.0>`_
+      - `0.19.0 <https://github.com/pytorch/vision/tree/v0.19.0>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13.0>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.14.1 <https://github.com/openucx/ucx/tree/v1.14.1>`_
+      - `4.1.5 <https://github.com/open-mpi/ompi/tree/v4.1.5>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.4_ubuntu24.04_py3.12_pytorch_release_2.3.0/images/sha256-0434cbc9b07b2c26e39480d7447f676f9057a1054dcff00e0050c25a6eddbd3c"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `2.3.0 <https://github.com/ROCm/pytorch/tree/release/2.3>`_
+      - 24.04
+      - `3.12.9 <https://www.python.org/downloads/release/python-3129/>`_
+      - `1.3.0 <https://github.com/ROCm/apex/tree/release/1.3.0>`_
+      - `0.18.0 <https://github.com/pytorch/vision/tree/v0.18.0>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+      - `4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.4_ubuntu22.04_py3.10_pytorch_release_2.3.0/images/sha256-688b1c0073092615fb98778d78b16191e506097ee116a2d3d2628b264d5d367b"><i class="fab fa-docker fa-lg"></i></a>
 
       - `2.3.0 <https://github.com/ROCm/pytorch/tree/release/2.3>`_
       - 22.04
-      - `3.10 <https://www.python.org/downloads/release/python-31016/>`_
+      - `3.10.16 <https://www.python.org/downloads/release/python-31016/>`_
       - `1.3.0 <https://github.com/ROCm/apex/tree/release/1.3.0>`_
       - `0.18.0 <https://github.com/pytorch/vision/tree/v0.18.0>`_
-      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13.0>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13>`_
-      - `master <https://bitbucket.org/icl/magma/src/master/>`_
-      - `1.14.1 <https://github.com/openucx/ucx/tree/v1.14.1>`_
-      - `4.1.5 <https://github.com/open-mpi/ompi/tree/v4.1.5>`_
-      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
-
-    * - .. raw:: html
-
-           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu22.04_py3.10_pytorch_release_2.2.1/images/sha256-051976f26beab8f9aa65d999e3ad546c027b39240a0cc3ee81b114a9024f2912?context=explore"><i class="fab fa-docker fa-lg"></i></a>
-
-      - `2.2.1 <https://github.com/ROCm/pytorch/tree/release/2.2>`_
-      - 22.04
-      - `3.10 <https://www.python.org/downloads/release/python-31016/>`_
-      - `1.2.0 <https://github.com/ROCm/apex/tree/release/1.2.0>`_
-      - `0.17.1 <https://github.com/pytorch/vision/tree/v0.17.1>`_
-      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13.0>`_
-      - `master <https://bitbucket.org/icl/magma/src/master/>`_
-      - `1.14.1 <https://github.com/openucx/ucx/tree/v1.14.1>`_
-      - `4.1.5 <https://github.com/open-mpi/ompi/tree/v4.1.5>`_
-      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
-
-    * - .. raw:: html
-
-           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu20.04_py3.9_pytorch_release_2.2.1/images/sha256-88c839a364d109d3748c100385bfa100d28090d25118cc723fd0406390ab2f7e?context=explore"><i class="fab fa-docker fa-lg"></i></a>
-
-      - `2.2.1 <https://github.com/ROCm/pytorch/tree/release/2.2>`_
-      - 20.04
-      - `3.9.21 <https://www.python.org/downloads/release/python-3921/>`_
-      - `1.2.0 <https://github.com/ROCm/apex/tree/release/1.2.0>`_
-      - `0.17.1 <https://github.com/pytorch/vision/tree/v0.17.1>`_
-      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13.0>`_
       - `master <https://bitbucket.org/icl/magma/src/master/>`_
       - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
       - `4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
       - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
 
-    * - .. raw:: html
+Key ROCm libraries for PyTorch
-
-           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu22.04_py3.9_pytorch_release_1.13.1/images/sha256-994424ed07a63113f79dd9aa72159124c00f5fbfe18127151e6658f7d0b6f821?context=explore"><i class="fab fa-docker fa-lg"></i></a>
-
-      - `1.13.1 <https://github.com/ROCm/pytorch/tree/release/1.13>`_
-      - 22.04
-      - `3.9.21 <https://www.python.org/downloads/release/python-3921/>`_
-      - `1.0.0 <https://github.com/ROCm/apex/tree/release/1.0.0>`_
-      - `0.14.0 <https://github.com/pytorch/vision/tree/v0.14.0>`_
-      - `2.18.0 <https://github.com/tensorflow/tensorboard/tree/2.18>`_
-      - `master <https://bitbucket.org/icl/magma/src/master/>`_
-      - `1.14.1 <https://github.com/openucx/ucx/tree/v1.14.1>`_
-      - `4.1.5 <https://github.com/open-mpi/ompi/tree/v4.1.5>`_
-      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
-
-    * - .. raw:: html
-
-           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu20.04_py3.9_pytorch_release_1.13.1/images/sha256-7b8139fe40a9aeb4bca3aecd15c22c1fa96e867d93479fa3a24fdeeeeafa1219?context=explore"><i class="fab fa-docker fa-lg"></i></a>
-
-      - `1.13.1 <https://github.com/ROCm/pytorch/tree/release/1.13>`_
-      - 20.04
-      - `3.9.21 <https://www.python.org/downloads/release/python-3921/>`_
-      - `1.0.0 <https://github.com/ROCm/apex/tree/release/1.0.0>`_
-      - `0.14.0 <https://github.com/pytorch/vision/tree/v0.14.0>`_
-      - `2.18.0 <https://github.com/tensorflow/tensorboard/tree/2.18>`_
-      - `master <https://bitbucket.org/icl/magma/src/master/>`_
-      - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
-      - `4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
-      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
-
-Critical ROCm libraries for PyTorch
 ================================================================================
 
-The functionality of PyTorch with ROCm is determined by its underlying library
+PyTorch functionality on ROCm is determined by its underlying library
-dependencies. These critical ROCm components affect the capabilities,
+dependencies. These ROCm components affect the capabilities, performance, and
-performance, and feature set available to developers.
+feature set available to developers.
 
 .. list-table::
     :header-rows: 1
@@ -210,122 +264,121 @@ performance, and feature set available to developers.
       - Purpose
       - Used in
     * - `Composable Kernel <https://github.com/ROCm/composable_kernel>`_
-      - 1.1.0
+      - :version-ref:`"Composable Kernel" rocm_version`
       - Enables faster execution of core operations like matrix multiplication
         (GEMM), convolutions and transformations.
       - Speeds up ``torch.permute``, ``torch.view``, ``torch.matmul``,
         ``torch.mm``, ``torch.bmm``, ``torch.nn.Conv2d``, ``torch.nn.Conv3d``
         and ``torch.nn.MultiheadAttention``.
     * - `hipBLAS <https://github.com/ROCm/hipBLAS>`_
-      - 2.3.0
+      - :version-ref:`hipBLAS rocm_version`
       - Provides GPU-accelerated Basic Linear Algebra Subprograms (BLAS) for
         matrix and vector operations.
-      - Supports operations like matrix multiplication, matrix-vector products,
+      - Supports operations such as matrix multiplication, matrix-vector
-        and tensor contractions. Utilized in both dense and batched linear
+        products, and tensor contractions. Utilized in both dense and batched
-        algebra operations.
+        linear algebra operations.
     * - `hipBLASLt <https://github.com/ROCm/hipBLASLt>`_
-      - 0.10.0
+      - :version-ref:`hipBLASLt rocm_version`
       - hipBLASLt is an extension of the hipBLAS library, providing additional
         features like epilogues fused into the matrix multiplication kernel or
         use of integer tensor cores.
-      - It accelerates operations like ``torch.matmul``, ``torch.mm``, and the
+      - Accelerates operations such as ``torch.matmul``, ``torch.mm``, and the
         matrix multiplications used in convolutional and linear layers.
     * - `hipCUB <https://github.com/ROCm/hipCUB>`_
-      - 3.3.0
+      - :version-ref:`hipCUB rocm_version`
       - Provides a C++ template library for parallel algorithms for reduction,
         scan, sort and select.
-      - Supports operations like ``torch.sum``, ``torch.cumsum``, ``torch.sort``
+      - Supports operations such as ``torch.sum``, ``torch.cumsum``,
-        and ``torch.topk``. Operations on sparse tensors or tensors with
+        ``torch.sort`` irregular shapes often involve scanning, sorting, and
-        irregular shapes often involve scanning, sorting, and filtering, which
+        filtering, which hipCUB handles efficiently.
-        hipCUB handles efficiently.
     * - `hipFFT <https://github.com/ROCm/hipFFT>`_
-      - 1.0.17
+      - :version-ref:`hipFFT rocm_version`
       - Provides GPU-accelerated Fast Fourier Transform (FFT) operations.
       - Used in functions like the ``torch.fft`` module.
     * - `hipRAND <https://github.com/ROCm/hipRAND>`_
-      - 2.11.0
+      - :version-ref:`hipRAND rocm_version`
       - Provides fast random number generation for GPUs.
-      - The ``torch.rand``, ``torch.randn`` and stochastic layers like
+      - The ``torch.rand``, ``torch.randn``, and stochastic layers like
-        ``torch.nn.Dropout``.
+        ``torch.nn.Dropout`` rely on hipRAND.
     * - `hipSOLVER <https://github.com/ROCm/hipSOLVER>`_
-      - 2.3.0
+      - :version-ref:`hipSOLVER rocm_version`
       - Provides GPU-accelerated solvers for linear systems, eigenvalues, and
         singular value decompositions (SVD).
       - Supports functions like ``torch.linalg.solve``,
         ``torch.linalg.eig``, and ``torch.linalg.svd``.
     * - `hipSPARSE <https://github.com/ROCm/hipSPARSE>`_
-      - 3.1.2
+      - :version-ref:`hipSPARSE rocm_version`
       - Accelerates operations on sparse matrices, such as sparse matrix-vector
         or matrix-matrix products.
       - Sparse tensor operations ``torch.sparse``.
     * - `hipSPARSELt <https://github.com/ROCm/hipSPARSELt>`_
-      - 0.2.2
+      - :version-ref:`hipSPARSELt rocm_version`
       - Accelerates operations on sparse matrices, such as sparse matrix-vector
         or matrix-matrix products.
       - Sparse tensor operations ``torch.sparse``.
     * - `hipTensor <https://github.com/ROCm/hipTensor>`_
-      - 1.4.0
+      - :version-ref:`hipTensor rocm_version`
       - Optimizes for high-performance tensor operations, such as contractions.
       - Accelerates tensor algebra, especially in deep learning and scientific
         computing.
     * - `MIOpen <https://github.com/ROCm/MIOpen>`_
-      - 3.3.0
+      - :version-ref:`MIOpen rocm_version`
       - Optimizes deep learning primitives such as convolutions, pooling,
         normalization, and activation functions.
       - Speeds up convolutional neural networks (CNNs), recurrent neural
         networks (RNNs), and other layers. Used in operations like
         ``torch.nn.Conv2d``, ``torch.nn.ReLU``, and ``torch.nn.LSTM``.
     * - `MIGraphX <https://github.com/ROCm/AMDMIGraphX>`_
-      - 2.11.0
+      - :version-ref:`MIGraphX rocm_version`
       - Adds graph-level optimizations, ONNX models and mixed precision support
         and enable Ahead-of-Time (AOT) Compilation.
       - Speeds up inference models and executes ONNX models for
         compatibility with other frameworks.
         ``torch.nn.Conv2d``, ``torch.nn.ReLU``, and ``torch.nn.LSTM``.
     * - `MIVisionX <https://github.com/ROCm/MIVisionX>`_
-      - 3.1.0
+      - :version-ref:`MIVisionX rocm_version`
       - Optimizes acceleration for computer vision and AI workloads like
         preprocessing, augmentation, and inferencing.
       - Faster data preprocessing and augmentation pipelines for datasets like
         ImageNet or COCO and easy to integrate into PyTorch's ``torch.utils.data``
         and ``torchvision`` workflows.
     * - `rocAL <https://github.com/ROCm/rocAL>`_
-      - 2.1.0
+      - :version-ref:`rocAL rocm_version`
       - Accelerates the data pipeline by offloading intensive preprocessing and
         augmentation tasks. rocAL is part of MIVisionX.
       - Easy to integrate into PyTorch's ``torch.utils.data`` and
         ``torchvision`` data load workloads.
     * - `RCCL <https://github.com/ROCm/rccl>`_
-      - 2.21.5
+      - :version-ref:`RCCL rocm_version`
       - Optimizes for multi-GPU communication for operations like AllReduce and
         Broadcast.
       - Distributed data parallel training (``torch.nn.parallel.DistributedDataParallel``).
         Handles communication in multi-GPU setups.
     * - `rocDecode <https://github.com/ROCm/rocDecode>`_
-      - 0.8.0
+      - :version-ref:`rocDecode rocm_version`
       - Provides hardware-accelerated data decoding capabilities, particularly
         for image, video, and other dataset formats.
       - Can be integrated in ``torch.utils.data``, ``torchvision.transforms``
         and ``torch.distributed``.
     * - `rocJPEG <https://github.com/ROCm/rocJPEG>`_
-      - 0.6.0
+      - :version-ref:`rocJPEG rocm_version`
       - Provides hardware-accelerated JPEG image decoding and encoding.
       - GPU accelerated ``torchvision.io.decode_jpeg`` and
         ``torchvision.io.encode_jpeg`` and can be integrated in
         ``torch.utils.data`` and ``torchvision``.
     * - `RPP <https://github.com/ROCm/RPP>`_
-      - 1.9.1
+      - :version-ref:`RPP rocm_version`
       - Speeds up data augmentation, transformation, and other preprocessing steps.
       - Easy to integrate into PyTorch's ``torch.utils.data`` and
-        ``torchvision`` data load workloads.
+        ``torchvision`` data load workloads to speed up data processing.
     * - `rocThrust <https://github.com/ROCm/rocThrust>`_
-      - 3.3.0
+      - :version-ref:`rocThrust rocm_version`
       - Provides a C++ template library for parallel algorithms like sorting,
         reduction, and scanning.
       - Utilized in backend operations for tensor computations requiring
         parallel processing.
     * - `rocWMMA <https://github.com/ROCm/rocWMMA>`_
-      - 1.6.0
+      - :version-ref:`rocWMMA rocm_version`
       - Accelerates warp-level matrix-multiply and matrix-accumulate to speed up matrix
         multiplication (GEMM) and accumulation operations with mixed precision
         support.
@@ -334,11 +387,11 @@ performance, and feature set available to developers.
         involve matrix products, such as ``torch.matmul``, ``torch.bmm``, and
         more.
 
-Supported and unsupported features
+Supported features
 ================================================================================
 
-The following section maps GPU-accelerated PyTorch features to their supported
+This section maps GPU-accelerated PyTorch features to their supported ROCm and
-ROCm and PyTorch versions.
+PyTorch versions.
 
 torch
 --------------------------------------------------------------------------------
@@ -346,23 +399,24 @@ torch
 `torch <https://pytorch.org/docs/stable/index.html>`_ is the central module of
 PyTorch, providing data structures for multi-dimensional tensors and
 implementing mathematical operations on them. It also includes utilities for
-efficient serialization of tensors and arbitrary data types, along with various
+efficient serialization of tensors and arbitrary data types and other tools.
-other tools.
 
 Tensor data types
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The data type of a tensor is specified using the ``dtype`` attribute or argument, and PyTorch supports a wide range of data types for different use cases.
+The tensor data type is specified using the ``dtype`` attribute or argument. 
+PyTorch supports many data types for different use cases.
 
-The following table lists `torch.Tensor <https://pytorch.org/docs/stable/tensors.html>`_'s single data types:
+The following table lists `torch.Tensor <https://pytorch.org/docs/stable/tensors.html>`_
+single data types:
 
 .. list-table::
     :header-rows: 1
 
     * - Data type
       - Description
-      - Since PyTorch
+      - As of PyTorch
-      - Since ROCm
+      - As of ROCm
     * - ``torch.float8_e4m3fn``
       - 8-bit floating point, e4m3
       - 2.3
@@ -454,11 +508,11 @@ The following table lists `torch.Tensor <https://pytorch.org/docs/stable/tensors
 
 .. note::
 
-  Unsigned types aside from ``uint8`` are currently only have limited support in
+  Unsigned types except ``uint8`` have limited support in eager mode. They
-  eager mode (they primarily exist to assist usage with ``torch.compile``).
+  primarily exist to assist usage with ``torch.compile``.
 
-  The :doc:`ROCm precision support page <rocm:reference/precision-support>`
+  See :doc:`ROCm precision support <rocm:reference/precision-support>` for the
-  collected the native HW support of different data types.
+  native hardware support of data types.
 
 torch.cuda
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -473,8 +527,8 @@ leveraging ROCm and CUDA as the underlying frameworks.
 
     * - Feature
       - Description
-      - Since PyTorch
+      - As of PyTorch
-      - Since ROCm
+      - As of ROCm
     * - Device management
       - Utilities for managing and interacting with GPUs.
       - 0.4.0
@@ -548,8 +602,8 @@ PyTorch interacts with the ROCm or CUDA environment.
 
     * - Feature
       - Description
-      - Since PyTorch
+      - As of PyTorch
-      - Since ROCm
+      - As of ROCm
     * - ``cufft_plan_cache``
       - Manages caching of GPU FFT plans to optimize repeated FFT computations.
       - 1.7.0
@@ -597,8 +651,8 @@ Supported ``torch`` options include:
 
     * - Option
       - Description
-      - Since PyTorch
+      - As of PyTorch
-      - Since ROCm
+      - As of ROCm
     * - ``allow_tf32``
       - TensorFloat-32 tensor cores may be used in cuDNN convolutions on NVIDIA
         Ampere or newer GPUs.
@@ -613,28 +667,28 @@ Supported ``torch`` options include:
 Automatic mixed precision: torch.amp
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-PyTorch that automates the process of using both 16-bit (half-precision,
+PyTorch automates the process of using both 16-bit (half-precision, float16) and
-float16) and 32-bit (single-precision, float32) floating-point types in model
+32-bit (single-precision, float32) floating-point types in model training and
-training and inference.
+inference.
 
 .. list-table::
     :header-rows: 1
 
     * - Feature
       - Description
-      - Since PyTorch
+      - As of PyTorch
-      - Since ROCm
+      - As of ROCm
     * - Autocasting
-      - Instances of autocast serve as context managers or decorators that allow
+      - Autocast instances serve as context managers or decorators that allow
         regions of your script to run in mixed precision.
       - 1.9
       - 2.5
     * - Gradient scaling
       - To prevent underflow, “gradient scaling” multiplies the network’s
-        loss(es) by a scale factor and invokes a backward pass on the scaled
+        loss by a scale factor and invokes a backward pass on the scaled
-        loss(es). Gradients flowing backward through the network are then
+        loss. The same factor then scales gradients flowing backward through
-        scaled by the same factor. In other words, gradient values have a
+        the network. In other words, gradient values have a larger magnitude so
-        larger magnitude, so they don’t flush to zero.
+        that they don’t flush to zero.
       - 1.9
       - 2.5
     * - CUDA op-specific behavior
@@ -648,7 +702,7 @@ training and inference.
 Distributed library features
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The PyTorch distributed library includes a collective of parallelism modules, a
+PyTorch distributed library includes a collective of parallelism modules, a
 communications layer, and infrastructure for launching and debugging large
 training jobs. See :ref:`rocm-for-ai-pytorch-distributed` for more information.
 
@@ -662,13 +716,13 @@ of computational resources and scalability for large-scale tasks.
 
     * - Feature
       - Description
-      - Since PyTorch
+      - As of PyTorch
-      - Since ROCm
+      - As of ROCm
     * - TensorPipe
       - A point-to-point communication library integrated into
-        PyTorch for distributed training. It is designed to handle tensor data
+        PyTorch for distributed training. It handles tensor data transfers
-        transfers efficiently between different processes or devices, including
+        efficiently between different processes or devices, including those on
-        those on separate machines.
+        separate machines.
       - 1.8
       - 5.4
     * - Gloo
@@ -687,8 +741,8 @@ torch.compiler
 
     * - Feature
       - Description
-      - Since PyTorch
+      - As of PyTorch
-      - Since ROCm
+      - As of ROCm
     * - ``torch.compiler`` (AOT Autograd)
       - Autograd captures not only the user-level code, but also backpropagation,
         which results in capturing the backwards pass “ahead-of-time”. This
@@ -711,8 +765,8 @@ The `torchaudio <https://pytorch.org/audio/stable/index.html>`_ library provides
 utilities for processing audio data in PyTorch, such as audio loading,
 transformations, and feature extraction.
 
-To ensure GPU-acceleration with ``torchaudio.transforms``, you need to move audio
+To ensure GPU-acceleration with ``torchaudio.transforms``, you need to
-data (waveform tensor) explicitly to GPU using ``.to('cuda')``.
+explicitly move audio data (waveform tensor) to GPU using ``.to('cuda')``.
 
 The following ``torchaudio`` features are GPU-accelerated.
 
@@ -721,10 +775,10 @@ The following ``torchaudio`` features are GPU-accelerated.
 
     * - Feature
       - Description
-      - Since torchaudio version
+      - As of torchaudio version
-      - Since ROCm
+      - As of ROCm
     * - ``torchaudio.transforms.Spectrogram``
-      - Generates spectrogram of an input waveform using STFT.
+      - Generate a spectrogram of an input waveform using STFT.
       - 0.6.0
       - 4.5
     * - ``torchaudio.transforms.MelSpectrogram``
@@ -744,7 +798,7 @@ torchvision
 --------------------------------------------------------------------------------
 
 The `torchvision <https://pytorch.org/vision/stable/index.html>`_ library
-provide datasets, model architectures, and common image transformations for
+provides datasets, model architectures, and common image transformations for
 computer vision.
 
 The following ``torchvision`` features are GPU-accelerated.
@@ -754,8 +808,8 @@ The following ``torchvision`` features are GPU-accelerated.
 
     * - Feature
       - Description
-      - Since torchvision version
+      - As of torchvision version
-      - Since ROCm
+      - As of ROCm
     * - ``torchvision.transforms.functional``
       - Provides GPU-compatible transformations for image preprocessing like
         resize, normalize, rotate and crop.
@@ -801,18 +855,19 @@ torchtune
 The `torchtune <https://pytorch.org/torchtune/stable/index.html>`_ library for
 authoring, fine-tuning and experimenting with LLMs.
 
-* Usage: It works out-of-the-box, enabling developers to fine-tune ROCm PyTorch solutions.
+* Usage: Enabling developers to fine-tune ROCm PyTorch solutions.
 
 * Only official release exists.
 
 torchserve
 --------------------------------------------------------------------------------
 
-The `torchserve <https://pytorch.org/torchserve/>`_ is a PyTorch domain library
+The `torchserve <https://pytorch.org/serve/>`_ is a PyTorch domain library
 for common sparsity and parallelism primitives needed for large-scale recommender
 systems.
 
-* torchtext does not implement its own kernels. ROCm support is enabled by linking against ROCm libraries.
+* torchtext does not implement its own kernels. ROCm support is enabled by
+  linking against ROCm libraries.
 
 * Only official release exists.
 
@@ -823,14 +878,16 @@ The `torchrec <https://pytorch.org/torchrec/>`_ is a PyTorch domain library for
 common sparsity and parallelism primitives needed for large-scale recommender
 systems.
 
-* torchrec does not implement its own kernels. ROCm support is enabled by linking against ROCm libraries.
+* torchrec does not implement its own kernels. ROCm support is enabled by
+  linking against ROCm libraries.
 
 * Only official release exists.
 
 Unsupported PyTorch features
-----------------------------
+================================================================================
 
-The following are GPU-accelerated PyTorch features not currently supported by ROCm.
+The following GPU-accelerated PyTorch features are not supported by ROCm for
+the listed supported PyTorch versions.
 
 .. list-table::
     :widths: 30, 60, 10
@@ -838,7 +895,7 @@ The following are GPU-accelerated PyTorch features not currently supported by RO
 
     * - Feature
       - Description
-      - Since PyTorch
+      - As of PyTorch
     * - APEX batch norm
       - Use APEX batch norm instead of PyTorch batch norm.
       - 1.6.0
@@ -894,31 +951,3 @@ The following are GPU-accelerated PyTorch features not currently supported by RO
         utilized effectively through custom CUDA extensions or advanced
         workflows.
       - Not a core feature
-
-Use cases and recommendations
-================================================================================
-
-* :doc:`Using ROCm for AI: training a model </how-to/rocm-for-ai/training/train-a-model>` provides
-  guidance on how to leverage the ROCm platform for training AI models. It covers the steps, tools, and best practices
-  for optimizing training workflows on AMD GPUs using PyTorch features.
-
-* :doc:`Single-GPU fine-tuning and inference </how-to/rocm-for-ai/fine-tuning/single-gpu-fine-tuning-and-inference>`
-  describes and demonstrates how to use the ROCm platform for the fine-tuning and inference of
-  machine learning models, particularly large language models (LLMs), on systems with a single AMD
-  Instinct MI300X accelerator. This page provides a detailed guide for setting up, optimizing, and
-  executing fine-tuning and inference workflows in such environments.
-
-* :doc:`Multi-GPU fine-tuning and inference optimization </how-to/rocm-for-ai/fine-tuning/multi-gpu-fine-tuning-and-inference>`
-  describes and demonstrates the fine-tuning and inference of machine learning models on systems
-  with multi MI300X accelerators.
-
-* The :doc:`Instinct MI300X workload optimization guide </how-to/rocm-for-ai/inference-optimization/workload>` provides detailed
-  guidance on optimizing workloads for the AMD Instinct MI300X accelerator using ROCm. This guide is aimed at helping
-  users achieve optimal performance for deep learning and other high-performance computing tasks on the MI300X
-  accelerator.
-
-* The :doc:`Inception with PyTorch documentation </conceptual/ai-pytorch-inception>`
-  describes how PyTorch integrates with ROCm for AI workloads It outlines the use of PyTorch on the ROCm platform and
-  focuses on how to efficiently leverage AMD GPU hardware for training and inference tasks in AI applications.
-
-For more use cases and recommendations, see `ROCm PyTorch blog posts <https://rocm.blogs.amd.com/blog/tag/pytorch.html>`_.

docs/compatibility/ml-compatibility/tensorflow-compatibility.rst

@@ -4,6 +4,8 @@
     :description: TensorFlow compatibility
     :keywords: GPU, TensorFlow compatibility
 
+.. version-set:: rocm_version latest
+
 *******************************************************************************
 TensorFlow compatibility
 *******************************************************************************
@@ -54,7 +56,7 @@ Docker image compatibility
 AMD validates and publishes ready-made `TensorFlow images
 <https://hub.docker.com/r/rocm/tensorflow>`_ with ROCm backends on
 Docker Hub. The following Docker image tags and associated inventories are
-validated for `ROCm 6.3.1 <https://repo.radeon.com/rocm/apt/6.3.1/>`_. Click
+validated for `ROCm 6.4.0 <https://repo.radeon.com/rocm/apt/6.4/>`_. Click
 the |docker-icon| icon to view the image on Docker Hub.
 
 .. list-table:: TensorFlow Docker image components
@@ -62,54 +64,91 @@ the |docker-icon| icon to view the image on Docker Hub.
 
     * - Docker image
       - TensorFlow
+      - Ubuntu
       - Dev
       - Python
       - TensorBoard
 
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.3.1-py3.12-tf2.17.0-dev/images/sha256-804121ee4985718277ba7dcec53c57bdade130a1ef42f544b6c48090ad379c17"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
+           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.4-py3.12-tf2.18-dev/images/sha256-fa9cf5fa6c6079a7118727531ccd0056c6e3224a42c3d6e78a49e7781daafff4"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
 
-      - `tensorflow-rocm 2.17.0 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.3/tensorflow_rocm-2.17.0-cp312-cp312-manylinux_2_28_x86_64.whl>`__
+      - `tensorflow-rocm 2.18.1 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.4/tensorflow_rocm-2.18.1-cp312-cp312-manylinux_2_28_x86_64.whl>`__
       - dev
-      - `Python 3.12 <https://www.python.org/downloads/release/python-3124/>`_
+      - 24.04
+      - `Python 3.12.4 <https://www.python.org/downloads/release/python-3124/>`_
+      - `TensorBoard 2.18.0 <https://github.com/tensorflow/tensorboard/tree/2.18.0>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.4-py3.12-tf2.18-runtime/images/sha256-14addca4b92a47c806b83ebaeed593fc6672cd99f0017ed8dad759fe72ed0309"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
+
+      - `tensorflow-rocm 2.18.1 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.4/tensorflow_rocm-2.18.1-cp312-cp312-manylinux_2_28_x86_64.whl>`__
+      - runtime
+      - 24.04
+      - `Python 3.12.4 <https://www.python.org/downloads/release/python-3124/>`_
+      - `TensorBoard 2.18.0 <https://github.com/tensorflow/tensorboard/tree/2.18.0>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.4-py3.10-tf2.18-dev/images/sha256-f5e151060df04ff5fb59f5604b49cd371931bbe75b06aec9fe7781397c4be0ce"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
+
+      - `tensorflow-rocm 2.18.1 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.4/tensorflow_rocm-2.18.1-cp310-cp310-manylinux_2_28_x86_64.whl>`__
+      - dev
+      - 22.04
+      - `Python 3.10.16 <https://www.python.org/downloads/release/python-31016/>`_
+      - `TensorBoard 2.18.0 <https://github.com/tensorflow/tensorboard/tree/2.18.0>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.4-py3.10-tf2.18-runtime/images/sha256-5cd4c03fdb1036570c0d4929da60a65c4466998dc80f1dc8a5a0b173eae017fb"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
+
+      - `tensorflow-rocm 2.18.1 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.4/tensorflow_rocm-2.18.1-cp310-cp310-manylinux_2_28_x86_64.whl>`__
+      - runtime
+      - 22.04
+      - `Python 3.10.16 <https://www.python.org/downloads/release/python-31016/>`_
+      - `TensorBoard 2.18.0 <https://github.com/tensorflow/tensorboard/tree/2.18.0>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.4-py3.12-tf2.17-dev/images/sha256-b3add80e374a2db2d1088d746e740afa89d439aca02cacba959ad298f5cd2b3f"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
+
+      - `tensorflow-rocm 2.17.1 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.4/tensorflow_rocm-2.17.1-cp312-cp312-manylinux_2_28_x86_64.whl>`__
+      - dev
+      - 24.04
+      - `Python 3.12.4 <https://www.python.org/downloads/release/python-3124/>`_
       - `TensorBoard 2.17.1 <https://github.com/tensorflow/tensorboard/tree/2.17.1>`_
 
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.3.1-py3.10-tf2.17.0-dev/images/sha256-776837ffa945913f6c466bfe477810a11453d21d5b6afb200be1c36e48fbc08e"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
+           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.4-py3.12-tf2.17-runtime/images/sha256-3a244f026c32177eff7958ffbad390de85b438b2b48b455cc39f15d70fa1270d"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
 
-      - `tensorflow-rocm 2.17.0 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.3/tensorflow_rocm-2.17.0-cp310-cp310-manylinux_2_28_x86_64.whl>`__
+      - `tensorflow-rocm 2.18.1 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.4/tensorflow_rocm-2.17.1-cp312-cp312-manylinux_2_28_x86_64.whl>`__
-      - dev
+      - runtime
-      - `Python 3.10 <https://www.python.org/downloads/release/python-31012/>`_
+      - 24.04
-      - `TensorBoard 2.17.0 <https://github.com/tensorflow/tensorboard/tree/2.17.0>`_
+      - `Python 3.12.4 <https://www.python.org/downloads/release/python-3124/>`_
+      - `TensorBoard 2.17.1 <https://github.com/tensorflow/tensorboard/tree/2.17.1>`_
 
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.3.1-py3.12-tf2.16.2-dev/images/sha256-c793e1483e30809c3c28fc5d7805bedc033c73da224f839fff370717cb100944"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
+           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.4-py3.10-tf2.17-dev/images/sha256-e0cecdfacb59169335049983cdab6da578c209bb9f4d08aad97e184ae59171a6"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
 
-      - `tensorflow-rocm 2.16.2 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.3/tensorflow_rocm-2.16.2-cp312-cp312-manylinux_2_28_x86_64.whl>`__
+      - `tensorflow-rocm 2.17.1 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.4/tensorflow_rocm-2.17.1-cp310-cp310-manylinux_2_28_x86_64.whl>`__
       - dev
-      - `Python 3.12 <https://www.python.org/downloads/release/python-3124/>`_
+      - 22.04
-      - `TensorBoard 2.16.2 <https://github.com/tensorflow/tensorboard/tree/2.16.2>`_
+      - `Python 3.10.16 <https://www.python.org/downloads/release/python-31016/>`_
+      - `TensorBoard 2.17.1 <https://github.com/tensorflow/tensorboard/tree/2.17.1>`_
 
     * - .. raw:: html
 
-           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.3.1-py3.10-tf2.16.0-dev/images/sha256-263e78414ae85d7bcd52a025a94131d0a279872a45ed632b9165336dfdcd4443"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
+           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.4-py3.10-tf2.17-runtime/images/sha256-6f43de12f7eb202791b698ac51d28b72098de90034dbcd48486629b0125f7707"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
 
-      - `tensorflow-rocm 2.16.2 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.3/tensorflow_rocm-2.16.2-cp310-cp310-manylinux_2_28_x86_64.whl>`__
+      - `tensorflow-rocm 2.17.1 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.4/tensorflow_rocm-2.17.1-cp310-cp310-manylinux_2_28_x86_64.whl>`__
-      - dev
+      - runtime
-      - `Python 3.10 <https://www.python.org/downloads/release/python-31012/>`_
+      - 22.04
-      - `TensorBoard 2.16.2 <https://github.com/tensorflow/tensorboard/tree/2.16.2>`_
+      - `Python 3.10.16 <https://www.python.org/downloads/release/python-31016/>`_
+      - `TensorBoard 2.17.1 <https://github.com/tensorflow/tensorboard/tree/2.17.1>`_
 
-    * - .. raw:: html
-
-           <a href="https://hub.docker.com/layers/rocm/tensorflow/rocm6.3.1-py3.10-tf2.15.0-dev/images/sha256-479046a8477ca701a9494a813ab17e8ab4f6baa54641e65dc8d07629f1e6a880"><i class="fab fa-docker fa-lg"></i> rocm/tensorflow</a>
-
-      - `tensorflow-rocm 2.15.1 <https://repo.radeon.com/rocm/manylinux/rocm-rel-6.3/tensorflow_rocm-2.15.1-cp310-cp310-manylinux_2_28_x86_64.whl>`_
-      - dev
-      - `Python 3.10 <https://www.python.org/downloads/release/python-31012/>`_
-      - `TensorBoard 2.15.2 <https://github.com/tensorflow/tensorboard/tree/2.15.2>`_
 
 Critical ROCm libraries for TensorFlow
 ===============================================================================
@@ -117,7 +156,8 @@ Critical ROCm libraries for TensorFlow
 TensorFlow depends on multiple components and the supported features of those
 components can affect the TensorFlow ROCm supported feature set. The versions
 in the following table refer to the first TensorFlow version where the ROCm
-library was introduced as a dependency.
+library was introduced as a dependency. The versions described
+are available in ROCm :version:`rocm_version`.
 
 .. list-table::
     :widths: 25, 10, 35, 30
@@ -128,43 +168,43 @@ library was introduced as a dependency.
       - Purpose
       - Used in
     * - `hipBLAS <https://github.com/ROCm/hipBLAS>`_
-      - 2.3.0
+      - :version-ref:`hipBLAS rocm_version`
       - Provides GPU-accelerated Basic Linear Algebra Subprograms (BLAS) for
         matrix and vector operations.
       - Accelerates operations like ``tf.matmul``, ``tf.linalg.matmul``, and
         other matrix multiplications commonly used in neural network layers.
     * - `hipBLASLt <https://github.com/ROCm/hipBLASLt>`_
-      - 0.10.0
+      - :version-ref:`hipBLASLt rocm_version`
       - Extends hipBLAS with additional optimizations like fused kernels and
         integer tensor cores.
       - Optimizes matrix multiplications and linear algebra operations used in
         layers like dense, convolutional, and RNNs in TensorFlow.
     * - `hipCUB <https://github.com/ROCm/hipCUB>`_
-      - 3.3.0
+      - :version-ref:`hipCUB rocm_version`
       - Provides a C++ template library for parallel algorithms for reduction,
         scan, sort and select.
       - Supports operations like ``tf.reduce_sum``, ``tf.cumsum``, ``tf.sort``
         and other tensor operations in TensorFlow, especially those involving
         scanning, sorting, and filtering.
     * - `hipFFT <https://github.com/ROCm/hipFFT>`_
-      - 1.0.17
+      - :version-ref:`hipFFT rocm_version`
       - Accelerates Fast Fourier Transforms (FFT) for signal processing tasks.
       - Used for operations like signal processing, image filtering, and
         certain types of neural networks requiring FFT-based transformations.
     * - `hipSOLVER <https://github.com/ROCm/hipSOLVER>`_
-      - 2.3.0
+      - :version-ref:`hipSOLVER rocm_version`
       - Provides GPU-accelerated direct linear solvers for dense and sparse
         systems.
       - Optimizes linear algebra functions such as solving systems of linear
         equations, often used in optimization and training tasks.
     * - `hipSPARSE <https://github.com/ROCm/hipSPARSE>`_
-      - 3.1.2
+      - :version-ref:`hipSPARSE rocm_version`
       - Optimizes sparse matrix operations for efficient computations on sparse
         data.
       - Accelerates sparse matrix operations in models with sparse weight
         matrices or activations, commonly used in neural networks.
     * - `MIOpen <https://github.com/ROCm/MIOpen>`_
-      - 3.3.0
+      - :version-ref:`MIOpen rocm_version`
       - Provides optimized deep learning primitives such as convolutions,
         pooling,
         normalization, and activation functions.
@@ -172,13 +212,13 @@ library was introduced as a dependency.
         in TensorFlow for layers like ``tf.nn.conv2d``, ``tf.nn.relu``, and
         ``tf.nn.lstm_cell``.
     * - `RCCL <https://github.com/ROCm/rccl>`_
-      - 2.21.5
+      - :version-ref:`RCCL rocm_version`
       - Optimizes for multi-GPU communication for operations like AllReduce and
         Broadcast.
       - Distributed data parallel training (``tf.distribute.MirroredStrategy``).
         Handles communication in multi-GPU setups.
     * - `rocThrust <https://github.com/ROCm/rocThrust>`_
-      - 3.3.0
+      - :version-ref:`rocThrust rocm_version`
       - Provides a C++ template library for parallel algorithms like sorting,
         reduction, and scanning.
       - Reduction operations like ``tf.reduce_sum``, ``tf.cumsum`` for computing