Fix white paper links

* Corrected typo

Corrected typo in line 119 prerequisities -> prerequisites

* Corrected typo in README.md

Corrected typo in line 119 prerequisities -> prerequisites

.azuredevops/README.md

@@ -23,8 +23,40 @@ ROCm-CI Azure DevOps Pipelines contains markup language files that orchestrate b
 - [YAML schema](https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/?view=azure-pipelines&viewFallbackFrom=azure-devops)
 - [Azure Pipelines Task Index](https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/?view=azure-pipelines)
 
+## VMSS Setup
+
+The Azure VMSS used for build jobs have docker installed during provisioning through the Azure Portal's settings.
+Select the VMSS, then go to Settings, Operating system.
+In the Custom Data section, select to Modify Custom Data. Enter the code block below and Apply.
+
+```bash
+#cloud-config
+
+bootcmd:
+  - mkdir -p /etc/systemd/system/walinuxagent.service.d
+  - echo "[Unit]\nAfter=cloud-final.service" > /etc/systemd/system/walinuxagent.service.d/override.conf
+  - sed "s/After=multi-user.target//g" /lib/systemd/system/cloud-final.service > /etc/systemd/system/cloud-final.service
+  - systemctl daemon-reload
+
+apt:
+  sources:
+    docker.list:
+      source: deb [arch=amd64] https://download.docker.com/linux/ubuntu $RELEASE stable
+      keyid: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
+
+packages:
+  - docker-ce
+  - docker-ce-cli
+
+groups:
+  - docker
+
+runcmd:
+  - usermod -aG docker $USER
+  - systemctl restart docker
+  - systemctl enable docker
+```
+
 ## Disclaimer
 
 The information presented in this document is for informational purposes only and may contain technical inaccuracies, omissions, and typographical errors. The information contained herein is subject to change and may be rendered inaccurate for many reasons, including but not limited to product and roadmap changes, component and motherboard versionchanges, new model and/or product releases, product differences between differing manufacturers, software changes, BIOS flashes, firmware upgrades, or the like. Any computer system has risks of security vulnerabilities that cannot be completely prevented or mitigated.AMD assumes no obligation to update or otherwise correct or revise this information. However, AMD reserves the right to revise this information and to make changes from time to time to the content hereof without obligation of AMD to notify any person of such revisions or changes.THIS INFORMATION IS PROVIDED ‘AS IS.” AMD MAKES NO REPRESENTATIONS OR WARRANTIES WITH RESPECT TO THE CONTENTS HEREOF AND ASSUMES NO RESPONSIBILITY FOR ANY INACCURACIES, ERRORS, OR OMISSIONS THAT MAY APPEAR IN THIS INFORMATION. AMD SPECIFICALLY DISCLAIMS ANY IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR ANY PARTICULAR PURPOSE. IN NO EVENT WILL AMD BE LIABLE TO ANY PERSON FOR ANY RELIANCE, DIRECT, INDIRECT, SPECIAL, OR OTHER CONSEQUENTIAL DAMAGES ARISING FROM THE USE OF ANY INFORMATION CONTAINED HEREIN, EVEN IF AMD IS EXPRESSLY ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. AMD, the AMD Arrow logo, and combinations thereof are trademarks of Advanced Micro Devices, Inc. Other product names used in this publication are for identification purposes only and may be trademarks of their respective companies.
-
-© 2024 Advanced Micro Devices, Inc. All Rights Reserved.

.azuredevops/ci-builds/aomp-mainline.yml

@@ -0,0 +1,42 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+resources:
+  repositories:
+  - repository: aomp_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp
+    ref: amd-mainline-open
+  - repository: aomp-extras_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp-extras
+    ref: amd-mainline-open
+  - repository: flang_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/flang
+    ref: amd-mainline-open
+  - repository: llvm-project_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/llvm-project
+    ref: amd-mainline-open
+  pipelines:
+  - pipeline: rocr-runtime_pipeline
+    source: \ROCR-Runtime
+    trigger:
+      branches:
+        include:
+        - amd-master
+# this job will only be triggered after successful build sequence of llvm-project and ROCR-Runtime
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/aomp.yml
+    parameters:
+      checkoutRepo: aomp_repo

.azuredevops/ci-builds/aomp-staging.yml

@@ -8,17 +8,17 @@ resources:
     type: github
     endpoint: ROCm
     name: ROCm/aomp
-    ref: aomp-dev
+    ref: amd-staging
   - repository: aomp-extras_repo
     type: github
     endpoint: ROCm
     name: ROCm/aomp-extras
-    ref: aomp-dev
+    ref: amd-staging
   - repository: flang_repo
     type: github
     endpoint: ROCm
     name: ROCm/flang
-    ref: aomp-dev
+    ref: amd-staging
   - repository: llvm-project_repo
     type: github
     endpoint: ROCm
@@ -27,7 +27,10 @@ resources:
   pipelines:
   - pipeline: rocr-runtime_pipeline
     source: \ROCR-Runtime
-    trigger: true
+    trigger:
+      branches:
+        include:
+        - amd-staging
 # this job will only be triggered after successful build sequence of llvm-project and ROCR-Runtime
 
 trigger: none

.azuredevops/components/AMDMIGraphX.yml

@@ -8,30 +8,31 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - git
     - cmake
-    - ninja-build
+    - git
+    - libdnnl-dev
     - libdrm-dev
-    - libnuma-dev
-    - python3-pip
-    - python3-venv
-    - libtbb-dev
-    - nlohmann-json3-dev
     - libmsgpack-dev
-    - libsqlite3-dev
+    - libnuma-dev
     - libprotobuf-dev
+    - libsqlite3-dev
+    - libtbb-dev
+    - ninja-build
+    - nlohmann-json3-dev
     - protobuf-compiler
+    - python3-pip
     - python3-pybind11
+    - python3-venv
 - name: pipModules
   type: object
   default:
     - https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
-    - onnx==1.14.1
-    - numpy==1.21.6
-    - typing==3.7.4
-    - pytest==6.0.1
-    - packaging==23.0
-    - protobuf==3.20.2
+    - onnx>=1.14.1
+    - numpy>=1.21.6
+    - typing>=3.7.4
+    - pytest>=6.0.1
+    - packaging>=23.0
+    - protobuf>=3.20.2
 - name: rocmDependencies
   type: object
   default:
@@ -50,6 +51,26 @@ parameters:
     - rocminfo
     - rocMLIR
     - rocprofiler-register
+    - roctracer
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - aomp-extras
+    - clr
+    - composable_kernel
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - MIOpen
+    - rocm-cmake
+    - ROCR-Runtime
+    - rocBLAS
+    - rocminfo
+    - rocMLIR
+    - rocprofiler-register
+    - roctracer
 
 jobs:
 - job: AMDMIGraphX
@@ -75,37 +96,102 @@ jobs:
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
 # half version should be fixed to 5.6.0
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      buildType: specific
+      definitionId: ${{ variables.HALF560_PIPELINE_ID }}
+      buildId: ${{ variables.HALF560_BUILD_ID }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
-      dependencySource: fixed
-      fixedComponentName: half
-      fixedPipelineIdentifier: ${{ variables.HALF560_PIPELINE_ID }}
-      skipLibraryLinking: true
-      skipLlvmSymlink: true
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
         -DCMAKE_BUILD_TYPE=Release
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
         -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
-        -DMIGRAPHX_USE_COMPOSABLEKERNEL=OFF
         -DBUILD_TESTING=ON
         -GNinja
-# REFERENCE: https://github.com/ROCm/composable_kernel/issues/782
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: AMDMIGraphX_testing
+  dependsOn: AMDMIGraphX
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: MIGRAPHX_TRACE_BENCHMARKING
+    value: 1
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+# half version should be fixed to 5.6.0
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      buildType: specific
+      definitionId: ${{ variables.HALF560_PIPELINE_ID }}
+      buildId: ${{ variables.HALF560_BUILD_ID }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - task: CMake@1
+    displayName: MIGraphXTest CMake Flags
+    inputs:
+      cmakeArgs: >-
+        -DCMAKE_BUILD_TYPE=Release
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
+        -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
+        -DBUILD_TESTING=ON
+        -DMIGRAPHX_ENABLE_C_API_TEST=ON
+        ..
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: AMDMIGraphX
+      testExecutable: make
+      testParameters: -j$(nproc) check
+      testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - MIGRAPHX_TRACE_BENCHMARKING:::1

.azuredevops/components/HIP.yml

@@ -8,11 +8,13 @@ parameters:
 - name: aptPackages
   type: object
   default:
+    - cmake
     - libnuma-dev
     - mesa-common-dev
     - ocl-icd-libopencl1
     - ocl-icd-opencl-dev
     - opencl-headers
+    - python3-pip
 - name: pipModules
   type: object
   default:
@@ -39,8 +41,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -62,13 +63,8 @@ jobs:
       checkoutRepo: hipother_repo
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependenciesAMD }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
 # compile clr
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -82,17 +78,25 @@ jobs:
         -DHIPCC_BIN_DIR=$(Agent.BuildDirectory)/rocm/bin
         -DCLR_BUILD_HIP=ON
         -DCLR_BUILD_OCL=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      artifactName: amd
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: amd
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: amd
 
 # HIP with Nvidia backend
 - job: hip_clr_combined_nvidia
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -115,13 +119,8 @@ jobs:
       checkoutRepo: hipother_repo
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependenciesNvidia }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - script: 'ls -1R $(Agent.BuildDirectory)/rocm'
     displayName: 'Artifact listing'
 # compile clr
@@ -139,3 +138,9 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: nvidia
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: nvidia

.azuredevops/components/HIPIFY.yml

@@ -37,9 +37,9 @@ jobs:
     inputs:
       targetType: inline
       script: |
-        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
-        wget -q -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-archive-keyring.gpg | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/cuda-archive-keyring.gpg > /dev/null
-        echo "deb [signed-by=/etc/apt/trusted.gpg.d/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" | sudo tee /etc/apt/sources.list.d/cuda-ubuntu2204-x86_64.list
+        wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+        sudo dpkg -i cuda-keyring_1.1-1_all.deb
+        sudo rm -f cuda-keyring_1.1-1_all.deb
         sudo apt update
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -97,6 +97,7 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Pipeline.Workspace)/llvm;/usr/local/cuda/targets/x86_64-linux/lib
         -DLLVM_EXTERNAL_LIT=$(Pipeline.Workspace)/llvm-project/llvm/build/bin/llvm-lit
       multithreadFlag: -- -j32
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
@@ -105,3 +106,14 @@ jobs:
       testExecutable: make
       testParameters: test-hipify
       testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: combined
+      registerCUDAPackages: true
+      extraCopyDirectories:
+        - llvm-project
+      extraEnvVars:
+        - UPSTREAM_LLVM_GIT_URL:::https://github.com/llvm/llvm-project.git
+        - UPSTREAM_LLVM_TAG:::llvmorg-18.1.2

.azuredevops/components/MIOpen.yml

@@ -8,26 +8,29 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - software-properties-common
-    - python3-pip
     - cmake
-    - ninja-build
-    - libsqlite3-dev
-    - libbz2-dev
-    - nlohmann-json3-dev
-    - libgtest-dev
+    - jq
     - libdrm-dev
+    - libsqlite3-dev
+    - libstdc++-12-dev
+    - ninja-build
+    - python3-pip
+    - python3-venv
+    - software-properties-common
+    - zip
+- name: pipModules
+  type: object
+  default:
+    - cget
 - name: rocmDependencies
   type: object
   default:
-    - rocMLIR
+    - half
     - rocRAND
     - rocBLAS
     - hipBLAS
     - hipBLASLt
     - hipBLAS-common
-    - half
-    - composable_kernel
     - rocm-cmake
     - llvm-project
     - ROCR-Runtime
@@ -35,6 +38,22 @@ parameters:
     - clr
     - rocminfo
     - roctracer
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - half
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler-register
+    - rocRAND
+    - roctracer
 
 jobs:
 - job: MIOpen
@@ -54,43 +73,135 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
-  # The default boost library from apt is 1.74, which does not satisfy MIOpen's build requirement (1.79+)
-  # Upgrade boost from apt by following https://launchpad.net/~mhier/+archive/ubuntu/libboost-latest
-  - task: Bash@3
-    displayName: 'update boost version'
-    inputs:
-      targetType: inline
-      script: sudo add-apt-repository ppa:mhier/libboost-latest -y
-  - task: Bash@3
-    displayName: 'install boost'
-    inputs:
-      targetType: inline
-      script: sudo apt-get --yes install libboost1.83-dev libboost-system1.83-dev libboost-filesystem1.83-dev
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Build and install other dependencies
+    inputs:
+      targetType: inline
+      workingDirectory: $(Build.SourcesDirectory)
+      script: |
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        sed -i '/composable_kernel/d' requirements.txt
+        mkdir -p $(Agent.BuildDirectory)/miopen-deps
+        cmake -P install_deps.cmake --prefix $(Agent.BuildDirectory)/miopen-deps
+        sudo rm -rf /opt/rocm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DMIOPEN_BACKEND=HIP
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/miopen-deps
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -DMIOPEN_ENABLE_AI_KERNEL_TUNING=OFF
         -DMIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK=OFF
         -DCMAKE_BUILD_TYPE=Release
         -DBUILD_TESTING=ON
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraCopyDirectories:
+        - miopen-deps
+
+- job: MIOpen_testing
+  timeoutInMinutes: 180
+  dependsOn: MIOpen
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: ROCM_PATH
+    value: $(Agent.BuildDirectory)/rocm
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - task: Bash@3
+    displayName: Build and install other dependencies
+    inputs:
+      targetType: inline
+      workingDirectory: $(Build.SourcesDirectory)
+      script: |
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        sed -i '/composable_kernel/d' requirements.txt
+        mkdir -p $(Agent.BuildDirectory)/miopen-deps
+        cmake -P install_deps.cmake --prefix $(Agent.BuildDirectory)/miopen-deps
+        sudo rm -rf /opt/rocm
+  - task: CMake@1
+    displayName: 'MIOpen Test CMake Flags'
+    inputs:
+      cmakeArgs: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Build.SourcesDirectory)/bin;$(Agent.BuildDirectory)/miopen-deps
+        -DCMAKE_INSTALL_PREFIX=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DMIOPEN_BACKEND=HIP
+        -DMIOPEN_TEST_FLAGS=" --disable-verification-cache"
+        -DCMAKE_BUILD_TYPE=release
+        -DBUILD_DEV=OFF
+        -DMIOPEN_USE_MLIR=ON
+        -DMIOPEN_GPU_SYNC=OFF
+        ..
+  - task: Bash@3
+    displayName: 'MIOpen Test Build'
+    inputs:
+      targetType: inline
+      script: |
+        cmake --build . --target tests -- -j$(nproc)
+      workingDirectory: $(Build.SourcesDirectory)/build
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: MIOpen
+      testParameters: '-VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml --exclude-regex test_rnn_seq_api'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraCopyDirectories:
+        - miopen-deps

.azuredevops/components/MIVisionX.yml

@@ -10,39 +10,31 @@ parameters:
   default:
     - cmake
     - ninja-build
-    - git
     - wget
     - unzip
+    - libstdc++-12-dev
     - pkg-config
-    - protobuf-compiler
-    - libprotoc-dev
     - ffmpeg
     - libavcodec-dev
     - libavformat-dev
     - libavutil-dev
     - libswscale-dev
-    - build-essential
-    - libgtk2.0-dev
-    - libavcodec-dev
-    - libavformat-dev
-    - libswscale-dev
-    - libtbb2
-    - libtbb-dev
-    - libjpeg-dev
-    - libpng-dev
-    - libtiff-dev
-    - libdc1394-dev
     - libgmp-dev
+    - libomp-dev
     - libopencv-dev
+    - protobuf-compiler
+    - libprotoc-dev
+    - python3-pip
 - name: pipModules
   type: object
   default:
-    - future==0.18.2
+    - future==1.0.0
     - pytz==2022.1
-    - numpy==1.21
+    - numpy==1.23
     - google==3.0.0
     - protobuf==3.12.4
     - onnx==1.12.0
+    - nnef==1.0.7
 - name: rocmDependencies
   type: object
   default:
@@ -78,8 +70,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -99,14 +90,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -114,13 +100,22 @@ jobs:
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DROCM_DEP_ROCMCORE=ON
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
 
 - job: MIVisionX_testing
   dependsOn: MIVisionX
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -141,19 +136,11 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   # anything in /opt may be persistent across runs
   # so we need to remove the symlink if it already exists
   - script: |
@@ -162,7 +149,15 @@ jobs:
       mkdir mivisionx-tests
       cd mivisionx-tests
       cmake /opt/rocm/share/mivisionx/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: MIVisionX
       testDir: 'mivisionx-tests'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      optSymLink: true

.azuredevops/components/ROCR-Runtime.yml

@@ -8,23 +8,31 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - libelf-dev
+    - cmake
     - g++
     - libdrm-dev
+    - libelf-dev
     - libnuma-dev
+    - pkg-config
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
     - llvm-project
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - llvm-project
+    - rocm_smi_lib
+    - rocprofiler-register
 
 jobs:
 - job: ROCR_Runtime
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -37,16 +45,102 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_SHARED_LIBS=ON
+        -DCMAKE_BUILD_TYPE=Release
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+
+- job: ROCR_Runtime_testing
+  dependsOn: ROCR_Runtime
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - task: Bash@3
+    displayName: Install libhwloc5
+    inputs:
+      targetType: 'inline'
+      script: |
+        wget http://ftp.us.debian.org/debian/pool/main/h/hwloc/libhwloc5_1.11.12-3_amd64.deb
+        wget http://ftp.us.debian.org/debian/pool/main/h/hwloc/libhwloc-dev_1.11.12-3_amd64.deb
+        sudo apt install -y --allow-downgrades ./libhwloc5_1.11.12-3_amd64.deb ./libhwloc-dev_1.11.12-3_amd64.deb
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
+  - task: Bash@3
+    displayName: Build kfdtest
+    continueOnError: true
+    inputs:
+      targetType: 'inline'
+      workingDirectory: $(Build.SourcesDirectory)/libhsakmt/tests/kfdtest
+      script: |
+        mkdir build && cd build
+        cmake -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm ..
+        make
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: kfdtest
+      testExecutable: BIN_DIR=$(Build.SourcesDirectory)/libhsakmt/tests/kfdtest/build ./run_kfdtest.sh
+      testParameters: '-p core --gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: $(Build.SourcesDirectory)/libhsakmt/tests/kfdtest/scripts
+  - task: Bash@3
+    displayName: Build rocrtst
+    continueOnError: true
+    inputs:
+      targetType: 'inline'
+      workingDirectory: $(Build.SourcesDirectory)/rocrtst/suites/test_common
+      script: |
+        mkdir build && cd build
+        cmake .. \
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm \
+          -DTARGET_DEVICES=$(JOB_GPU_TARGET) \
+          -DROCM_DIR=$(Agent.BuildDirectory)/rocm \
+          -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm/bin \
+          -DOPENCL_INC_DIR=$(Agent.BuildDirectory)/rocm/llvm/lib/clang/21/include
+        make
+        make rocrtst_kernels
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocrtst
+      testExecutable: ./rocrtst64
+      testParameters: '--gtest_filter="-rocrtstNeg.Memory_Negative_Tests:rocrtstFunc.Memory_Max_Mem" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: $(Build.SourcesDirectory)/rocrtst/suites/test_common/build/$(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+# docker image will be missing libhwloc5

.azuredevops/components/ROCT-Thunk-Interface.yml

@@ -16,8 +16,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -29,4 +28,5 @@ jobs:
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/ROCdbgapi.yml

@@ -10,6 +10,7 @@ parameters:
   default:
     - cmake
     - ninja-build
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
@@ -23,8 +24,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -37,17 +37,17 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_BUILD_TYPE=Release
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/ROCgdb.yml

@@ -5,24 +5,47 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# reference: https://github.com/ROCm/ROCgdb/blob/amd-staging/README-ROCM.md
 - name: aptPackages
   type: object
   default:
-    - libgmp-dev
-    - libmpfr-dev
-    - texinfo
     - bison
+    - dejagnu
     - flex
+    - libbabeltrace-dev
+    - libexpat-dev
+    - libgmp-dev
+    - liblzma-dev
+    - libmpfr-dev
+    - ncurses-dev
+    - texinfo
+    - zlib1g-dev
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - ROCdbgapi
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
-- job: rocgdb
+- job: ROCgdb
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  - name: PKG_CONFIG_PATH
+    value: $(Agent.BuildDirectory)/rocm/share/pkgconfig
+  pool: $(JOB_TEST_POOL)
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -31,5 +54,63 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-autotools.yml
+    parameters:
+      configureFlags: >-
+        --program-prefix=roc
+        --enable-64-bit-bfd
+        --enable-targets="x86_64-linux-gnu,amdgcn-amd-amdhsa"
+        --disable-ld
+        --disable-gas
+        --disable-gdbserver
+        --disable-sim
+        --enable-tui
+        --disable-gdbtk
+        --disable-shared
+        --disable-gprofng
+        --with-expat
+        --with-system-zlib
+        --without-guile
+        --with-babeltrace
+        --with-lzma
+        --with-python=python3
+        --with-rocm-dbgapi=$(Agent.BuildDirectory)/rocm
+        LDFLAGS="-Wl,--enable-new-dtags,-rpath=$(Agent.BuildDirectory)/rocm/lib"
+      makeCallPrefix: LD_RUN_PATH='${ORIGIN}/../lib'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        echo "##vso[task.prependpath]/opt/rocm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - task: Bash@3
+    displayName: check-gdb
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: make check-gdb TESTS=gdb.rocm/simple.exp
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: print gdb log
+    inputs:
+      targetType: inline
+      script: find -name gdb.log -exec cat {} \;
+      workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: combined
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - PKG_CONFIG_PATH:::/home/user/workspace/rocm/share/pkgconfig

.azuredevops/components/ROCmValidationSuite.yml

@@ -10,11 +10,14 @@ parameters:
   default:
     - cmake
     - ninja-build
+    - libdrm-dev
     - libyaml-cpp-dev
     - libpci-dev
     - libpci3
+    - libgst-dev
     - libgtest-dev
     - git
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
@@ -40,9 +43,11 @@ parameters:
     - llvm-project
     - rocBLAS
     - rocm_smi_lib
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
     - rocRAND
+    - roctracer
 
 jobs:
 - job: ROCmValidationSuite
@@ -55,8 +60,7 @@ jobs:
     value: $(Agent.BuildDirectory)/rocm
   - name: HIP_INC_DIR
     value: $(Agent.BuildDirectory)/rocm
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -75,14 +79,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -91,13 +90,25 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCPACK_PACKAGING_INSTALL_PREFIX=$(Build.BinariesDirectory)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+        - ROCM_PATH:::/home/user/workspace/rocm
+        - HIP_INC_DIR:::/home/user/workspace/rocm
 
 - job: ROCmValidationSuite_testing
   dependsOn: ROCmValidationSuite
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -118,19 +129,12 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: ROCmValidationSuite
@@ -138,3 +142,8 @@ jobs:
       testParameters: ''
       testDir: $(Agent.BuildDirectory)
       testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/Tensile.yml

@@ -0,0 +1,189 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - python3-pip
+    - cmake
+    - libmsgpack-dev
+    - libboost-program-options-dev
+- name: pipModules
+  type: object
+  default:
+    - tox
+    - pytest
+- name: rocmDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - rocm_smi_lib
+    - rocprofiler-register
+    - ROCR-Runtime
+
+jobs:
+- job: Tensile
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.LOW_BUILD_POOL }}
+  workspace:
+    clean: all
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+  - task: Bash@3
+    displayName: Create wheel file
+    inputs:
+      targetType: inline
+      script: python3 setup.py bdist_wheel
+      workingDirectory: $(Build.SourcesDirectory)
+    env:
+      ROCM_PATH: $(Agent.BuildDirectory)/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.SourcesDirectory)/dist
+      contentsString: '*.whl'
+      targetDir: $(Build.ArtifactStagingDirectory)
+      clean: false
+  - task: PublishPipelineArtifact@1
+    displayName: 'wheel file Publish'
+    retryCountOnTaskFailure: 3
+    inputs:
+      targetPath: $(Build.ArtifactStagingDirectory)
+  - task: Bash@3
+    displayName: Save pipeline artifact file names
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: |
+        echo "$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_drop_$(JOB_GPU_TARGET).tar.gz" >> pipelineArtifacts.txt
+        whlFile=$(find "$(Build.ArtifactStagingDirectory)" -type f -name "*.whl" | head -n 1)
+        if [ -n "$whlFile" ]; then
+          echo $(basename "$whlFile") >> pipelineArtifacts.txt
+        fi
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+
+- job: Tensile_testing
+  timeoutInMinutes: 90
+  dependsOn: Tensile
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Wheel Files'
+    inputs:
+      itemPattern: '**/*.whl'
+      targetPath: $(Agent.BuildDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - task: Bash@3
+    displayName: pip install
+    inputs:
+      targetType: inline
+      script: find -name *.whl -exec pip install {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: Add Python site-packages binaries to path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.prependpath]$USER_BASE/bin"
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - task: Bash@3
+    displayName: tox test
+    inputs:
+      targetType: inline
+      script: tox run -v -e ci -- -m pre_checkin
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Remove Python site-packages binaries from path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$USER_BASE/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      optSymLink: true
+      pythonEnvVars: true
+      extraPaths: /home/user/workspace/rocm/llvm/bin:/home/user/workspace/rocm/bin
+# docker image will not have python site-packages in path, but the env vars will make it easier

.azuredevops/components/TransferBench.yml

@@ -0,0 +1,125 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - libnuma-dev
+    - ninja-build
+    - python3-pip
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+
+jobs:
+- job: TransferBench
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.LOW_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: TransferBench_testing
+  dependsOn: TransferBench
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: TransferBench All-to-all
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/bin/TransferBench'
+      testParameters: 'a2a'
+      testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: TransferBench Peer-to-peer
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/bin/TransferBench'
+      testParameters: 'p2p'
+      testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/amdsmi.yml

@@ -8,15 +8,17 @@ parameters:
 - name: aptPackages
   type: object
   default:
+    - cmake
     - libdrm-dev
+    - python3-pip
+    - pkg-config
 
 jobs:
 - job: amdsmi
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -31,11 +33,16 @@ jobs:
     parameters:
       extraBuildFlags: >-
         -DBUILD_TESTS=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
 
 - job: amdsmi_testing
   dependsOn: amdsmi
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -53,9 +60,17 @@ jobs:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: amdsmi
       testDir: '$(Agent.BuildDirectory)'
       testExecutable: './rocm/share/amd_smi/tests/amdsmitst'
       testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/aomp-extras.yml

@@ -23,8 +23,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -37,13 +36,8 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: aomp-extras
@@ -54,4 +48,9 @@ jobs:
         -DAOMP_VERSION_STRING=$(LATEST_RELEASE_TAG)
         -GNinja
       installDir: $(Build.BinariesDirectory)/llvm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/aomp.yml

@@ -42,6 +42,7 @@ parameters:
     - python3-setuptools
     - python3-dev
     - libudev-dev
+    - parallel
   # Referencing comment snippet below but excluding rocprofiler and roctracer.
   # This is to remove need for separate build per gpu target.
   # With our selected build flags, compilation and installation work fine without these two.
@@ -55,16 +56,18 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
+    - amdsmi
     - clr
-    - rocminfo
-    - rocprofiler-register
+    - llvm-project
     - ROCdbgapi
     - ROCgdb
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
     - rocm_smi_lib
-    - amdsmi
+    - rocprofiler-register
+    - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: aomp
@@ -94,23 +97,19 @@ jobs:
       checkoutRepo: llvm-project_repo
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
 # Because clang is not being rebuilt and to separate downloaded ROCm
 # dependencies from the new artifacts, we use temporary symbolic links
 # for the compilation and installation to go through.
   - script: |
       sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-      mkdir -p $(Build.BinariesDirectory)/bin
-      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang $(Build.BinariesDirectory)/bin/clang
-      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang++ $(Build.BinariesDirectory)/bin/clang++
-      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/llvm-config $(Build.BinariesDirectory)/bin/llvm-config
+      mkdir -p $(Build.BinariesDirectory)/lib/llvm/bin
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang $(Build.BinariesDirectory)/lib/llvm/bin/clang
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang++ $(Build.BinariesDirectory)/lib/llvm/bin/clang++
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/llvm-config $(Build.BinariesDirectory)/lib/llvm/bin/llvm-config
       ln -s $(Agent.BuildDirectory)/rocm/llvm $(Build.BinariesDirectory)/llvm
+      ls -1R $(Build.BinariesDirectory)
     displayName: Extra build environment setup
 # We follow the sequence described in the aomp repo instructions
 # https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL.md
@@ -123,7 +122,7 @@ jobs:
 # method leads to a giant build log compared to separate logs per script call.
 #
 # Components compiled and the order for non-standalone build found at
-# https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L135-L143
+# https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L135-L142
   - task: Bash@3
     displayName: Build Prereq
     inputs:
@@ -173,7 +172,6 @@ jobs:
       AOMP_USE_NINJA: 1
       ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
@@ -193,12 +191,11 @@ jobs:
       AOMP_USE_NINJA: 1
       ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
       INSTALL_OPENMP: $(Build.BinariesDirectory)
-  - script: ln -s $(Build.BinariesDirectory)/include/omp.h $(Build.SourcesDirectory)/llvm-project/llvm/include/omp.h
+  - script: ln -s $(Build.BinariesDirectory)/lib/llvm/include/omp.h $(Build.SourcesDirectory)/llvm-project/llvm/include/omp.h
     displayName: Link omp header
   - task: Bash@3
     displayName: Build offload
@@ -234,15 +231,14 @@ jobs:
       AOMP_USE_NINJA: 1
       ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
       INSTALL_OPENMP: $(Build.BinariesDirectory)
   - task: Bash@3
-    displayName: Build flang-legacy
+    displayName: Build llvm-classic
     inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-legacy.sh
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_llvm-classic.sh
     env:
       AOMP_REPOS: $(Build.SourcesDirectory)
       AOMP_PROJECT_REPO_NAME: llvm-project
@@ -251,14 +247,13 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
   - task: Bash@3
-    displayName: Install flang-legacy
+    displayName: Install llvm-classic
     inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-legacy.sh
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_llvm-classic.sh
       arguments: install
     env:
       AOMP_REPOS: $(Build.SourcesDirectory)
@@ -272,6 +267,37 @@ jobs:
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build flang-classic
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-classic.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install flang-classic
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-classic.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Build pgmath
     inputs:
@@ -284,11 +310,9 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Install pgmath
     inputs:
@@ -302,11 +326,9 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Build flang
     inputs:
@@ -319,11 +341,9 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Install flang
     inputs:
@@ -337,11 +357,9 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Build flang_runtime
     inputs:
@@ -354,11 +372,9 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Install flang_runtime
     inputs:
@@ -372,24 +388,36 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
 # Clean up build environment before publish artifact
   - script: |
-      rm $(Build.BinariesDirectory)/bin/clang
-      rm $(Build.BinariesDirectory)/bin/clang++
-      rm $(Build.BinariesDirectory)/bin/llvm-config
+      rm $(Build.BinariesDirectory)/lib/llvm/bin/clang
+      rm $(Build.BinariesDirectory)/lib/llvm/bin/clang++
+      rm $(Build.BinariesDirectory)/lib/llvm/bin/llvm-config
+      rm $(Build.BinariesDirectory)/lib/llvm/bin/flang
       rm $(Build.BinariesDirectory)/llvm
     displayName: Remove temporary symbolic links
+# aomp scripts changed where files get installed in scripts, copy to expected location
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.BinariesDirectory)/lib/llvm
+      targetDir: $(Build.ArtifactStagingDirectory)
+# Remove temporary directory used to deal with expected paths of scripts
+  - script: |
+      rm -rf $(Build.BinariesDirectory)/lib/llvm
+    displayName: Remove temporary directories
 # Copy the files to artifact staging temporarily to clean up binaries directory
 # and then copy files back to llvm subdirectory in the cleaned up binaries directory
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
       sourceDir: $(Build.BinariesDirectory)
       targetDir: $(Build.ArtifactStagingDirectory)
+      clean: false
+  - script: |
+      ln -s $(Build.ArtifactStagingDirectory)/bin/flang-classic $(Build.ArtifactStagingDirectory)/bin/flang
+    displayName: Recreate flang symlink
   - task: DeleteFiles@1
     displayName: 'Cleanup Binaries Directory'
     inputs:
@@ -406,4 +434,90 @@ jobs:
       SourceFolder: $(Build.ArtifactStagingDirectory)
       Contents: '/**/*'
       RemoveDotFiles: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      optSymLink: true
+
+- job: aomp_testing
+  dependsOn: aomp
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: aomp-extras_repo
+# these copy steps are from the aomp prototype script for test prep
+  - task: CopyFiles@2
+    displayName: 'Copy AOMP contents'
+    inputs:
+      CleanTargetFolder: false
+      SourceFolder: $(Build.SourcesDirectory)/aomp
+      Contents: |
+        **
+        !**/.git/**
+        !**/.github/**
+        !**/.gitignore
+      TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests
+      retryCount: 3
+  - task: CopyFiles@2
+    displayName: 'Copy FileCheck'
+    inputs:
+      CleanTargetFolder: false
+      SourceFolder: $(Agent.BuildDirectory)/rocm/llvm/bin
+      Contents: FileCheck
+      TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests/bin
+      retryCount: 3
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - task: Bash@3
+    displayName: Test AOMP
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: ./run_rocm_test.sh
+      workingDirectory: $(Build.SourcesDirectory)/aomp/bin
+    env:
+      AOMP: $(Agent.BuildDirectory)/rocm/llvm
+      AOMP_REPOS_TEST: $(Build.SourcesDirectory)/aomp-test
+      AOMP_TEST_DIR: $(Build.SourcesDirectory)/aomp-test
+      SKIP_TEST_PACKAGE: 1
+      MAINLINE_BUILD: 1
+      SUITE_LIST: smoke
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      optSymLink: true

.azuredevops/components/composable_kernel.yml

@@ -60,14 +60,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - script: |
       mkdir -p $(CCACHE_DIR)
       echo "##vso[task.prependpath]/usr/lib/ccache"
@@ -90,16 +85,27 @@ jobs:
         -DCMAKE_C_COMPILER_LAUNCHER=ccache
         -DCMAKE_HIP_FLAGS="-Wno-missing-include-dirs"
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCK_BUILD_JIT_LIB=ON
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
         -DCMAKE_BUILD_TYPE=Release
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
 
 - job: composable_kernel_testing
+  timeoutInMinutes: 90
   dependsOn: composable_kernel
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -122,19 +128,12 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - task: Bash@3
     displayName: Iterate through test scripts
     inputs:
@@ -144,3 +143,8 @@ jobs:
           ./$file | tee -a $(TEST_LOG_FILE)
         done
       workingDirectory: $(Agent.BuildDirectory)/rocm/bin
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/copyHIP.yml

@@ -25,8 +25,10 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-download.yml
     parameters:
       componentName: HIP
-      pipelineId: $(hip-pipeline-id)
+      pipelineId: $(HIP_PIPELINE_ID)
   - template:  ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
       sourceDir: $(Agent.BuildDirectory)/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml

.azuredevops/components/half.yml

@@ -25,8 +25,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -39,19 +38,15 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   # only run test03 because test11 has too many test cases, taking way too long
   - task: Bash@3
@@ -63,3 +58,8 @@ jobs:
         make test03
         ./bin/test
       workingDirectory: $(Build.SourcesDirectory)/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: combined

.azuredevops/components/hip-tests.yml

@@ -28,6 +28,14 @@ parameters:
     - clr
     - rocminfo
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 # HIP with AMD backend
 jobs:
@@ -56,13 +64,8 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   # compile hip-tests
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -76,6 +79,71 @@ jobs:
         -DHIP_PATH=$(Agent.BuildDirectory)/rocm
         -DOFFLOAD_ARCH_STR="--offload-arch=$(JOB_GPU_TARGET)"
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+
+- job: hip_tests_testing
+  timeoutInMinutes: 240
+  dependsOn: hip_tests
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - task: Bash@3
+    displayName: Symlink rocm_agent_enumerator
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo mkdir -p /opt/rocm/bin
+        sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rocm_agent_enumerator /opt/rocm/bin/rocm_agent_enumerator
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hip_tests
+      testDir: $(Agent.BuildDirectory)/rocm/share/hip
+  - task: Bash@3
+    displayName: Clean up symlink
+    condition: always()
+    inputs:
+      targetType: inline
+      script: sudo rm -rf /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      optSymLink: true

.azuredevops/components/hipBLAS-common.yml

@@ -29,8 +29,7 @@ jobs:
   - name: ROCM_PATH
     value: $(Agent.BuildDirectory)/rocm
   - template: /.azuredevops/variables-global.yml
-  pool: 
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -41,22 +40,21 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
-        dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      extraEnvVars:
+        - ROCM_PATH:::/home/user/workspace/rocm

.azuredevops/components/hipBLAS.yml

@@ -16,7 +16,10 @@ parameters:
     - libgtest-dev
     - wget
     - python3-pip
-    - libomp-dev
+- name: pipModules
+  type: object
+  default:
+    - pyyaml
 - name: rocmDependencies
   type: object
   default:
@@ -46,6 +49,7 @@ parameters:
     - rocBLAS
     - rocSPARSE
     - rocSOLVER
+    - roctracer
 
 jobs:
 - job: hipBLAS
@@ -65,6 +69,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
@@ -72,14 +77,9 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -92,13 +92,23 @@ jobs:
         -DBUILD_CLIENTS_SAMPLES=OFF
         -DCPACK_SET_DESTDIR=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      installAOCL: true
+      gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipBLAS_testing
   dependsOn: hipBLAS
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -114,27 +124,27 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipBLAS
       testExecutable: $(Agent.BuildDirectory)/rocm/bin/hipblas-test
       testParameters: '--yaml hipblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'
       testDir: '$(Agent.BuildDirectory)/rocm/bin'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/hipBLASLt.yml

@@ -16,12 +16,14 @@ parameters:
     - python3-pip
     - python3-venv
     - gfortran
-    - libomp-dev
     - libblas-dev
+    - ccache
 - name: pipModules
   type: object
   default:
     - joblib
+    - packaging>=22.0
+    - --upgrade
 - name: rocmDependencies
   type: object
   default:
@@ -33,6 +35,7 @@ parameters:
     - rocm_smi_lib
     - rocprofiler-register
     - ROCR-Runtime
+    - roctracer
 - name: rocmTestDependencies
   type: object
   default:
@@ -44,6 +47,7 @@ parameters:
     - rocm_smi_lib
     - rocprofiler-register
     - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: hipBLASLt
@@ -55,15 +59,13 @@ jobs:
     value: $(Build.BinariesDirectory)/rocm
   - name: TENSILE_ROCM_ASSEMBLER_PATH
     value: $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-  - name: CMAKE_CXX_COMPILER
-    value: $(Agent.BuildDirectory)/rocm/bin/hipcc
   - name: TENSILE_ROCM_OFFLOAD_BUNDLER_PATH
     value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang-offload-bundler
-  - name: TENSILE_ROCM_PATH
-    value: $(Agent.BuildDirectory)/rocm/bin/hipcc
-  - name: PATH
-    value: $(Agent.BuildDirectory)/rocm/llvm/bin:$(Agent.BuildDirectory)/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  - name: ROCM_PATH
+    value: $(Agent.BuildDirectory)/rocm
+  - name: DAY_STRING
+    value: $[format('{0:ddMMyyyy}', pipeline.startTime)]
+  pool: ${{ variables.ULTRA_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -84,14 +86,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
     displayName: ROCm symbolic link
 # Build and install gtest, lapack, hipBLAS-common
@@ -110,28 +107,59 @@ jobs:
   - script: sudo make install
     displayName: Install hipBLASLt external dependencies
     workingDirectory: $(Pipeline.Workspace)/deps
+  - script: |
+      mkdir -p $(CCACHE_DIR)
+      echo "##vso[task.prependpath]/usr/lib/ccache"
+    displayName: Update path for ccache
+  - task: Cache@2
+    displayName: Ccache caching
+    inputs:
+      key: hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING) | $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+      path: $(CCACHE_DIR)
+      restoreKeys: |
+        hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING)
+        hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET)
+        hipBLASLt | $(Agent.OS)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_BUILD_TYPE=Release
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DTensile_LOGIC=
         -DTensile_CPU_THREADS=
-        -DTensile_CODE_OBJECT_VERSION=default
         -DTensile_LIBRARY_FORMAT=msgpack
         -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
         -DBUILD_CLIENTS_TESTS=ON
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
-
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraPaths: /home/user/workspace/rocm/llvm/bin:/home/user/workspace/rocm/bin
+      installLatestCMake: true
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+        - TENSILE_ROCM_ASSEMBLER_PATH:::/home/user/workspace/rocm/llvm/bin/amdclang
+        - TENSILE_ROCM_OFFLOAD_BUNDLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang-offload-bundler
+        - ROCM_PATH:::/home/user/workspace/rocm
+      extraCopyDirectories:
+        - deps
 
 - job: hipBLASLt_testing
   dependsOn: hipBLASLt
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -147,27 +175,27 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipBLASLt
       testDir: '$(Agent.BuildDirectory)/rocm/bin'
       testExecutable: './hipblaslt-test'
       testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/hipCUB.yml

@@ -55,14 +55,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -72,13 +67,21 @@ jobs:
         -DBUILD_TEST=ON
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipCUB_testing
   dependsOn: hipCUB
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -99,20 +102,18 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipCUB
       testDir: '$(Agent.BuildDirectory)/rocm/bin/hipcub'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/hipFFT.yml

@@ -14,11 +14,11 @@ parameters:
     - libgtest-dev
     - libfftw3-dev
     - python3-pip
-    - libomp-14-dev
 # rocm dependencies should match dependencies-rocm.yml
 - name: rocmDependencies
   type: object
   default:
+    - aomp
     - rocRAND
     - hipRAND
     - llvm-project
@@ -47,8 +47,7 @@ jobs:
   - template: /.azuredevops/variables-global.yml
   - name: HIP_ROCCLR_HOME
     value: $(Build.BinariesDirectory)/rocm
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -67,14 +66,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -90,13 +84,21 @@ jobs:
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipFFT_testing
   dependsOn: hipFFT
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -117,22 +119,20 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipFFT
       testDir: '$(Agent.BuildDirectory)/rocm/bin'
       testExecutable: './hipfft-test'
       testParameters: '--test_prob 0.002 --gtest_output=xml:./test_output.xml --gtest_color=yes'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/hipRAND.yml

@@ -12,6 +12,7 @@ parameters:
     - ninja-build
     - googletest
     - git
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
@@ -37,8 +38,7 @@ jobs:
   - template: /.azuredevops/variables-global.yml
   - name: HIP_ROCCLR_HOME
     value: $(Build.BinariesDirectory)/rocm
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -57,14 +57,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -76,13 +71,23 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
 
 - job: hipRAND_testing
   dependsOn: hipRAND
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -103,20 +108,18 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipRAND
       testDir: '$(Agent.BuildDirectory)/rocm/bin/hipRAND'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/hipSOLVER.yml

@@ -15,6 +15,7 @@ parameters:
     - git
     - googletest
     - libgtest-dev
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
@@ -42,14 +43,14 @@ parameters:
     - ROCR-Runtime
     - rocSOLVER
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: hipSOLVER
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -68,14 +69,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
 # build external gtest and lapack
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -96,13 +92,23 @@ jobs:
         -DBUILD_CLIENTS_TESTS=ON
         -DUSE_CUDA=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraCopyDirectories:
+        - deps-install
 
 - job: hipSOLVER_testing
   dependsOn: hipSOLVER
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -123,22 +129,20 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipSOLVER
       testDir: '$(Agent.BuildDirectory)/rocm/bin'
       testExecutable: './hipsolver-test'
       testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/hipSPARSE.yml

@@ -16,6 +16,7 @@ parameters:
     - git
     - gfortran
     - libgtest-dev
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
@@ -37,14 +38,14 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: hipSPARSE
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -63,14 +64,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -80,6 +76,10 @@ jobs:
         -DBUILD_CLIENTS_TESTS=ON
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      artifactName: hipSPARSE
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: hipSPARSE
@@ -93,10 +93,16 @@ jobs:
     parameters:
       artifactName: testMatrices
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipSPARSE_testing
   dependsOn: hipSPARSE
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -117,22 +123,20 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipSPARSE
       testDir: '$(Agent.BuildDirectory)/rocm/bin'
       testExecutable: './hipsparse-test'
       testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/hipSPARSELt.yml

@@ -15,7 +15,6 @@ parameters:
     - python3-pip
     - gfortran
     - libgfortran5
-    - libomp-dev
     - libopenblas-dev
 - name: pipModules
   type: object
@@ -32,6 +31,7 @@ parameters:
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
+    - roctracer
 - name: rocmTestDependencies
   type: object
   default:
@@ -44,6 +44,7 @@ parameters:
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: hipSPARSELt
@@ -79,14 +80,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
 # Build and install gtest and lapack
 # $(Pipeline.Workspace)/deps is a temporary folder for the build process
 # $(Pipeline.Workspace)/s/deps is part of the hipSPARSELt repo
@@ -112,19 +108,36 @@ jobs:
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DTensile_LOGIC=
         -DTensile_CPU_THREADS=
-        -DTensile_CODE_OBJECT_VERSION=default
         -DTensile_LIBRARY_FORMAT=msgpack
         -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_CLIENTS_TESTS=ON
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraCopyDirectories:
+        - deps
+      extraPaths: /home/user/workspace/rocm/llvm/bin:/home/user/workspace/rocm/bin
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+        - TENSILE_ROCM_ASSEMBLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang
+        - CMAKE_CXX_COMPILER:::/home/user/workspace/rocm/llvm/bin/hipcc
+        - TENSILE_ROCM_OFFLOAD_BUNDLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang-offload-bundler
+      installLatestCMake: true
 
 - job: hipSPARSELt_testing
   dependsOn: hipSPARSELt
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -146,15 +159,19 @@ jobs:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipSPARSELt
       testDir: '$(Agent.BuildDirectory)/rocm/bin'
       testExecutable: './hipsparselt-test'
       testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/hipTensor.yml

@@ -54,14 +54,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -72,14 +67,22 @@ jobs:
         -DHIPTENSOR_BUILD_TESTS=ON
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
       multithreadFlag: -- -j32
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipTensor_testing
   timeoutInMinutes: 90
   dependsOn: hipTensor
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -100,21 +103,19 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipTensor
       testDir: '$(Agent.BuildDirectory)/rocm/bin/hiptensor'
-      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testParameters: '-E ".*-extended" -VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/hipfort.yml

@@ -11,26 +11,140 @@ parameters:
     - gfortran
     - git
     - cmake
+    - python3-pip
+    - python3-dev
+    - ninja-build
+- name: rocmDependencies
+  type: object
+  default:
+    - aomp
+    - llvm-project
+    - clr
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - hipFFT
+    - hipRAND
+    - hipSOLVER
+    - hipSPARSE
+    - rocBLAS
+    - rocFFT
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler
+    - rocprofiler-register
+    - rocRAND
+    - ROCR-Runtime
+    - rocSOLVER
+    - rocSPARSE
+    - roctracer
 
 jobs:
 - job: hipfort
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/llvm
+        -DHIPFORT_INSTALL_DIR=$(Build.BinariesDirectory)
+        -DHIPFORT_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/flang
+        -DCMAKE_Fortran_FLAGS="-Mfree -fPIC"
+        -DCMAKE_Fortran_FLAGS_DEBUG=""
+        -DHIPFORT_COMPILER_FLAGS="-cpp"
+        -DHIPFORT_AR=$(Agent.BuildDirectory)/rocm/llvm/bin/llvm-ar
+        -DHIPFORT_RANLIB=$(Agent.BuildDirectory)/rocm/llvm/bin/llvm-ranlib
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DBUILD_TESTING=ON
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      installLatestCMake: true
+
+- job: hipfort_testing
+  dependsOn: hipfort
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+  - task: Bash@3
+    displayName: 'ROCm symbolic link'
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - task: Bash@3
+    displayName: 'Test hipfort'
+    inputs:
+      targetType: inline
+      script: PATH=$(Agent.BuildDirectory)/rocm/bin:$PATH make run_all
+      workingDirectory: $(Build.SourcesDirectory)/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
     parameters:
-      extraBuildFlags: >-
-        -DHIPFORT_INSTALL_DIR=/tmp/hipfort
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      optSymLink: true

.azuredevops/components/llvm-project.yml

@@ -27,6 +27,8 @@ jobs:
   - template: /.azuredevops/variables-global.yml
   - name: HIP_DEVICE_LIB_PATH
     value: '$(Build.BinariesDirectory)/amdgcn/bitcode'
+  - name: HIP_PATH
+    value: '$(Agent.BuildDirectory)/rocm'
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
@@ -40,14 +42,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       skipLlvmSymlink: true
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: rocm-llvm
@@ -104,6 +101,14 @@ jobs:
       testExecutable: './bin/llvm-lit'
       testParameters: '-q --xunit-xml-output=lld_test_output.xml ./tools/lld/test'
       testOutputFile: lld_test_output.xml
+  - task: CopyFiles@2
+    displayName: Copy FileCheck for Publishing
+    inputs:
+      CleanTargetFolder: false
+      SourceFolder: llvm/build/bin
+      Contents: FileCheck
+      TargetFolder: $(Build.BinariesDirectory)/llvm/bin
+      retryCount: 3
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: device-libs
@@ -116,6 +121,7 @@ jobs:
       componentName: comgr
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH="$(Build.SourcesDirectory)/llvm/build;$(Build.SourcesDirectory)/amd/device-libs/build"
+        -DCOMGR_DISABLE_SPIRV=1
         -DCMAKE_BUILD_TYPE=Release
       cmakeBuildDir: 'amd/comgr/build'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
@@ -131,4 +137,13 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DHIPCC_BACKWARD_COMPATIBILITY=OFF
       cmakeBuildDir: 'amd/hipcc/build'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: combined
+      extraEnvVars:
+        - HIP_DEVICE_LIB_PATH:::/home/user/workspace/bin/amdgcn/bitcode
+        - HIP_PATH:::/home/user/workspace/rocm

.azuredevops/components/rccl.yml

@@ -8,32 +8,50 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - python3-pip
     - cmake
-    - libboost-program-options-dev
-    - googletest
-    - libfftw3-dev
     - git
-    - ninja-build
-    - libstdc++-12-dev
+    - googletest
+    - libboost-program-options-dev
+    - libdrm-dev
+    - libfftw3-dev
     - libnuma-dev
+    - libstdc++-12-dev
+    - ninja-build
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
-    - clr
-    - rocminfo
-    - rocm_smi_lib
-    - rocprofiler-register
-    - rocm-core
-    - HIPIFY
     - aomp
     - aomp-extras
+    - clr
+    - HIPIFY
+    - llvm-project
+    - rocm-cmake
+    - rocm-core
+    - rocm_smi_lib
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - roctracer
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - aomp-extras
+    - clr
+    - HIPIFY
+    - llvm-project
+    - rocm-cmake
+    - rocm-core
+    - rocm_smi_lib
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: rccl
+  timeoutInMinutes: 90
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -60,15 +78,9 @@ jobs:
       submoduleBehaviour: recursive
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
-  - script: chmod +x $(Agent.BuildDirectory)/rocm/bin/hipify-perl
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -78,9 +90,62 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_TESTS=ON
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/share/rocm/cmake/
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/share/rocm/cmake;$(Agent.BuildDirectory)/rocm/libexec/hipify
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+      installLatestCMake: true
+
+- job: rccl_testing
+  timeoutInMinutes: 120
+  dependsOn: rccl
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rccl
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rccl-UnitTests'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rdc.yml

@@ -18,16 +18,40 @@ parameters:
     - autoconf
     - libtool
     - pkg-config
+    - libdrm-dev
+    - libnuma-dev
+    - libyaml-cpp-dev
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
+    - amdsmi
     - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocm-core
     - rocminfo
     - rocm_smi_lib
+    - ROCmValidationSuite
+    - rocprofiler
+    - rocprofiler-register
+    - rocprofiler-sdk
+    - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
     - amdsmi
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocm_smi_lib
+    - ROCmValidationSuite
+    - rocprofiler
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rdc
@@ -37,6 +61,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -47,13 +77,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+      gpuTarget: $(JOB_GPU_TARGET)
 # Build grpc
   - task: Bash@3
     displayName: 'git clone grpc'
@@ -71,11 +97,81 @@ jobs:
         -DBUILD_SHARED_LIBS=ON
         -DCMAKE_INSTALL_LIBDIR=lib
         -DCMAKE_BUILD_TYPE=Release
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DGRPC_ROOT="$(Build.SourcesDirectory)/bin"
+        -DBUILD_RVS=ON
+        -DBUILD_PROFILER=ON
         -DBUILD_TESTS=ON
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rdc_testing
+  dependsOn: rdc
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo rm -rf /usr/sbin/rdcd
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rdcd /usr/sbin/rdcd
+        echo $(Agent.BuildDirectory)/rocm/lib/rdc/grpc/lib | sudo tee /etc/ld.so.conf.d/grpc.conf
+        sudo ldconfig -v
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - task: Bash@3
+    displayName: Test rdc
+    inputs:
+      targetType: inline
+      script: >-
+        $(Agent.BuildDirectory)/rocm/share/rdc/rdctst_tests/rdctst
+        --batch_mode
+        --start_rdcd
+        --unauth_comm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraPaths: /home/user/workspace/rocm/bin

.azuredevops/components/rocAL.yml

@@ -21,6 +21,8 @@ parameters:
     - libavcodec-dev
     - libavformat-dev
     - libavutil-dev
+    - libdlpack-dev
+    - libsndfile1-dev
     - libswscale-dev
     - libturbojpeg-dev
     - libjpeg-turbo-official=3.0.2-20240124
@@ -36,15 +38,15 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
-    - clr
-    - rocDecode
-    - half
-    - rpp
-    - MIVisionX
     - aomp
+    - clr
+    - half
+    - llvm-project
+    - MIVisionX
+    - rocDecode
+    - rocm-cmake
+    - ROCR-Runtime
+    - rpp
 - name: rocmTestDependencies
   type: object
   default:
@@ -53,6 +55,8 @@ parameters:
     - half
     - llvm-project
     - MIVisionX
+    - rocDecode
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
     - rpp
@@ -62,8 +66,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -135,14 +138,9 @@ jobs:
       workingDirectory: '$(Build.SourcesDirectory)/rapidjson/build'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -150,14 +148,27 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;/opt/libjpeg-turbo
         -DCMAKE_INSTALL_PREFIX_PYTHON=$Python3_STDARCH
         -DCMAKE_BUILD_TYPE=Release
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      registerJPEGPackages: true
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraCopyDirectories:
+        - /opt/libjpeg-turbo
 
 - job: rocAL_testing
   dependsOn: rocAL
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -165,7 +176,9 @@ jobs:
     value: $(Agent.BuildDirectory)/rocm
   - name: CMAKE_INCLUDE_PATH
     value: $(Agent.BuildDirectory)/rocm/include/rocal
-  pool: $(JOB_TEST_POOL)
+  pool:
+    name: $(JOB_TEST_POOL)
+    demands: firstRenderDeviceAccess
   workspace:
     clean: all
   strategy:
@@ -192,19 +205,11 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - task: Bash@3
     displayName: Link libjpeg-turbo
     inputs:
@@ -220,15 +225,26 @@ jobs:
         mkdir rocAL-tests
         cd rocAL-tests
         cmake $(Agent.BuildDirectory)/rocm/share/rocal/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocAL
       testDir: rocAL-tests
-      reloadAMDGPU: true
   - task: Bash@3
     displayName: Clean up libjpeg-turbo
+    condition: always()
     inputs:
       targetType: inline
       script: |
         sudo rm /etc/ld.so.conf.d/libjpeg-turbo.conf
         sudo ldconfig -v
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      registerJPEGPackages: true
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraCopyDirectories:
+        - /opt/libjpeg-turbo
+# docker image will be missing the ldconfig to libjpeg-turbo

.azuredevops/components/rocALUTION.yml

@@ -15,6 +15,7 @@ parameters:
     - git
     - mpich
     - ninja-build
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
@@ -43,6 +44,7 @@ parameters:
     - ROCR-Runtime
     - rocRAND
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: rocALUTION
@@ -51,8 +53,7 @@ jobs:
   - template: /.azuredevops/variables-global.yml
   - name: HIP_ROCCLR_HOME
     value: $(Build.BinariesDirectory)/rocm
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -71,14 +72,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -91,13 +87,23 @@ jobs:
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
 
 - job: rocALUTION_testing
   dependsOn: rocALUTION
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -118,22 +124,20 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocALUTION
       testDir: '$(Agent.BuildDirectory)/rocm/bin'
       testExecutable: './rocalution-test'
       testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocBLAS.yml

@@ -40,6 +40,7 @@ parameters:
     - aomp-extras
     - hipBLAS-common
     - hipBLASLt
+    - roctracer
 - name: rocmTestDependencies
   type: object
   default:
@@ -52,6 +53,7 @@ parameters:
     - rocprofiler-register
     - rocm_smi_lib
     - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: rocBLAS
@@ -62,10 +64,10 @@ jobs:
     value: $(Build.BinariesDirectory)/rocm
   - name: TENSILE_ROCM_ASSEMBLER_PATH
     value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang
-  - name: CMAKE_CXX_COMPILER
-    value: $(Agent.BuildDirectory)/rocm/bin/hipcc
   - name: TENSILE_ROCM_OFFLOAD_BUNDLER_PATH
     value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang-offload-bundler
+  - name: ROCM_PATH
+    value: $(Agent.BuildDirectory)/rocm
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
@@ -87,23 +89,18 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_TOOLCHAIN_FILE=toolchain-linux.cmake
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm;$(Pipeline.Workspace)/deps-install
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/amdclang++
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/amdclang
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -DTensile_CODE_OBJECT_VERSION=default
         -DTensile_LOGIC=asm_full
         -DTensile_SEPARATE_ARCHITECTURES=ON
@@ -114,13 +111,28 @@ jobs:
         -DBUILD_CLIENTS_SAMPLES=OFF
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      installAOCL: true
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+        - TENSILE_ROCM_ASSEMBLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang
+        - TENSILE_ROCM_OFFLOAD_BUNDLER_PATH:::/home/user/workspace/rocm/llvm/bin/clang-offload-bundler
+        - ROCM_PATH:::/home/user/workspace/rocm
 
 - job: rocBLAS_testing
   dependsOn: rocBLAS
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -142,22 +154,21 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocBLAS
       testDir: '$(Agent.BuildDirectory)/rocm/bin'
       testExecutable: './rocblas-test'
       testParameters: '--yaml rocblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocDecode.yml

@@ -17,9 +17,10 @@ parameters:
     - libavformat-dev
     - libavutil-dev
     - libstdc++-12-dev
-    - libva-dev
+    - libva-amdgpu-dev
     - mesa-amdgpu-va-drivers
     - libdrm-dev
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
@@ -35,6 +36,7 @@ parameters:
   default:
     - clr
     - llvm-project
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
 
@@ -43,40 +45,22 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
-  # Since mesa-amdgpu-multimedia-devel is not directly available from apt, register it
-  - task: Bash@3
-    displayName: 'Register ROCm packages'
-    inputs:
-      targetType: inline
-      script: |
-        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
-        wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
-        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/${{ variables.KEYRING_VERSION }}/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/amdgpu.list
-        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${{ variables.KEYRING_VERSION }} jammy main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
-        echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
-        sudo apt update
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      registerROCmPackages: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -84,15 +68,23 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      registerROCmPackages: true
 
 - job: rocDecode_testing
   dependsOn: rocDecode
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+  pool:
+    name: $(JOB_TEST_POOL)
+    demands: firstRenderDeviceAccess
   workspace:
     clean: all
   strategy:
@@ -104,22 +96,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      registerROCmPackages: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   # anything in /opt may be persistent across runs
   # so we need to remove the symlink if it already exists
   - script: |
@@ -128,8 +113,16 @@ jobs:
       mkdir rocDecode-tests
       cd rocDecode-tests
       cmake /opt/rocm/share/rocdecode/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocDecode
       testDir: 'rocDecode-tests'
   - script: sudo rm /opt/rocm
+    condition: always()
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      registerROCmPackages: true
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocFFT.yml

@@ -67,14 +67,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -89,13 +84,23 @@ jobs:
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
 
 - job: rocFFT_testing
   dependsOn: rocFFT
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -116,22 +121,20 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocFFT
       testDir: '$(Agent.BuildDirectory)/rocm/bin'
       testExecutable: './rocfft-test'
       testParameters: '--test_prob 0.004 --gtest_output=xml:./test_output.xml --gtest_color=yes'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocJPEG.yml

@@ -0,0 +1,133 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - libdrm-dev
+    - libstdc++-12-dev
+    - libva-amdgpu-dev
+    - mesa-amdgpu-va-drivers
+    - ninja-build
+    - pkg-config
+    - python3-pip
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocm-core
+    - rocprofiler-register
+    - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+
+jobs:
+- job: rocJPEG
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.LOW_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      registerROCmPackages: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      registerROCmPackages: true
+
+- job: rocJPEG_testing
+  dependsOn: rocJPEG
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    name: $(JOB_TEST_POOL)
+    demands: firstRenderDeviceAccess
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      registerROCmPackages: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  # anything in /opt may be persistent across runs
+  # so we need to remove the symlink if it already exists
+  - script: |
+      sudo rm -rf /opt/rocm
+      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      mkdir rocJPEG-tests
+      cd rocJPEG-tests
+      cmake $(Agent.BuildDirectory)/rocm/share/rocjpeg/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocJPEG
+      testDir: 'rocJPEG-tests'
+  - script: sudo rm /opt/rocm
+    condition: always()
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      registerROCmPackages: true
+      optSymLink: true

.azuredevops/components/rocMLIR.yml

@@ -13,6 +13,7 @@ parameters:
     - git
     - python3-pip
     - libdrm-dev
+    - libstdc++-12-dev
 - name: pipModules
   type: object
   default:
@@ -28,7 +29,7 @@ parameters:
     - ROCR-Runtime
 
 jobs:
-- job: rocMLIR_library
+- job: rocMLIR
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -46,13 +47,8 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -62,20 +58,29 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_FAT_LIBROCKCOMPILER=1
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
 
 # compiling and running test on the test system together
 - job: rocMLIR_testing
+  dependsOn: rocMLIR
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.GFX942_TEST_POOL }}
+  pool: $(JOB_TEST_POOL)
   workspace:
     clean: all
   strategy:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -86,26 +91,18 @@ jobs:
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - task: Bash@3
     displayName: ROCm symbolic link
     inputs:
       targetType: inline
-      script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       installEnabled: false
@@ -117,9 +114,16 @@ jobs:
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DROCM_TEST_CHIPSET=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocMLIR
       testDir: $(Build.SourcesDirectory)/build
       testExecutable: ninja
       testParameters: check-rocmlir
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocPRIM.yml

@@ -54,14 +54,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -71,13 +66,21 @@ jobs:
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DBUILD_TEST=ON
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocPRIM_testing
   dependsOn: rocPRIM
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -98,20 +101,18 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocPRIM
       testDir: '$(Agent.BuildDirectory)/rocm/bin/rocprim'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocPyDecode.yml

@@ -8,25 +8,22 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - python3-pip
     - cmake
-    - ninja-build
-    - pkg-config
     - ffmpeg
     - libavcodec-dev
     - libavformat-dev
     - libavutil-dev
-    - libva-dev
-    - libdrm-dev
-    - pybind11-dev
-    - python3-pybind11
     - libdlpack-dev
+    - libdrm-dev
+    - libva-dev
+    - ninja-build
+    - pkg-config
+    - python3-pip
 - name: pipModules
   type: object
   default:
-    - -i
-    - https://test.pypi.org/simple
-    - hip-python
+    - numpy
+    - pybind11
 - name: rocmDependencies
   type: object
   default:
@@ -44,8 +41,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -59,29 +55,198 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
       pipModules: ${{ parameters.pipModules }}
+  - task: Bash@3
+    displayName: 'pip install hip-python'
+    inputs:
+      targetType: inline
+      script: pip install -i https://test.pypi.org/simple hip-python
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: 'ROCm symbolic link'
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: 'Save Python Package Paths'
+    inputs:
+      targetType: inline
+      script: |
+        echo "##vso[task.setvariable variable=PYTHON_USER_SITE;]$(python3 -m site --user-site)"
+        echo "##vso[task.setvariable variable=PYTHON_DIST_PACKAGES;]$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
+        echo "##vso[task.setvariable variable=PYBIND11_PATH;]$(python3 -c 'import pybind11; print(pybind11.get_cmake_dir())')"
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
+      installEnabled: false
       extraBuildFlags: >-
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(PYTHON_USER_SITE)/pybind11;$(PYTHON_DIST_PACKAGES)/pybind11;$(PYBIND11_PATH)
         -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -DCMAKE_INSTALL_PREFIX_PYTHON=$(Build.BinariesDirectory)
         -GNinja
+  - task: Bash@3
+    displayName: 'rocPyDecode install'
+    inputs:
+      targetType: inline
+      script: |
+        sudo cmake --build . --target install
+        sudo chown -R $(whoami):$(id -gn) $(Build.BinariesDirectory)
+      workingDirectory: $(Build.SourcesDirectory)/build
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+      publish: false
+  - task: Bash@3
+    displayName: Create wheel file
+    inputs:
+      targetType: inline
+      script: |
+        export ROCM_PATH=$(Agent.BuildDirectory)/rocm
+        export HIP_INCLUDE_DIRS=$(Agent.BuildDirectory)/rocm/include/hip
+        sudo python3 setup.py bdist_wheel
+        sudo chown -R $(whoami):$(id -gn) $(find . -name "*.whl")
+      workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.SourcesDirectory)/dist
+      contentsString: '*.whl'
+      targetDir: $(Build.ArtifactStagingDirectory)
+      clean: false
+  - task: PublishPipelineArtifact@1
+    displayName: 'wheel file Publish'
+    retryCountOnTaskFailure: 3
+    inputs:
+      targetPath: $(Build.ArtifactStagingDirectory)
+  - task: Bash@3
+    displayName: Save pipeline artifact file names
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: |
+        whlFile=$(find "$(Build.ArtifactStagingDirectory)" -type f -name "*.whl" | head -n 1)
+        if [ -n "$whlFile" ]; then
+          echo $(basename "$whlFile") >> pipelineArtifacts.txt
+        fi
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      optSymLink: true
+
+- job: rocPyDecode_testing
+  dependsOn: rocPyDecode
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    name: $(JOB_TEST_POOL)
+    demands: firstRenderDeviceAccess
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - task: Bash@3
+    displayName: Ensure pybind11-dev is not installed
+    inputs:
+      targetType: inline
+      script: |
+        if dpkg -l | grep -q pybind11-dev; then
+          echo "Removing pybind11-dev..."
+          sudo apt remove -y pybind11-dev
+        else
+          echo "pybind11-dev is not installed."
+        fi
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - task: Bash@3
+    displayName: 'pip install hip-python'
+    inputs:
+      targetType: inline
+      script: pip install -i https://test.pypi.org/simple hip-python
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Wheel Files'
+    inputs:
+      itemPattern: '**/*.whl'
+      targetPath: $(Agent.BuildDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      setupHIPLibrarySymlinks: true
+  - task: Bash@3
+    displayName: pip install
+    inputs:
+      targetType: inline
+      script: |
+        pip uninstall -y rocPyDecode
+        find -name *.whl -exec pip install {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Setup search paths
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        echo "##vso[task.setvariable variable=PYTHON_USER_SITE;]$(python3 -m site --user-site)"
+        echo "##vso[task.setvariable variable=PYTHON_DIST_PACKAGES;]$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
+        echo "##vso[task.setvariable variable=PYBIND11_PATH;]$(python3 -c 'import pybind11; print(pybind11.get_cmake_dir())')"
+  - task: CMake@1
+    displayName: 'rocPyDecode Test CMake Flags'
+    inputs:
+      cmakeArgs: >-
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(PYTHON_USER_SITE)/pybind11;$(PYTHON_DIST_PACKAGES)/pybind11;$(PYBIND11_PATH)
+        -DCMAKE_BUILD_TYPE=Release
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        ..
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocPyDecode
+      testDir: $(Build.SourcesDirectory)/build
+# sudo required for pip install but screws up permissions for next pipeline run
+  - task: Bash@3
+    displayName: Clean up test environment
+    condition: always()
+    inputs:
+      targetType: inline
+      script: |
+        pip uninstall -y rocPyDecode
+        pip uninstall -y hip-python
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      pythonEnvVars: true
+  # note that this docker won't have hip-python installed via pip

.azuredevops/components/rocRAND.yml

@@ -9,10 +9,11 @@ parameters:
   type: object
   default:
     - cmake
-    - ninja-build
+    - git
     - googletest
     - libgtest-dev
-    - git
+    - ninja-build
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
@@ -37,8 +38,7 @@ jobs:
   - template: /.azuredevops/variables-global.yml
   - name: HIP_ROCCLR_HOME
     value: $(Build.BinariesDirectory)/rocm
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -57,14 +57,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -73,13 +68,23 @@ jobs:
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
 
 - job: rocRAND_testing
   dependsOn: rocRAND
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -100,20 +105,18 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocRAND
       testDir: '$(Agent.BuildDirectory)/rocm/bin/rocRAND'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocSOLVER.yml

@@ -44,6 +44,7 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: rocSOLVER
@@ -75,14 +76,9 @@ jobs:
       workingDirectory: '$(Build.SourcesDirectory)'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: lapack
@@ -106,13 +102,23 @@ jobs:
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraCopyDirectories:
+        - deps-install
 
 - job: rocSOLVER_testing
   dependsOn: rocSOLVER
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -133,22 +139,20 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocSOLVER
       testDir: '$(Agent.BuildDirectory)/rocm/bin'
       testExecutable: './rocsolver-test'
       testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocSPARSE.yml

@@ -40,6 +40,7 @@ parameters:
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: rocSPARSE
@@ -67,14 +68,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -89,6 +85,10 @@ jobs:
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip;$(Agent.BuildDirectory)/rocm/hip/cmake
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      artifactName: rocSPARSE
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: rocSPARSE
@@ -102,11 +102,18 @@ jobs:
     parameters:
       artifactName: testMatrices
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
 
 - job: rocSPARSE_testing
   timeoutInMinutes: 90
   dependsOn: rocSPARSE
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -127,23 +134,20 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocSPARSE
       testDir: '$(Agent.BuildDirectory)/rocm/bin'
       testExecutable: './rocsparse-test'
       testParameters: '--gtest_filter="*quick*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
-
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocThrust.yml

@@ -59,14 +59,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -76,13 +71,21 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DBUILD_TEST=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocThrust_testing
   dependsOn: rocThrust
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -103,20 +106,18 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocThrust
       testDir: '$(Agent.BuildDirectory)/rocm/bin/rocthrust'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocWMMA.yml

@@ -12,14 +12,15 @@ parameters:
     - cmake
     - ninja-build
     - libboost-program-options-dev
+    - libdrm-dev
     - libgtest-dev
     - googletest
     - libfftw3-dev
     - git
-    - libomp-dev
 - name: rocmDependencies
   type: object
   default:
+    - aomp
     - rocm-cmake
     - llvm-project
     - ROCR-Runtime
@@ -42,6 +43,7 @@ parameters:
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: rocWMMA
@@ -67,14 +69,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -83,17 +80,27 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DROCWMMA_BUILD_TESTS=ON
         -DROCWMMA_BUILD_SAMPLES=OFF
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON
+        -DROCM_PLATFORM_VERSION=$(NEXT_RELEASE_VERSION)
         -GNinja
 # gfx1030 not supported in documentation
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocWMMA_testing
   timeoutInMinutes: 90
   dependsOn: rocWMMA
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -114,20 +121,18 @@ jobs:
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocWMMA
       testDir: '$(Agent.BuildDirectory)/rocm/bin/rocwmma'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocm-cmake.yml

@@ -8,9 +8,11 @@ parameters:
 - name: aptPackages
   type: object
   default:
+    - cmake
     - doxygen
     - doxygen-doc
     - ninja-build
+    - python3-pip
     - python3-sphinx
 - name: pipModules
   type: object
@@ -23,8 +25,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -47,4 +48,11 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocm-cmake
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: combined

.azuredevops/components/rocm-core.yml

@@ -5,17 +5,24 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - python3-pip
 
 jobs:
 - job: rocm_core
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
@@ -29,5 +36,10 @@ jobs:
         -DCPACK_GENERATOR=DEB
         -DCPACK_DEBIAN_PACKAGE_RELEASE="local.9999~99.99"
         -DCPACK_RPM_PACKAGE_RELEASE="local.9999"
-        -DROCM_VERSION="$(next-release)"
+        -DROCM_VERSION="$(NEXT_RELEASE_VERSION)"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/rocm-examples.yml

@@ -10,6 +10,7 @@ parameters:
   default:
     - cmake
     - libglfw3-dev
+    - libtbb-dev
     - python3-pip
 - name: rocmDependencies
   type: object
@@ -60,6 +61,7 @@ parameters:
     - rocSOLVER
     - rocSPARSE
     - rocThrust
+    - roctracer
 
 jobs:
 - job: rocm_examples
@@ -85,14 +87,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       # https://github.com/ROCm/HIP/issues/2203
@@ -110,13 +107,21 @@ jobs:
       script: |
         mkdir -p $(Build.BinariesDirectory)/examples
         mv $(Build.BinariesDirectory)/bin/* $(Build.BinariesDirectory)/examples
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocm_examples_testing
   dependsOn: rocm_examples
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -135,37 +140,32 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+      checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
-  - task: Bash@3
-    displayName: Unload and reload AMDGPU
-    inputs:
-      targetType: inline
-      script: |
-        sudo modprobe -r amdgpu
-        sudo modprobe amdgpu
-  - task: Bash@3
-    displayName: Iterate through examples
-    inputs:
-      targetType: inline
-      script: |
-        for file in *; do
-          echo Now running: $file
-          ./$file | tee -a $(TEST_LOG_FILE)
-        done
-      workingDirectory: $(Agent.BuildDirectory)/rocm/examples
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      # https://github.com/ROCm/HIP/issues/2203
+      extraBuildFlags: >-
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_ROOT=$(Agent.BuildDirectory)/rocm
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
+        -DCMAKE_EXE_LINKER_FLAGS=-fgpu-rdc
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm-examples
+      testDir: $(Build.SourcesDirectory)/build
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocm_bandwidth_test.yml

@@ -27,6 +27,7 @@ parameters:
 - name: rocmTestDependencies
   type: object
   default:
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
 
@@ -39,8 +40,7 @@ jobs:
     value: $(Agent.BuildDirectory)/rocm
   - name: ROCR_LIB_DIR
     value: $(Agent.BuildDirectory)/rocm
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -54,13 +54,8 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -68,11 +63,20 @@ jobs:
         -DCMAKE_MODULE_PATH=$(Build.SourcesDirectory)/cmake_modules
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/include;$(Agent.BuildDirectory)/rocm/include/hsa
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      extraEnvVars:
+        - ROCR_INC_DIR:::/home/user/workspace/rocm
+        - ROCR_LIB_DIR:::/home/user/workspace/rocm
 
 - job: rocm_bandwidth_test_testing
   dependsOn: rocm_bandwidth_test
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -93,12 +97,10 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocm_bandwidth_test
@@ -106,3 +108,9 @@ jobs:
       testExecutable: './rocm/bin/rocm-bandwidth-test'
       testParameters: ''
       testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocm_smi_lib.yml

@@ -5,17 +5,26 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - libdrm-dev
+    - pkg-config
+    - python3-pip
 
 jobs:
 - job: rocm_smi_lib
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
@@ -25,11 +34,16 @@ jobs:
       extraBuildFlags: >-
         -DBUILD_TESTS=ON
         -DROCM_DEP_ROCMCORE=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
 
 - job: rocm_smi_lib_testing
   dependsOn: rocm_smi_lib
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -44,9 +58,17 @@ jobs:
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocm_smi_lib
       testDir: '$(Agent.BuildDirectory)'
       testExecutable: './rocm/share/rocm_smi/rsmitst_tests/rsmitst'
       testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocminfo.yml

@@ -5,6 +5,13 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - libdrm-amdgpu-dev
+    - libdrm-dev
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
@@ -21,35 +28,39 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      registerROCmPackages: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       skipLlvmSymlink: true
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCRTST_BLD_TYPE=release
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      registerROCmPackages: true
 
 - job: rocminfo_testing
   dependsOn: rocminfo
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -62,16 +73,19 @@ jobs:
         JOB_GPU_TARGET: gfx942
         JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocminfo
@@ -86,3 +100,9 @@ jobs:
       testExecutable: './rocm/bin/rocm_agent_enumerator'
       testParameters: ''
       testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      registerROCmPackages: true
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocprofiler-compute.yml

@@ -0,0 +1,178 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - locales
+    - python3-pip
+- name: pipModules
+  type: object
+  default:
+    - astunparse==1.6.2
+    - colorlover
+    - dash>=1.12.0
+    - matplotlib
+    - numpy>=1.17.5
+    - pandas>=1.4.3
+    - pymongo
+    - pyyaml
+    - tabulate
+    - tqdm
+    - dash-svg
+    - dash-bootstrap-components
+    - kaleido
+    - setuptools
+    - plotille
+    - mock
+    - pytest
+    - pytest-cov
+    - pytest-xdist
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - rocm_smi_lib
+    - ROCR-Runtime
+    - rocprofiler
+    - rocprofiler-register
+    - roctracer
+
+jobs:
+- job: rocprofiler_compute
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.LOW_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocprofiler_compute_testing
+  timeoutInMinutes: 120
+  dependsOn: rocprofiler_compute
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: PYTHON_VERSION
+    value: 3.10
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: Bash@3
+    displayName: Add en_US.UTF-8 locale
+    inputs:
+      targetType: inline
+      script: |
+        sudo locale-gen en_US.UTF-8
+        sudo update-locale
+        locale -a
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -DENABLE_TESTS=ON
+        -DINSTALL_TESTS=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofiler-compute
+      testDir: $(Build.BinariesDirectory)/libexec/rocprofiler-compute
+      testExecutable: ROCPROFCOMPUTE_ARCH_OVERRIDE="MI300X" ROCM_PATH=$(Agent.BuildDirectory)/rocm ctest
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    condition: always()
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    condition: always()
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocprofiler-register.yml

@@ -5,17 +5,24 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - python3-pip
 
 jobs:
 - job: rocprofiler_register
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
@@ -34,4 +41,10 @@ jobs:
     parameters:
       componentName: rocprofiler-register
       testDir: 'tests/build'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: combined

.azuredevops/components/rocprofiler-sdk.yml

@@ -0,0 +1,163 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - build-essential
+    - libdrm-amdgpu-dev
+    - libdrm-dev
+    - libdw-dev
+    - libelf-dev
+    - pkg-config
+    - python3-pip
+- name: pipModules
+  type: object
+  default:
+    - black
+    - clang-format
+    - clang-tidy
+    - cmake
+    - cmake-format
+    - dataclasses
+    - numpy
+    - otf2
+    - pandas
+    - perfetto
+    - pycobertura
+    - pytest>=6.2.5
+    - pyyaml
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rccl
+    - rocDecode
+    - rocJPEG
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler-register
+
+jobs:
+- job: rocprofiler_sdk
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.LOW_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      registerROCmPackages: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - task: Bash@3
+    displayName: Add Python site-packages binaries to path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.prependpath]$USER_BASE/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCPROFILER_BUILD_TESTS=ON
+        -DROCPROFILER_BUILD_SAMPLES=ON
+        -DROCPROFILER_BUILD_RELEASE=ON
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j4
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      registerROCmPackages: true
+
+- job: rocprofiler_sdk_testing
+  dependsOn: rocprofiler_sdk
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      registerROCmPackages: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - task: Bash@3
+    displayName: Add Python site-packages binaries to path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.prependpath]$USER_BASE/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCPROFILER_BUILD_TESTS=ON
+        -DROCPROFILER_BUILD_SAMPLES=ON
+        -DROCPROFILER_BUILD_RELEASE=ON
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j16
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofiler-sdk
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      registerROCmPackages: true

.azuredevops/components/rocprofiler-systems.yml

@@ -0,0 +1,247 @@
+# largely referenced from: https://github.com/ROCm/omnitrace/blob/main/.github/workflows/ubuntu-jammy.yml
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - autoconf
+    - autotools-dev
+    - bison
+    - build-essential
+    - bzip2
+    - clang
+    - cmake
+    - environment-modules
+    - ffmpeg
+    - g++-12
+    - libavcodec-dev
+    - libavformat-dev
+    - libavutil-dev
+    - libdrm-amdgpu-dev
+    - libdrm-dev
+    - libfabric-dev
+    - libiberty-dev
+    - libpapi-dev
+    - libpfm4-dev
+    - libtool
+    - libopenmpi-dev
+    - m4
+    - openmpi-bin
+    - pkg-config
+    - python3-pip
+    - software-properties-common
+    - texinfo
+    - zlib1g-dev
+- name: pipModules
+  type: object
+  default:
+    - numpy
+    - perfetto
+    - dataclasses
+- name: rocmDependencies
+  type: object
+  default:
+    - amdsmi
+    - aomp
+    - clr
+    - llvm-project
+    - rccl
+    - rocDecode
+    - rocJPEG
+    - rocm-core
+    - rocminfo
+    - rocm_smi_lib
+    - rocprofiler-register
+    - rocprofiler-sdk
+    - ROCR-Runtime
+
+jobs:
+- job: rocprofiler_systems
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      registerROCmPackages: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+# build flags reference: https://rocm.docs.amd.com/projects/omnitrace/en/latest/install/install.html
+      extraBuildFlags: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCPROFSYS_BUILD_TESTING=ON
+        -DROCPROFSYS_BUILD_DYNINST=ON
+        -DROCPROFSYS_BUILD_LIBUNWIND=ON
+        -DROCPROFSYS_DISABLE_EXAMPLES="openmp-target"
+        -DDYNINST_BUILD_TBB=ON
+        -DDYNINST_BUILD_ELFUTILS=ON
+        -DDYNINST_BUILD_LIBIBERTY=ON
+        -DDYNINST_BUILD_BOOST=ON
+        -DROCPROFSYS_USE_PAPI=ON
+        -DROCPROFSYS_USE_MPI=ON
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j32
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      registerROCmPackages: true
+      optSymLink: true
+      extraPaths: /home/user/workspace/rocm/bin:/home/user/workspace/rocm/llvm/bin
+
+- job: rocprofiler_systems_testing
+  dependsOn: rocprofiler_systems
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  timeoutInMinutes: 180
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    name: $(JOB_TEST_POOL)
+    demands: firstRenderDeviceAccess
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      registerROCmPackages: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+# build flags reference: https://rocm.docs.amd.com/projects/omnitrace/en/latest/install/install.html
+      extraBuildFlags: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCPROFSYS_BUILD_TESTING=ON
+        -DROCPROFSYS_BUILD_DYNINST=ON
+        -DROCPROFSYS_BUILD_LIBUNWIND=ON
+        -DROCPROFSYS_DISABLE_EXAMPLES="openmp-target"
+        -DDYNINST_BUILD_TBB=ON
+        -DDYNINST_BUILD_ELFUTILS=ON
+        -DDYNINST_BUILD_LIBIBERTY=ON
+        -DDYNINST_BUILD_BOOST=ON
+        -DROCPROFSYS_USE_PAPI=ON
+        -DROCPROFSYS_USE_MPI=ON
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j32
+  - task: Bash@3
+    displayName: Set up rocprofiler-systems env
+    inputs:
+      targetType: inline
+      script: source share/rocprofiler-systems/setup-env.sh
+      workingDirectory: build
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofiler-systems
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    condition: always()
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    condition: always()
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      registerROCmPackages: true
+      gpuTarget: $(JOB_GPU_TARGET)
+      optSymLink: true
+      extraPaths: /home/user/workspace/rocm/bin:/home/user/workspace/rocm/llvm/bin

.azuredevops/components/rocprofiler.yml

@@ -69,21 +69,11 @@ jobs:
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -92,6 +82,77 @@ jobs:
         -DENABLE_LDCONFIG=OFF
         -DUSE_PROF_API=1
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j32
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      extraEnvVars:
+        - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
+        - ROCM_PATH:::/home/user/workspace/rocm
+
+- job: rocprofiler_testing
+  dependsOn: rocprofiler
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofilerV1
+      testExecutable: LD_LIBRARY_PATH="$(Agent.BuildDirectory)/rocm/lib/rocprofiler:$(Agent.BuildDirectory)/rocm/share/rocprofiler/tests-v1/test" ./run.sh
+      testDir: $(Agent.BuildDirectory)/rocm/share/rocprofiler/tests-v1
+      testParameters: ''
+      testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofilerV2
+      testExecutable: LD_LIBRARY_PATH="$(Agent.BuildDirectory)/rocm/lib/rocprofiler:$(Agent.BuildDirectory)/rocm/share/rocprofiler/tests" share/rocprofiler/tests/runUnitTests
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: $(Agent.BuildDirectory)/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      optSymLink: true

.azuredevops/components/rocr_debug_agent.yml

@@ -14,6 +14,7 @@ parameters:
     - libdw-dev
     - libstdc++-12-dev
     - python-is-python3
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
@@ -39,8 +40,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -53,13 +53,8 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -68,11 +63,16 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
 
 - job: rocr_debug_agent_testing
   dependsOn: rocr_debug_agent
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -91,19 +91,11 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: rocr_debug_agent-tests
@@ -114,7 +106,13 @@ jobs:
       cmakeBuildDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'
       cmakeSourceDir: '.'
       installEnabled: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocr_debug_agent
       testDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/roctracer.yml

@@ -9,9 +9,11 @@ parameters:
   type: object
   default:
     - cmake
-    - ninja-build
     - doxygen
     - graphviz
+    - libdrm-amdgpu-dev
+    - ninja-build
+    - python3-pip
 - name: pipModules
   type: object
   default:
@@ -30,6 +32,7 @@ parameters:
   default:
     - clr
     - llvm-project
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
 
@@ -40,8 +43,7 @@ jobs:
   - template: /.azuredevops/variables-global.yml
   - name: HIP_ROCCLR_HOME
     value: $(Build.BinariesDirectory)/rocm
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -55,20 +57,16 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
       pipModules: ${{ parameters.pipModules }}
+      registerROCmPackages: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -77,14 +75,25 @@ jobs:
         -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      registerROCmPackages: true
 
 - job: roctracer_testing
   dependsOn: roctracer
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -100,24 +109,19 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      registerROCmPackages: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
-    parameters:
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmTestDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: roctracer
@@ -125,3 +129,10 @@ jobs:
       testParameters: ''
       testDir: $(Agent.BuildDirectory)
       testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)
+      registerROCmPackages: true

.azuredevops/components/rpp.yml

@@ -8,27 +8,49 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - cmake
-    - libomp-dev # needed to pass flag step
-    - ninja-build
     - clang
+    - cmake
+    - imagemagick
+    - libopencv-dev
+    - libsndfile1-dev
+    - libstdc++-12-dev
+    - ninja-build
+    - python3-pip
+- name: pipModules
+  type: object
+  default:
+    - openpyxl
+    - pandas
+    - sphinx
 - name: rocmDependencies
   type: object
   default:
-    - aomp # needed to pass build step
+    - aomp
     - clr
     - half
     - llvm-project
+    - rocm-cmake
     - rocminfo
     - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - half
+    - hipTensor
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rpp
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.LOW_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -47,14 +69,9 @@ jobs:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -64,8 +81,107 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DHALF_INCLUDE_DIRS=$(Agent.BuildDirectory)/rocm/include
         -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DROCM_PLATFORM_VERSION=$(NEXT_RELEASE_VERSION)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rpp_testing
+  dependsOn: rpp
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: LD_LIBRARY_PATH
+    value: $(Agent.BuildDirectory)/rocm/lib;$(Agent.BuildDirectory)/rocm/llvm/lib
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+  # Dependencies from: https://github.com/ROCm/rpp/blob/develop/utilities/test_suite/README.md
+  - task: Bash@3
+    displayName: Build and install Turbo JPEG
+    inputs:
+      targetType: 'inline'
+      script: |
+        sudo apt-get install nasm
+        sudo apt-get install wget
+        git clone -b 3.0.2 https://github.com/libjpeg-turbo/libjpeg-turbo.git
+        cd libjpeg-turbo
+        mkdir build
+        cd build
+        cmake -DCMAKE_INSTALL_PREFIX=/usr \
+              -DCMAKE_BUILD_TYPE=RELEASE  \
+              -DENABLE_STATIC=FALSE       \
+              -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib  \
+              -DWITH_JPEG8=TRUE           \
+              ..
+        make -j$nproc
+        sudo make install
+  - task: Bash@3
+    displayName: Build and install Nifti
+    inputs:
+      targetType: 'inline'
+      script: |
+        git clone -b v3.0.1 https://github.com/NIFTI-Imaging/nifti_clib.git
+        cd nifti_clib
+        mkdir build
+        cd build
+        cmake ..
+        sudo make -j$nproc install
+  - task: Bash@3
+    displayName: Build rpp tests
+    inputs:
+      targetType: 'inline'
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        mkdir rpp-tests
+        cd rpp-tests
+        cmake /opt/rocm/share/rpp/test \
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ \
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rpp
+      testExecutable: 'export PATH=$(Agent.BuildDirectory)/rocm/llvm/bin:$PATH; CC=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang CMAKE_VERBOSE_MAKEFILE=ON VERBOSE=1 ctest'
+      testDir: 'rpp-tests'
+  - script: sudo rm /opt/rocm
+    condition: always()
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+      environment: test
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/dependencies/boost.yml

@@ -0,0 +1,65 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: boostVersion
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - git
+- name: rocmDependencies
+  type: object
+  default:
+    - llvm-project
+
+jobs:
+- job: boost
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: Bash@3
+    displayName: 'git clone boost'
+    inputs:
+      targetType: inline
+      script: git clone -b ${{ parameters.boostVersion }} https://github.com/boostorg/boost --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      dependencySource: staging
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - task: Bash@3
+    displayName: 'Build Boost with clang'
+    inputs:
+      targetType: inline
+      script: |
+        export CC=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
+        export CXX=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
+        ./bootstrap.sh --with-toolset=clang --prefix=$(Build.BinariesDirectory)
+        ./b2 --toolset=clang threading=multi link=shared --prefix=$(Build.BinariesDirectory) cxxflags="-std=c++20"
+        ./b2 install
+      workingDirectory: $(Build.SourcesDirectory)/boost
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/dependencies/grpc.yml

@@ -20,7 +20,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/dependencies/gtest.yml

@@ -20,7 +20,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/dependencies/half560.yml

@@ -19,7 +19,7 @@ jobs:
   pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   container:
-    image: ${{ variables.DOCKER_IMAGE_NAME }}:${{ variables.LATEST_DOCKER_VERSION }}
+    image: rocm/dev-ubuntu-22.04:${{ variables.LATEST_RELEASE_VERSION }}
   workspace:
     clean: all
   steps:

.azuredevops/dependencies/lapack.yml

@@ -21,7 +21,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/nightly/pytorch.yml

@@ -11,7 +11,6 @@ parameters:
     - ca-certificates
     - bc
     - bridge-utils
-    - cmake
     - devscripts
     - dkms
     - doxygen
@@ -62,11 +61,14 @@ parameters:
     - ffmpeg
     - libopenblas-dev
     - liblapack-dev
+    - libswscale-dev
+    - libavformat-dev
 - name: pipModules
   type: object
   default:
+    - cmake
     - astunparse
-    - expecttest!=0.2.0
+    - expecttest>=0.2.1
     - hypothesis
     - numpy
     - psutil
@@ -83,11 +85,15 @@ parameters:
     - lintrunner
     - ninja
     - packaging
-    - optree>=0.12.0
+    - optree>=0.13.0
+    - click>=8.0.3
   # list for vision
     - auditwheel
     - future
-# list from https://github.com/pytorch/builder/blob/main/manywheel/build_rocm.sh
+    - pytest
+    - pytest-azurepipelines
+    - pillow
+# list from https://github.com/pytorch/pytorch/blob/main/.ci/manywheel/build_rocm.sh
 - name: rocmDependencies
   type: object
   default:
@@ -117,6 +123,29 @@ parameters:
     - hipCUB
     - rocThrust
     - hipBLAS-common
+    - composable_kernel
+- name: rocmTestDependencies
+  type: object
+  default:
+    - rocminfo
+# Reference on what tests to run for torchvision found in private repo:
+# https://github.com/ROCm/rocAutomation/blob/jenkins-pipelines/pytorch/pytorch_ci/test_pytorch_test1.sh#L54
+# Will iterate through this list using pytest
+- name: torchTestList
+  type: object
+  default:
+    - nn
+    - torch
+#    - cuda seg faults and might need cuda installed on test system
+#    - ops takes too long
+    - unary_ufuncs
+    - binary_ufuncs
+    - autograd
+#    - inductor/torchinductor takes too long
+# set to false to disable torchvision build and test
+- name: includeVision
+  type: boolean
+  default: false
 
 trigger: none
 pr: none
@@ -136,13 +165,10 @@ jobs:
       amd-staging-gfx942:
         ROCM_BRANCH: amd-staging
         JOB_GPU_TARGET: gfx942
-      amd-staging-gfx90a:
-        ROCM_BRANCH: amd-staging
-        JOB_GPU_TARGET: gfx90a
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-# various flags/parameters expected by bash scripts in pytorch builder repo
+# various flags/parameters expected by bash scripts in pytorch repo's .ci directory
   - name: ROCM_VERSION
     value: 6.3.0
   - name: ROCM_PATH
@@ -151,8 +177,6 @@ jobs:
     value: 6.3.0
   - name: MKLROOT
     value: /opt/intel
-  - name: AOTRITON_INSTALLED_PREFIX
-    value: /opt/rocm/aotriton
   - name: DESIRED_PYTHON
     value: 3.10
   - name: PYTORCH_ROOT
@@ -163,7 +187,7 @@ jobs:
   workspace:
     clean: all
   steps:
-# copy environment setup from https://github.com/pytorch/builder/blob/main/manywheel/Dockerfile
+# copy environment setup from https://github.com/pytorch/pytorch/blob/main/.ci/docker/manywheel/Dockerfile
 # but instead of centos, use ubuntu environment
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
   - task: Bash@3
@@ -200,18 +224,27 @@ jobs:
       targetType: inline
       script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
   - checkout: self
-  - task: Bash@3
-    displayName: git clone pytorch builder
-    inputs:
-      targetType: inline
-      script: git clone https://github.com/pytorch/builder.git --depth=1 --recurse-submodules
-      workingDirectory: $(Build.SourcesDirectory)
   - task: Bash@3
     displayName: git clone upstream pytorch
     inputs:
       targetType: inline
       script: git clone https://github.com/pytorch/pytorch.git --depth=1 --recurse-submodules
       workingDirectory: $(Build.SourcesDirectory)
+# builder clone still needed due to run_tests.sh at end of build_common.sh call
+  - task: Bash@3
+    displayName: git clone pytorch builder
+    inputs:
+      targetType: inline
+      script: |
+        git clone https://github.com/pytorch/builder.git --depth=1 --recurse-submodules
+        sudo ln -s $(Build.SourcesDirectory)/builder /builder
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Temporarily Patch CK Submodule
+    inputs:
+      targetType: inline
+      script: git pull origin develop
+      workingDirectory: $(Build.SourcesDirectory)/pytorch/third_party/composable_kernel
   - task: Bash@3
     displayName: Install patchelf
     inputs:
@@ -240,12 +273,6 @@ jobs:
       script: |
         sudo PYTORCH_ROCM_ARCH=$(JOB_GPU_TARGET) MKLROOT=$(MKLROOT) bash pytorch/.ci/docker/common/install_rocm_magma.sh
       workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: Install AOTriton Shared Library
-    inputs:
-      targetType: inline
-      script: sudo bash ./common/install_aotriton.sh /opt/rocm
-      workingDirectory: $(Build.SourcesDirectory)/pytorch/.ci/docker
   - task: Bash@3
     displayName: Run ROCm Build Script
     inputs:
@@ -258,64 +285,277 @@ jobs:
         DESIRED_PYTHON=$(DESIRED_PYTHON)
         PYTORCH_ROOT=$(PYTORCH_ROOT)
         CMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        AOTRITON_INSTALLED_PREFIX=$(AOTRITON_INSTALLED_PREFIX)
         DESIRED_DEVTOOLSET=$(DESIRED_DEVTOOLSET)
         TORCH_PACKAGE_NAME=torch.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
         PYTORCH_BUILD_VERSION=$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)
         PYTORCH_BUILD_NUMBER=$(date -u +%Y%m%d)
         SKIP_ALL_TESTS=1
-        bash ./manywheel/build_rocm.sh
-      workingDirectory: $(Build.SourcesDirectory)/builder
+        bash ./.ci/manywheel/build_rocm.sh
+      workingDirectory: $(Build.SourcesDirectory)/pytorch
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
       sourceDir: /remote/wheelhouserocm$(ROCM_VERSION)
       contentsString: '*.whl'
 # common helper source for pytorch vision and audio
-  - task: Bash@3
-    displayName: git clone pytorch test-infra
-    inputs:
-      targetType: inline
-      script: git clone https://github.com/pytorch/test-infra.git --depth=1 --recurse-submodules
-      workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: install package helper
-    inputs:
-      targetType: inline
-      script: python3 -m pip install test-infra/tools/pkg-helpers
-      workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: pytorch pkg helpers
-    inputs:
-      targetType: inline
-      script: CU_VERSION=${CU_VERSION} CHANNEL=${CHANNEL} python -m pytorch_pkg_helpers
-# get torch vision source and build
-  - task: Bash@3
-    displayName: git clone pytorch vision
-    inputs:
-      targetType: inline
-      script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
-      workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: Build vision
-    inputs:
-      targetType: inline
-      script: >-
-        BUILD_VERSION=$(ROCM_BRANCH).$(JOB_GPU_TARGET)-$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)$(date -u +%Y%m%d)
-        python3 setup.py bdist_wheel
-      workingDirectory: $(Build.SourcesDirectory)/vision
-  - task: Bash@3
-    displayName: Relocate vision
-    inputs:
-      targetType: inline
-      script: python3 packaging/wheel/relocate.py
-      workingDirectory: $(Build.SourcesDirectory)/vision
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+  - ${{ if eq(parameters.includeVision, true) }}:
+    - task: Bash@3
+      displayName: git clone pytorch test-infra
+      inputs:
+        targetType: inline
+        script: git clone https://github.com/pytorch/test-infra.git --depth=1 --recurse-submodules
+        workingDirectory: $(Build.SourcesDirectory)
+    - task: Bash@3
+      displayName: install package helper
+      inputs:
+        targetType: inline
+        script: python3 -m pip install test-infra/tools/pkg-helpers
+        workingDirectory: $(Build.SourcesDirectory)
+    - task: Bash@3
+      displayName: pytorch pkg helpers
+      inputs:
+        targetType: inline
+        script: CU_VERSION=${CU_VERSION} CHANNEL=${CHANNEL} python -m pytorch_pkg_helpers
+  # get torch vision source and build
+    - task: Bash@3
+      displayName: git clone pytorch vision
+      inputs:
+        targetType: inline
+        script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
+        workingDirectory: $(Build.SourcesDirectory)
+    - task: Bash@3
+      displayName: Build vision
+      inputs:
+        targetType: inline
+        script: >-
+          TORCH_PACKAGE_NAME=torch.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
+          TORCHVISION_PACKAGE_NAME=torchvision.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
+          PYTORCH_VERSION=$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)post$(date -u +%Y%m%d)
+          BUILD_VERSION=$(cat $(Build.SourcesDirectory)/vision/version.txt | cut -da -f1)post$(date -u +%Y%m%d)
+          python3 setup.py bdist_wheel
+        workingDirectory: $(Build.SourcesDirectory)/vision
+    - task: Bash@3
+      displayName: Relocate vision
+      inputs:
+        targetType: inline
+        script: python3 packaging/wheel/relocate.py
+        workingDirectory: $(Build.SourcesDirectory)/vision
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+      parameters:
+        sourceDir: $(Build.SourcesDirectory)/vision/dist
+        contentsString: '*.whl'
+        clean: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
     parameters:
-      sourceDir: $(Build.SourcesDirectory)/vision/dist
-      contentsString: '*.whl'
-      clean: false
+      gpuTarget: $(JOB_GPU_TARGET)
   - task: PublishPipelineArtifact@1
     displayName: 'wheel file Publish'
     retryCountOnTaskFailure: 3
     inputs:
       targetPath: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Save pipeline artifact file name
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: |
+        whlFile=$(find "$(Build.BinariesDirectory)" -type f -name "*.whl" | head -n 1)
+        if [ -n "$whlFile" ]; then
+          echo $(basename "$whlFile") >> pipelineArtifacts.txt
+        fi
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+
+- job: pytorch_testing
+  dependsOn: pytorch
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: PYTORCH_TEST_WITH_ROCM
+    value: 1
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - task: Bash@3
+    displayName: 'Register libjpeg-turbo packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
+        sudo apt update
+        apt-cache show libjpeg-turbo-official | grep Version
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+# pytorch tests require an updated version of click, even if requirements is not called outright
+  - task: Bash@3
+    displayName: 'pip update click'
+    inputs:
+      targetType: inline
+      script: pip install --upgrade click
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Wheel Files'
+    inputs:
+      itemPattern: '**/*$(JOB_GPU_TARGET)*.whl'
+      targetPath: $(Agent.BuildDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      dependencySource: staging
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      dependencySource: staging
+      skipLlvmSymlink: true
+# get sources to run test scripts
+  - task: Bash@3
+    displayName: git clone upstream pytorch
+    inputs:
+      targetType: inline
+      script: git clone https://github.com/pytorch/pytorch.git --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - ${{ if eq(parameters.includeVision, true) }}:
+    - task: Bash@3
+      displayName: git clone pytorch vision
+      inputs:
+        targetType: inline
+        script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
+        workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Install Wheel Files
+    inputs:
+      targetType: inline
+      script: find . -name "*.whl" -exec pip install --no-index --find-links=. --no-dependencies -v {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Show Updated pip List
+    inputs:
+      targetType: inline
+      script: pip list -v
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add Python site-packages binaries to path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.prependpath]$USER_BASE/bin"
+  - task: Bash@3
+    displayName: Add torch libs to ldconfig
+    inputs:
+      targetType: inline
+      script: |
+        echo $(python3 -m site --user-site)/torch/lib | sudo tee /etc/ld.so.conf.d/torch.conf
+        sudo ldconfig -v
+        ldconfig -p
+# https://pytorch.org/get-started/locally/#linux-verification
+# https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/3rd-party/pytorch-install.html#testing-the-pytorch-installation
+  - task: Bash@3
+    displayName: Simple Import Torch Tests
+    inputs:
+      targetType: inline
+      script: |
+        python3 -c 'import torch' 2> /dev/null && echo 'Success' || echo 'Failure'
+        python3 -c 'import torch; print(torch.cuda.is_available())'
+        python3 -c 'import torch; x = torch.rand(5, 3); print(x)'
+# Test artifact build script has too many if statements for different environments
+# Based off the snippet of interest for this environment, with some adjustments
+# https://github.com/pytorch/pytorch/blob/main/.ci/pytorch/build.sh#L330-L371
+# Removing in-line comments since it does not fit with the yaml markup
+  - task: Bash@3
+    displayName: Build Pytorch Test Artifacts
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: |
+        CUSTOM_TEST_ARTIFACT_BUILD_DIR="build/custom_test_artifacts"
+        CUSTOM_TEST_USE_ROCM=ON
+        CUSTOM_TEST_MODULE_PATH="${PWD}/cmake/public"
+        mkdir -pv "${CUSTOM_TEST_ARTIFACT_BUILD_DIR}"
+
+        CUSTOM_OP_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-op-build"
+        CUSTOM_OP_TEST="${PWD}/test/custom_operator"
+        python --version
+        SITE_PACKAGES="$(python -c 'import site; print(";".join([x for x in site.getsitepackages()] + [x + "/torch" for x in site.getsitepackages()]))')"
+
+        mkdir -p "$CUSTOM_OP_BUILD"
+        pushd "$CUSTOM_OP_BUILD"
+        cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
+              -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
+        make VERBOSE=1
+        popd
+
+        JIT_HOOK_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/jit-hook-build"
+        JIT_HOOK_TEST="$PWD/test/jit_hooks"
+        python --version
+        SITE_PACKAGES="$(python -c 'import site; print(";".join([x for x in site.getsitepackages()] + [x + "/torch" for x in site.getsitepackages()]))')"
+        mkdir -p "$JIT_HOOK_BUILD"
+        pushd "$JIT_HOOK_BUILD"
+        cmake "$JIT_HOOK_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
+              -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
+        make VERBOSE=1
+        popd
+
+        CUSTOM_BACKEND_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-backend-build"
+        CUSTOM_BACKEND_TEST="${PWD}/test/custom_backend"
+        python --version
+        mkdir -p "$CUSTOM_BACKEND_BUILD"
+        pushd "$CUSTOM_BACKEND_BUILD"
+        cmake "$CUSTOM_BACKEND_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
+              -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
+        make VERBOSE=1
+        popd
+      workingDirectory: $(Build.SourcesDirectory)/pytorch
+  - ${{ each torchTest in parameters.torchTestList }}:
+    - task: Bash@3
+      displayName: Test ${{ torchTest }}
+      continueOnError: true
+      inputs:
+        targetType: inline
+        workingDirectory: $(Build.SourcesDirectory)/pytorch
+        ${{ if contains(torchTest, '/') }}:
+          script: pytest test/${{ split(torchTest, '/')[0] }}/test_${{ split(torchTest, '/')[1] }}.py
+        ${{ else }}:
+          script: pytest test/test_${{ torchTest }}.py
+# Reference on what tests to run for torchvision found in private repo:
+# https://github.com/ROCm/rocAutomation/blob/jenkins-pipelines/pytorch/pytorch_ci/test_torchvision.sh#L51
+  - ${{ if eq(parameters.includeVision, true) }}:
+    - task: Bash@3
+      displayName: Test vision/transforms
+      continueOnError: true
+      inputs:
+        targetType: inline
+        script: pytest test/test_transforms.py
+        workingDirectory: $(Build.SourcesDirectory)/vision
+  - task: Bash@3
+    displayName: Uninstall Wheel Files
+    inputs:
+      targetType: inline
+      script: find . -name "*.whl" -exec pip uninstall -y {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Remove Python site-packages binaries from path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$USER_BASE/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"

.azuredevops/nightly/rocm-nightly.yml

@@ -1,5 +1,10 @@
 parameters:
-# currently excludes clr and rocm-examples
+- name: dependencySource
+  type: string
+  default: staging
+  values: 
+    - staging
+    - mainline
 - name: rocmDependencies
   type: object
   default:
@@ -10,6 +15,7 @@ parameters:
     - composable_kernel
     - half
     - HIP
+    - hip-tests
     - hipBLAS
     - hipBLAS-common
     - hipBLASLt
@@ -34,16 +40,22 @@ parameters:
     - rocDecode
     - rocFFT
     - ROCgdb
+    - rocJPEG
     - rocm-cmake
     - rocm-core
+    - rocm-examples
     - rocminfo
     - rocMLIR
     - ROCmValidationSuite
     - rocm_bandwidth_test
     - rocm_smi_lib
     - rocPRIM
+    - rocprofiler-compute
     - rocprofiler-register
+    - rocprofiler-sdk
+    - rocprofiler-systems
     - rocprofiler
+    - rocPyDecode
     - ROCR-Runtime
     - rocRAND
     - rocr_debug_agent
@@ -53,6 +65,7 @@ parameters:
     - roctracer
     - rocWMMA
     - rpp
+    - TransferBench
 
 trigger: none
 pr: none
@@ -76,8 +89,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - task: DeleteFiles@1
     displayName: 'Cleanup checkout space'
@@ -94,14 +105,16 @@ jobs:
     displayName: System disk space before ROCm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
+      dependencySource: ${{ parameters.dependencySource }}
       dependencyList: ${{ parameters.rocmDependencies }}
-      dependencySource: staging
-      skipLibraryLinking: true
       gpuTarget: $(JOB_GPU_TARGET)
+      skipLibraryLinking: true
+      skipLlvmSymlink: true
   - script: df -h
     displayName: System disk space after ROCm
   - script: du -sh $(Agent.BuildDirectory)/rocm
     displayName: Uncompressed ROCm size
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - task: ArchiveFiles@2
     displayName: Compress rocm-nightly
     inputs:
@@ -113,7 +126,14 @@ jobs:
   - script: du -sh $(Build.ArtifactStagingDirectory)
     displayName: Compressed ROCm size
   - task: PublishPipelineArtifact@1
-    displayName: 'Public ROCm Nightly Artifact'
+    displayName: 'Publish ROCm Nightly Artifact'
     retryCountOnTaskFailure: 3
     inputs:
       targetPath: '$(Build.ArtifactStagingDirectory)'
+  - task: Bash@3
+    displayName: Save pipeline artifact file name
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: echo "$(Build.DefinitionName)_$(Build.BuildNumber)_ubuntu2204_$(JOB_GPU_TARGET).tar.gz" >> pipelineArtifacts.txt
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml

.azuredevops/tag-builds/Tensile.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/Tensile
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/Tensile.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/TransferBench.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/TransferBench
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/TransferBench.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/boost.yml

@@ -0,0 +1,23 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: boostVersion
+  type: string
+  default: 'master'
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_DEPENDENCIES_PATH }}/boost.yml
+    parameters:
+      boostVersion: ${{ parameters.boostVersion }}

.azuredevops/tag-builds/omnitrace.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/omnitrace
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/omnitrace.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocJPEG.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocJPEG
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocJPEG.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocprofiler-compute.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocprofiler-compute
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocprofiler-compute.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocprofiler-sdk.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocprofiler-sdk
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocprofiler-sdk.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocprofiler-systems.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocprofiler-systems
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocprofiler-systems.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/templates/steps/artifact-download.yml

@@ -6,88 +6,24 @@ parameters:
 - name: pipelineId
   type: string
   default: ''
-- name: useDefaultBranch
-  type: boolean
-  default: true
-- name: latestFromBranch
-  type: boolean
-  default: true
-- name: extractToMnt
-  type: boolean
-  default: false
-- name: fileFilter
-  type: string
-  default: ''
-- name: defaultBranchList
-  type: object
-  default:
-    AMDMIGraphX: develop
-    amdsmi: develop
-    aomp-extras: aomp-dev
-    aomp: aomp-dev
-    clr: amd-staging
-    composable_kernel: develop
-    half: rocm
-    HIP: amd-staging
-    hipBLAS: develop
-    hipBLASLt: develop
-    hipBLAS-common: develop
-    hipCUB: develop
-    hipFFT: develop
-    hipfort: develop
-    HIPIFY: amd-staging
-    hipRAND: develop
-    hipSOLVER: develop
-    hipSPARSE: develop
-    hipSPARSELt: develop
-    hipTensor: develop
-    llvm-project: amd-staging
-    MIOpen: develop
-    MIVisionX: develop
-    rccl: develop
-    rdc: develop
-    rocAL: develop
-    rocALUTION: develop
-    rocBLAS: develop
-    ROCdbgapi : amd-staging
-    rocDecode: develop
-    rocFFT: develop
-    ROCgdb: amd-staging
-    rocm-cmake: develop
-    rocm-core: master
-    rocm-examples: develop
-    rocminfo: amd-staging
-    rocMLIR: develop
-    ROCmValidationSuite: master
-    rocm_bandwidth_test: master
-    rocm_smi_lib: develop
-    rocPRIM: develop
-    rocprofiler-register: amd-mainline
-    rocprofiler: amd-staging
-    ROCR-Runtime: amd-staging
-    rocRAND: develop
-    rocr_debug_agent: amd-staging
-    rocSOLVER: develop
-    rocSPARSE: develop
-    rocThrust: develop
-    roctracer: amd-staging
-    rocWMMA: develop
-    rpp: develop
-- name: allowPartiallySucceededBuilds
-  type: object
-  default:
-    - rocm-cmake
-    - amdsmi
-    - HIPIFY
-    - rocm_smi_lib
-    - rocFFT
-    - MIVisionX
-# BELOW REQUIRED IF useDefaultBranch false
 - name: branchName
   type: string
   default: '$(Build.SourceBranchName)' # for tagged builds
+- name: fileFilter
+  type: string
+  default: ''
 
 steps:
+- task: Bash@3
+  displayName: Set allowPartiallySucceededBuilds
+  inputs:
+    targetType: inline
+    script: |
+      if [[ ",$ALLOWED_PARTIAL_SUCCEED_BUILDS," == *",${{ parameters.componentName }},"* ]]; then
+        echo "##vso[task.setvariable variable=allowPartiallySucceededBuilds;]true"
+      else
+        echo "##vso[task.setvariable variable=allowPartiallySucceededBuilds;]false"
+      fi
 - task: DownloadPipelineArtifact@2
   displayName: Download ${{ parameters.componentName }}
   inputs:
@@ -96,24 +32,16 @@ steps:
     definition: ${{ parameters.pipelineId }}
     specificBuildWithTriggering: true
     itemPattern: '**/*${{ parameters.fileFilter }}*'
-    ${{ if eq(parameters.latestFromBranch, true) }}:
-      ${{ if notIn(parameters.componentName, 'aomp') }}: # remove this once these pipelines are functional + up-to-date
-        buildVersionToDownload: latestFromBranch # default is 'latest'
-    ${{ if eq(parameters.useDefaultBranch, true) }}:
-      branchName: refs/heads/${{ parameters.defaultBranchList[parameters.componentName] }}
-    ${{ else }}:
-      branchName: ${{ parameters.branchName }}
-    ${{ if containsValue(parameters.allowPartiallySucceededBuilds, parameters.componentName) }}:
-      allowPartiallySucceededBuilds: true
+    ${{ if notIn(parameters.componentName, 'aomp') }}: # remove this once these pipelines are functional + up-to-date
+      buildVersionToDownload: latestFromBranch # default is 'latest'
+    branchName: refs/heads/${{ parameters.branchName }}
+    allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
     targetPath: '$(Pipeline.Workspace)/d'
 - task: ExtractFiles@1
   displayName: Extract ${{ parameters.componentName }}
   inputs:
     archiveFilePatterns: '$(Pipeline.Workspace)/d/**/*.tar.gz'
-    ${{ if parameters.extractToMnt }}:
-      destinationFolder: '/mnt/rocm'
-    ${{ else }}:
-      destinationFolder: '$(Agent.BuildDirectory)/rocm'
+    destinationFolder: '$(Agent.BuildDirectory)/rocm'
     cleanDestinationFolder: false
     overwriteExistingFiles: true
 - task: DeleteFiles@1

.azuredevops/templates/steps/artifact-links.yml

@@ -0,0 +1,32 @@
+# Every publish artifact call should be coupled with a script that
+# prints out the artifact name to a common text file
+# This template parses that text file line by line and prints out the download URL
+# for each artifact, so that they are easily accessible to the public
+# replace trailing '=' with their count in the encoded string
+steps:
+- task: Bash@3
+  displayName: "!! Download Links !!"
+  condition: always()
+  continueOnError: true
+  inputs:
+    workingDirectory: $(Pipeline.Workspace)
+    targetType: inline
+    script: |
+      URL_BEGIN="https://artprodcus3.artifacts.visualstudio.com/"
+      URL_MIDDLE="/_apis/artifact/"
+      URL_END="/content?format=file&subPath=%2F"
+      FORMATTED_JOB_NAME=$(echo $(Agent.JobName) | sed 's/ /./g; s/[-_]//g')
+      ARTIFACT_STRING="pipelineartifact://ROCm-CI/projectId/$(DOWNLOAD_PROJECT_ID)/buildId/$(Build.BuildId)/artifactName/${FORMATTED_JOB_NAME}"
+      ENCODED_STRING=$(echo -n "${ARTIFACT_STRING}" | base64 -w 0)
+      PADDING_COUNT=$(echo -n "${ENCODED_STRING}" | awk -F= '{print NF-1}')
+      if [ "$PADDING_COUNT" -gt 0 ]; then
+        FINAL_ENCODED_STRING=$(echo -n "${ENCODED_STRING}" | sed "s/=*$/${PADDING_COUNT}/")
+      else
+        FINAL_ENCODED_STRING="${ENCODED_STRING}0"
+      fi
+      while IFS= read -r fileName; do
+        echo "File Name:"
+        echo "$fileName"
+        printf "Download Link:\n%s%s/%s%s%s%s%s\n\n" "${URL_BEGIN}" "${DOWNLOAD_ORGANIZATION_ID}" "${DOWNLOAD_PROJECT_ID}" "${URL_MIDDLE}" "${FINAL_ENCODED_STRING}" "${URL_END}" "${fileName}"
+      done < pipelineArtifacts.txt
+      rm pipelineArtifacts.txt

.azuredevops/templates/steps/artifact-upload.yml

@@ -27,6 +27,12 @@ steps:
     SourceFolder: '$(Build.BinariesDirectory)'
     Contents: '/**/*'
     RemoveDotFiles: true
+- task: Bash@3
+  displayName: Save pipeline artifact file name
+  inputs:
+    workingDirectory: $(Pipeline.Workspace)
+    targetType: inline
+    script: echo "$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.tar.gz" >> pipelineArtifacts.txt
 # then publish it
 - ${{ if parameters.publish }}:
   - task: PublishPipelineArtifact@1

.azuredevops/templates/steps/build-autotools.yml

@@ -11,6 +11,9 @@ parameters:
 - name: installDir
   type: string
   default: '$(Build.BinariesDirectory)'
+- name: makeCallPrefix
+  type: string
+  default: ''
 
 steps:
 - task: Bash@3
@@ -23,7 +26,7 @@ steps:
   displayName: '${{ parameters.componentName }} make'
   inputs:
     targetType: inline
-    script: make -j$(nproc)
+    script: ${{ parameters.makeCallPrefix }} make -j$(nproc)
     workingDirectory: ${{ parameters.buildDir }}
 - task: Bash@3
   displayName: '${{ parameters.componentName }} make install'

.azuredevops/templates/steps/dependencies-aqlprofile.yml

@@ -1,27 +1,16 @@
-parameters:
-- name: dependencySource
-  type: string
-  default: staging
-  values:
-    - staging
-    - tag-builds
-- name: repositoryUrl
-  type: object
-  default: 
-    staging: https://repo.radeon.com/rocm/misc/aqlprofile/ubuntu-22.04
-    tag-builds: https://repo.radeon.com/rocm/apt/6.1/pool/main/h/hsa-amd-aqlprofile
-- name: packageName
-  type: object
-  default:
-    staging: hsa-amd-aqlprofile_1.0.0.60200.60200-crdnnh.14213~22.04_amd64.deb
-    tag-builds: hsa-amd-aqlprofile_1.0.0.60100.60100-82~22.04_amd64.deb
-
 steps:
+- task: Bash@3
+  displayName: Get aqlprofile package name
+  inputs:
+    targetType: inline
+    script: |
+      export packageName=$(curl -s https://repo.radeon.com/rocm/apt/$(REPO_RADEON_VERSION)/pool/main/h/hsa-amd-aqlprofile/ | grep -oP "href=\"\K[^\"]*$(lsb_release -rs)[^\"]*\.deb")
+      echo "##vso[task.setvariable variable=packageName;isreadonly=true]$packageName"
 - task: Bash@3
   displayName: 'Download aqlprofile'
   inputs:
     targetType: inline
-    script: wget -nv ${{ parameters.repositoryUrl[parameters.dependencySource] }}/${{ parameters.packageName[parameters.dependencySource] }}
+    script: wget -nv https://repo.radeon.com/rocm/apt/$(REPO_RADEON_VERSION)/pool/main/h/hsa-amd-aqlprofile/$(packageName)
     workingDirectory: '$(Pipeline.Workspace)'
 - task: Bash@3
   displayName: 'Extract aqlprofile'
@@ -29,7 +18,7 @@ steps:
     targetType: inline
     script: |
       mkdir hsa-amd-aqlprofile
-      dpkg-deb -R ${{ parameters.packageName[parameters.dependencySource] }} hsa-amd-aqlprofile
+      dpkg-deb -R $(packageName) hsa-amd-aqlprofile
     workingDirectory: '$(Pipeline.Workspace)'
 - task: Bash@3
   displayName: 'Copy aqlprofile files'
@@ -43,5 +32,5 @@ steps:
   displayName: 'Clean up aqlprofile'
   inputs:
     targetType: inline
-    script: rm -rf hsa-amd-aqlprofile ${{ parameters.packageName[parameters.dependencySource] }}
+    script: rm -rf hsa-amd-aqlprofile $(packageName)
     workingDirectory: '$(Pipeline.Workspace)'

.azuredevops/templates/steps/dependencies-boost.yml

@@ -0,0 +1,35 @@
+steps:
+- task: DownloadPipelineArtifact@2
+  displayName: Download Boost
+  inputs:
+    buildType: specific
+    project: ROCm-CI
+    definition: $(BOOST_DEPENDENCY_PIPELINE_ID)
+    targetPath: $(Pipeline.Workspace)/d
+- task: ExtractFiles@1
+  displayName: Extract Boost
+  inputs:
+    archiveFilePatterns: '$(Pipeline.Workspace)/d/**/*.tar.gz'
+    destinationFolder: $(Agent.BuildDirectory)/boost
+    cleanDestinationFolder: true
+    overwriteExistingFiles: true
+- task: DeleteFiles@1
+  displayName: Cleanup Compressed Boost
+  inputs:
+    SourceFolder: $(Pipeline.Workspace)/d
+    Contents: '**/*.tar.gz'
+    RemoveDotFiles: true
+- task: Bash@3
+  displayName: 'List Boost files'
+  inputs:
+    targetType: inline
+    script: ls -1R $(Agent.BuildDirectory)/boost
+- task: Bash@3
+  displayName: 'Link Boost shared libraries'
+  inputs:
+    targetType: inline
+    script: |
+      echo $(Agent.BuildDirectory)/boost/lib | sudo tee /etc/ld.so.conf.d/boost.conf
+      sudo cat /etc/ld.so.conf.d/boost.conf
+      sudo ldconfig -v
+      ldconfig -p

.azuredevops/templates/steps/dependencies-other.yml

@@ -6,8 +6,30 @@ parameters:
 - name: pipModules
   type: object
   default: []
+- name: registerROCmPackages
+  type: boolean
+  default: false
 
 steps:
+- ${{ if eq(parameters.registerROCmPackages, true) }}:
+  - task: Bash@3
+    displayName: 'Register AMDGPU & ROCm repos'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/$(REPO_RADEON_VERSION)/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/amdgpu.list
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$(REPO_RADEON_VERSION) jammy main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
+        echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+        sudo apt update
+# firefox takes time to upgrade and is not needed for CI workloads, hold version
+- task: Bash@3
+  continueOnError: true
+  displayName: 'sudo apt-mark hold firefox'
+  inputs:
+    targetType: inline
+    script: sudo apt-mark hold firefox
 - task: Bash@3
   displayName: 'sudo apt-get update'
   inputs:
@@ -34,4 +56,4 @@ steps:
     displayName: 'pip install  ...'
     inputs:
       targetType: inline
-      script: pip install ${{ join(' ', parameters.pipModules) }}
+      script: pip install -v ${{ join(' ', parameters.pipModules) }}

.azuredevops/templates/steps/dependencies-rocm.yml

@@ -1,143 +1,21 @@
 # download and install rocm dependencies through pipeline builds in the project
-# REQUIRED
 parameters:
+- name: checkoutRef
+  type: string
+  default: ''
+- name: dependencySource # optional, overrides checkoutRef
+  type: string
+  default: null
+  values:
+    - null # empty strings aren't allowed as values, use null instead
+    - staging
+    - mainline
 - name: dependencyList
   type: object
   default: []
-- name: dependencySource
-  type: string
-  default: staging
-  values:
-    - staging
-    - tag-builds
-    - fixed
-- name: extractToMnt
-  type: boolean
-  default: false
-# required values for fixed selection
-- name: fixedPipelineIdentifier
-  type: string
-  default: 0
-- name: fixedComponentName
+- name: gpuTarget
   type: string
   default: ''
-- name: latestFromBranch
-  type: boolean
-  default: true
-# match case of the repo in this object for the left side of the maps
-# should not need to replace these parameters
-- name: stagingPipelineIdentifiers
-  type: object
-  default:
-    AMDMIGraphX: $(AMDMIGRAPHX_PIPELINE_ID)
-    amdsmi: $(AMDSMI_PIPELINE_ID)
-    aomp-extras: $(AOMP_EXTRAS_PIPELINE_ID)
-    aomp: $(AOMP_PIPELINE_ID)
-    clr: $(CLR_PIPELINE_ID)
-    composable_kernel: $(COMPOSABLE_KERNEL_PIPELINE_ID)
-    half: $(HALF_PIPELINE_ID)
-    HIP: $(HIP_PIPELINE_ID)
-    hipBLAS: $(HIPBLAS_PIPELINE_ID)
-    hipBLAS-common: $(HIPBLAS_COMMON_PIPELINE_ID)
-    hipBLASLt: $(HIPBLASLT_PIPELINE_ID)
-    hipCUB: $(HIPCUB_PIPELINE_ID)
-    hipFFT: $(HIPFFT_PIPELINE_ID)
-    hipfort: $(HIPFORT_PIPELINE_ID)
-    HIPIFY: $(HIPIFY_PIPELINE_ID)
-    hipRAND: $(HIPRAND_PIPELINE_ID)
-    hipSOLVER: $(HIPSOLVER_PIPELINE_ID)
-    hipSPARSE: $(HIPSPARSE_PIPELINE_ID)
-    hipSPARSELt: $(HIPSPARSELT_PIPELINE_ID)
-    hipTensor: $(HIPTENSOR_PIPELINE_ID)
-    llvm-project: $(LLVM_PROJECT_PIPELINE_ID)
-    MIOpen: $(MIOpen_PIPELINE_ID)
-    MIVisionX: $(MIVISIONX_PIPELINE_ID)
-    rccl: $(RCCL_PIPELINE_ID)
-    rdc: $(RDC_PIPELINE_ID)
-    rocAL: $(ROCAL_PIPELINE_ID)
-    rocALUTION: $(ROCALUTION_PIPELINE_ID)
-    rocBLAS: $(ROCBLAS_PIPELINE_ID)
-    ROCdbgapi: $(ROCDBGAPI_PIPELINE_ID)
-    rocDecode: $(ROCDECODE_PIPELINE_ID)
-    rocFFT: $(ROCFFT_PIPELINE_ID)
-    ROCgdb: $(ROCGDB_PIPELINE_ID)
-    rocm-cmake: $(ROCM_CMAKE_PIPELINE_ID)
-    rocm-core: $(ROCM_CORE_PIPELINE_ID)
-    rocm-examples: $(ROCM_EXAMPLES_PIPELINE_ID)
-    rocminfo: $(ROCMINFO_PIPELINE_ID)
-    rocMLIR: $(ROCMLIR_PIPELINE_ID)
-    ROCmValidationSuite: $(ROCMVALIDATIONSUITE_PIPELINE_ID)
-    rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_PIPELINE_ID)
-    rocm_smi_lib: $(ROCM_SMI_LIB_PIPELINE_ID)
-    rocPRIM: $(ROCPRIM_PIPELINE_ID)
-    rocprofiler-register: $(ROCPROFILER_REGISTER_PIPELINE_ID)
-    rocprofiler: $(ROCPROFILER_PIPELINE_ID)
-    ROCR-Runtime: $(ROCR_RUNTIME_PIPELINE_ID)
-    rocRAND: $(ROCRAND_PIPELINE_ID)
-    rocr_debug_agent: $(ROCR_DEBUG_AGENT_PIPELINE_ID)
-    rocSOLVER: $(ROCSOLVER_PIPELINE_ID)
-    rocSPARSE: $(ROCSPARSE_PIPELINE_ID)
-    ROCT-Thunk-Interface: $(ROCT_THUNK_INTERFACE_PIPELINE_ID)
-    rocThrust: $(ROCTHRUST_PIPELINE_ID)
-    roctracer: $(ROCTRACER_PIPELINE_ID)
-    rocWMMA: $(ROCWMMA_PIPELINE_ID)
-    rpp: $(RPP_PIPELINE_ID)
-- name: taggedPipelineIdentifiers
-  type: object
-  default:
-    AMDMIGraphX: $(AMDMIGRAPHX_TAGGED_PIPELINE_ID)
-    amdsmi: $(AMDSMI_TAGGED_PIPELINE_ID)
-    aomp-extras: $(AOMP_EXTRAS_TAGGED_PIPELINE_ID)
-    aomp: $(AOMP_TAGGED_PIPELINE_ID)
-    clr: $(CLR_TAGGED_PIPELINE_ID)
-    composable_kernel: $(COMPOSABLE_KERNEL_TAGGED_PIPELINE_ID)
-    half: $(HALF_TAGGED_PIPELINE_ID)
-    HIP: $(HIP_TAGGED_PIPELINE_ID)
-    hipBLAS: $(HIPBLAS_TAGGED_PIPELINE_ID)
-    hipBLAS-common: $(HIPBLAS_COMMON_TAGGED_PIPELINE_ID)
-    hipBLASLt: $(HIPBLASLT_TAGGED_PIPELINE_ID)
-    hipCUB: $(HIPCUB_TAGGED_PIPELINE_ID)
-    hipFFT: $(HIPFFT_TAGGED_PIPELINE_ID)
-    hipfort: $(HIPFORT_TAGGED_PIPELINE_ID)
-    HIPIFY: $(HIPIFY_TAGGED_PIPELINE_ID)
-    hipRAND: $(HIPRAND_TAGGED_PIPELINE_ID)
-    hipSOLVER: $(HIPSOLVER_TAGGED_PIPELINE_ID)
-    hipSPARSE: $(HIPSPARSE_TAGGED_PIPELINE_ID)
-    hipSPARSELt: $(HIPSPARSELT_TAGGED_PIPELINE_ID)
-    hipTensor: $(HIPTENSOR_TAGGED_PIPELINE_ID)
-    llvm-project: $(LLVM_PROJECT_TAGGED_PIPELINE_ID)
-    MIOpen: $(MIOpen_TAGGED_PIPELINE_ID)
-    MIVisionX: $(MIVISIONX_TAGGED_PIPELINE_ID)
-    rccl: $(RCCL_TAGGED_PIPELINE_ID)
-    rdc: $(RDC_TAGGED_PIPELINE_ID)
-    rocAL: $(ROCAL_TAGGED_PIPELINE_ID)
-    rocALUTION: $(ROCALUTION_TAGGED_PIPELINE_ID)
-    rocBLAS: $(ROCBLAS_TAGGED_PIPELINE_ID)
-    ROCdbgapi: $(ROCDBGAPI_TAGGED_PIPELINE_ID)
-    rocDecode: $(ROCDECODE_TAGGED_PIPELINE_ID)
-    rocFFT: $(ROCFFT_TAGGED_PIPELINE_ID)
-    ROCgdb: $(ROCGDB_TAGGED_PIPELINE_ID)
-    rocm-cmake: $(ROCM_CMAKE_TAGGED_PIPELINE_ID)
-    rocm-core: $(ROCM_CORE_TAGGED_PIPELINE_ID)
-    rocm-examples: $(ROCM_EXAMPLES_TAGGED_PIPELINE_ID)
-    rocminfo: $(ROCMINFO_TAGGED_PIPELINE_ID)
-    rocMLIR: $(ROCMLIR_TAGGED_PIPELINE_ID)
-    ROCmValidationSuite: $(ROCMVALIDATIONSUITE_TAGGED_PIPELINE_ID)
-    rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_TAGGED_PIPELINE_ID)
-    rocm_smi_lib: $(ROCM_SMI_LIB_TAGGED_PIPELINE_ID)
-    rocPRIM: $(ROCPRIM_TAGGED_PIPELINE_ID)
-    rocprofiler-register: $(ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID)
-    rocprofiler: $(ROCPROFILER_TAGGED_PIPELINE_ID)
-    ROCR-Runtime: $(ROCR_RUNTIME_TAGGED_PIPELINE_ID)
-    rocRAND: $(ROCRAND_TAGGED_PIPELINE_ID)
-    rocr_debug_agent: $(ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID)
-    rocSOLVER: $(ROCSOLVER_TAGGED_PIPELINE_ID)
-    rocSPARSE: $(ROCSPARSE_TAGGED_PIPELINE_ID)
-    ROCT-Thunk-Interface: $(ROCT_THUNK_INTERFACE_TAGGED_PIPELINE_ID)
-    rocThrust: $(ROCTHRUST_TAGGED_PIPELINE_ID)
-    roctracer: $(ROCTRACER_TAGGED_PIPELINE_ID)
-    rocWMMA: $(ROCWMMA_TAGGED_PIPELINE_ID)
-    rpp: $(RPP_TAGGED_PIPELINE_ID)
 # set to true if you're calling this template file multiple files in same pipeline
 # only leave last call false to optimize sequence
 - name: skipLibraryLinking
@@ -148,46 +26,320 @@ parameters:
 - name: skipLlvmSymlink
   type: boolean
   default: false
-# some ROCm components can specify GPU target and this will affect downloads
-- name: gpuTarget
-  type: string
-  default: ''
-# array of ROCm components that can specify GPU target or have dependency of such a component
-# these components would have parallel build jobs per GPU target so they produce multiple artifacts
-# only download artifact tied to the GPU target
-- name: componentsWithGPUTarget
+# set to true if dlopen calls for HIP libraries are causing failures
+# because they do not follow shared library symlink convention
+- name: setupHIPLibrarySymlinks
+  type: boolean
+  default: false
+
+- name: componentVarList
   type: object
   default:
-    - AMDMIGraphX
-    - composable_kernel
-    - hipBLASLt
-    - hipCUB
-    - hipFFT
-    - hipRAND
-    - hipSPARSELt
-    - hipTensor
-    - rccl
-    - rocALUTION
-    - rocBLAS
-    - rocFFT
-    - rocm-examples
-    - rocPRIM
-    - rocprofiler
-    - rocRAND
-    - rocSOLVER
-    - rocSPARSE
-    - rocThrust
-    - roctracer
-    - rocWMMA
-    - rpp
-# list below do not have flag for gpu target but have dependencies of components that do, so build separately per gpu target
-    - hipBLAS
-    - hipSOLVER
-    - hipSPARSE
-    - MIOpen
-    - MIVision
-    - rocAL
-    - ROCmValidationSuite
+    AMDMIGraphX:
+      pipelineId: $(AMDMIGRAPHX_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    amdsmi:
+      pipelineId: $(AMDSMI_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: false
+    aomp-extras:
+      pipelineId: $(AOMP_EXTRAS_PIPELINE_ID)
+      stagingBranch: aomp-dev
+      mainlineBranch: aomp-dev
+      hasGpuTarget: false
+    aomp:
+      pipelineId: $(AOMP_PIPELINE_ID)
+      stagingBranch: aomp-dev
+      mainlineBranch: amd-mainline-open
+      hasGpuTarget: false
+    clr:
+      pipelineId: $(CLR_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: false
+    composable_kernel:
+      pipelineId: $(COMPOSABLE_KERNEL_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    half:
+      pipelineId: $(HALF_PIPELINE_ID)
+      stagingBranch: rocm
+      mainlineBranch: rocm
+      hasGpuTarget: false
+    HIP:
+      pipelineId: $(HIP_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: false
+    hip-tests:
+      pipelineId: $(HIP_TESTS_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: false
+    hipBLAS:
+      pipelineId: $(HIPBLAS_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    hipBLASLt:
+      pipelineId: $(HIPBLASLT_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    hipBLAS-common:
+      pipelineId: $(HIPBLAS_COMMON_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: false
+    hipCUB:
+      pipelineId: $(HIPCUB_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    hipFFT:
+      pipelineId: $(HIPFFT_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    hipfort:
+      pipelineId: $(HIPFORT_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: false
+    HIPIFY:
+      pipelineId: $(HIPIFY_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: false
+    hipRAND:
+      pipelineId: $(HIPRAND_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    hipSOLVER:
+      pipelineId: $(HIPSOLVER_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    hipSPARSE:
+      pipelineId: $(HIPSPARSE_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    hipSPARSELt:
+      pipelineId: $(HIPSPARSELT_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    hipTensor:
+      pipelineId: $(HIPTENSOR_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    llvm-project:
+      pipelineId: $(LLVM_PROJECT_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline-open
+      hasGpuTarget: false
+    MIOpen:
+      pipelineId: $(MIOpen_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    MIVisionX:
+      pipelineId: $(MIVISIONX_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    omnitrace: # deprecated
+      pipelineId: $(OMNITRACE_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: true
+    rccl:
+      pipelineId: $(RCCL_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    rdc:
+      pipelineId: $(RDC_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: false
+    rocAL:
+      pipelineId: $(ROCAL_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: master
+      hasGpuTarget: true
+    rocALUTION:
+      pipelineId: $(ROCALUTION_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    rocBLAS:
+      pipelineId: $(ROCBLAS_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    ROCdbgapi:
+      pipelineId: $(ROCDBGAPI_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: false
+    rocDecode:
+      pipelineId: $(ROCDECODE_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: false
+    rocFFT:
+      pipelineId: $(ROCFFT_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    ROCgdb:
+      pipelineId: $(ROCGDB_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline-rocgdb-15
+      hasGpuTarget: false
+    rocJPEG:
+      pipelineId: $(ROCJPEG_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: false
+    rocm-cmake:
+      pipelineId: $(ROCM_CMAKE_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: false
+    rocm-core:
+      pipelineId: $(ROCM_CORE_PIPELINE_ID)
+      stagingBranch: master
+      mainlineBranch: amd-master
+      hasGpuTarget: false
+    rocm-examples:
+      pipelineId: $(ROCM_EXAMPLES_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: develop
+      hasGpuTarget: true
+    rocminfo:
+      pipelineId: $(ROCMINFO_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-master
+      hasGpuTarget: false
+    rocMLIR:
+      pipelineId: $(ROCMLIR_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: false
+    ROCmValidationSuite:
+      pipelineId: $(ROCMVALIDATIONSUITE_PIPELINE_ID)
+      stagingBranch: master
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    rocm_bandwidth_test:
+      pipelineId: $(ROCM_BANDWIDTH_TEST_PIPELINE_ID)
+      stagingBranch: master
+      mainlineBranch: master
+      hasGpuTarget: false
+    rocm_smi_lib:
+      pipelineId: $(ROCM_SMI_LIB_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: false
+    rocPRIM:
+      pipelineId: $(ROCPRIM_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    rocprofiler:
+      pipelineId: $(ROCPROFILER_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-master
+      hasGpuTarget: true
+    rocprofiler-compute:
+      pipelineId: $(ROCPROFILER_COMPUTE_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: amd-mainline
+      hasGpuTarget: true
+    rocprofiler-register:
+      pipelineId: $(ROCPROFILER_REGISTER_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: false
+    rocprofiler-sdk:
+      pipelineId: $(ROCPROFILER_SDK_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: true
+    rocprofiler-systems:
+      pipelineId: $(ROCPROFILER_SYSTEMS_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: true
+    rocPyDecode:
+      pipelineId: $(ROCPYDECODE_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    ROCR-Runtime:
+      pipelineId: $(ROCR_RUNTIME_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-master
+      hasGpuTarget: false
+    rocRAND:
+      pipelineId: $(ROCRAND_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    rocr_debug_agent:
+      pipelineId: $(ROCR_DEBUG_AGENT_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-mainline
+      hasGpuTarget: false
+    rocSOLVER:
+      pipelineId: $(ROCSOLVER_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    rocSPARSE:
+      pipelineId: $(ROCSPARSE_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    ROCT-Thunk-Interface: # deprecated
+      pipelineId: $(ROCT_THUNK_INTERFACE_PIPELINE_ID)
+      stagingBranch: master
+      mainlineBranch: master
+      hasGpuTarget: false
+    rocThrust:
+      pipelineId: $(ROCTHRUST_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    roctracer:
+      pipelineId: $(ROCTRACER_PIPELINE_ID)
+      stagingBranch: amd-staging
+      mainlineBranch: amd-master
+      hasGpuTarget: true
+    rocWMMA:
+      pipelineId: $(ROCWMMA_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    rpp:
+      pipelineId: $(RPP_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
+    TransferBench:
+      pipelineId: $(TRANSFERBENCH_PIPELINE_ID)
+      stagingBranch: develop
+      mainlineBranch: mainline
+      hasGpuTarget: true
 
 steps:
 # assuming artifact-download.yml template file in same directory
@@ -201,39 +353,59 @@ steps:
     - template: artifact-download.yml
       parameters:
         componentName: ${{ split(dependency, ':')[0] }}
-        extractToMnt: ${{ parameters.extractToMnt }}
-        ${{ if eq(parameters.dependencySource, 'staging') }}:
-          pipelineId: ${{ parameters.stagingPipelineIdentifiers[ split(dependency, ':')[0] ] }}
-          latestFromBranch: ${{ parameters.latestFromBranch }}
-        ${{ elseif eq(parameters.dependencySource, 'tag-builds') }}:
-          pipelineId: ${{ parameters.taggedPipelineIdentifiers[ split(dependency, ':')[0] ] }}
-          latestFromBranch: false
-        ${{ if containsValue( parameters.componentsWithGPUTarget, split(dependency, ':')[0] ) }}:
+        pipelineId: ${{ parameters.componentVarList[split(dependency, ':')[0]].pipelineId }}
+        ${{ if parameters.componentVarList[split(dependency, ':')[0]].hasGpuTarget }}:
           fileFilter: "${{ split(dependency, ':')[1] }}*${{ parameters.gpuTarget }}"
+        # dependencySource = staging
+        ${{ if eq(parameters.dependencySource, 'staging')}}:
+          branchName: ${{ parameters.componentVarList[split(dependency, ':')[0]].stagingBranch }}
+        # dependencySource = mainline
+        ${{ elseif eq(parameters.dependencySource, 'mainline')}}:
+          branchName: ${{ parameters.componentVarList[split(dependency, ':')[0]].mainlineBranch }}
+        # checkoutRef = staging
+        ${{ elseif eq(parameters.checkoutRef, parameters.componentVarList[variables['Build.DefinitionName']].stagingBranch) }}:
+          branchName: ${{ parameters.componentVarList[split(dependency, ':')[0]].stagingBranch }}
+        # checkoutRef = mainline
+        ${{ elseif eq(parameters.checkoutRef, parameters.componentVarList[variables['Build.DefinitionName']].mainlineBranch) }}:
+          branchName: ${{ parameters.componentVarList[split(dependency, ':')[0]].mainlineBranch }}
+        # SourceBranchName = staging
+        ${{ elseif eq(variables['Build.SourceBranchName'], parameters.componentVarlist[variables['Build.DefinitionName']].stagingBranch) }}:
+          branchName: ${{ parameters.componentVarList[split(dependency, ':')[0]].stagingBranch }}
+        # SourceBranchName = mainline
+        ${{ elseif eq(variables['Build.SourceBranchName'], parameters.componentVarlist[variables['Build.DefinitionName']].mainlineBranch) }}:
+          branchName: ${{ parameters.componentVarList[split(dependency, ':')[0]].mainlineBranch }}
+        # default = staging
         ${{ else }}:
-          fileFilter: ${{ split(dependency, ':')[1] }}
+          branchName: ${{ parameters.componentVarList[split(dependency, ':')[0]].stagingBranch }}
 # no colon (:) found in this item in the list
   - ${{ else }}:
     - template: artifact-download.yml
       parameters:
         componentName: ${{ dependency }}
-        extractToMnt: ${{ parameters.extractToMnt }}
-        ${{ if eq(parameters.dependencySource, 'staging') }}:
-          pipelineId: ${{ parameters.stagingPipelineIdentifiers[dependency] }}
-          latestFromBranch: ${{ parameters.latestFromBranch }}
-        ${{ elseif eq(parameters.dependencySource, 'tag-builds') }}:
-          pipelineId: ${{ parameters.taggedPipelineIdentifiers[dependency] }}
-          latestFromBranch: false
-        ${{ if containsValue( parameters.componentsWithGPUTarget, dependency ) }}:
+        pipelineId: ${{ parameters.componentVarList[dependency].pipelineId }}
+        ${{ if parameters.componentVarList[dependency].hasGpuTarget }}:
           fileFilter: ${{ parameters.gpuTarget }}
-# fixed case only accepts one component at a time, so no array input
-- ${{ if eq(parameters.dependencySource, 'fixed') }}:
-  - template: artifact-download.yml
-    parameters:
-      componentName: ${{ parameters.fixedComponentName }}
-      pipelineId: ${{ parameters.fixedPipelineIdentifier }}
-      latestFromBranch: false
-      extractToMnt: ${{ parameters.extractToMnt }}
+        # dependencySource = staging
+        ${{ if eq(parameters.dependencySource, 'staging')}}:
+          branchName: ${{ parameters.componentVarList[dependency].stagingBranch }}
+        # dependencySource = mainline
+        ${{ elseif eq(parameters.dependencySource, 'mainline')}}:
+          branchName: ${{ parameters.componentVarList[dependency].mainlineBranch }}
+        # checkoutRef = staging
+        ${{ elseif eq(parameters.checkoutRef, parameters.componentVarList[variables['Build.DefinitionName']].stagingBranch) }}:
+          branchName: ${{ parameters.componentVarList[dependency].stagingBranch }}
+        # checkoutRef = mainline
+        ${{ elseif eq(parameters.checkoutRef, parameters.componentVarList[variables['Build.DefinitionName']].mainlineBranch) }}:
+          branchName: ${{ parameters.componentVarList[dependency].mainlineBranch }}
+        # SourceBranchName = staging
+        ${{ elseif eq(variables['Build.SourceBranchName'], parameters.componentVarlist[variables['Build.DefinitionName']].stagingBranch) }}:
+          branchName: ${{ parameters.componentVarList[dependency].stagingBranch }}
+        # SourceBranchName = mainline
+        ${{ elseif eq(variables['Build.SourceBranchName'], parameters.componentVarlist[variables['Build.DefinitionName']].mainlineBranch) }}:
+          branchName: ${{ parameters.componentVarList[dependency].mainlineBranch }}
+        # default = staging
+        ${{ else }}:
+          branchName: ${{ parameters.componentVarList[dependency].stagingBranch }}
 # Set link to redirect llvm folder
 - ${{ if eq(parameters.skipLlvmSymlink, false) }}:
   - task: Bash@3
@@ -249,31 +421,51 @@ steps:
         for file in amdclang amdclang++ amdclang-cl amdclang-cpp amdflang amdlld aompcc mygpu mycpu offload-arch; do
           sudo ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/$file $(Agent.BuildDirectory)/rocm/bin/$file
         done
+# dlopen calls within a ctest or pytest sequence runs into issues when shared library symlink convention is not followed
+# the convention is as follows:
+# unversioned .so is a symlink to major version .so
+# major version .so is a symlink to detailed version .so
+# HIP libraries do not follow this convention, and each .so is a copy of each other
+# changing the library structure to follow the symlink convention resolves some test failures
+- ${{ if eq(parameters.setupHIPLibrarySymlinks, true) }}:
+  - task: Bash@3
+    displayName: Setup symlinks for hip libraries
+    inputs:
+      targetType: inline
+      workingDirectory: $(Agent.BuildDirectory)/rocm/lib
+      script: |
+        LIBRARIES=("libamdhip64" "libhiprtc-builtins" "libhiprtc")
+        for LIB_NAME in "${LIBRARIES[@]}"; do
+            VERSIONED_SO=$(ls ${LIB_NAME}.so.* 2>/dev/null | grep -E "${LIB_NAME}\.so\.[0-9]+\.[0-9]+\.[0-9]+(-.*)?" | sort -V | tail -n 1)
+            if [[ -z "$VERSIONED_SO" ]]; then
+                continue
+            fi
+            MAJOR_VERSION=$(echo "$VERSIONED_SO" | grep -oP "${LIB_NAME}\.so\.\K[0-9]+")
+            if [[ -e "${LIB_NAME}.so.${MAJOR_VERSION}" && ! -L "${LIB_NAME}.so.${MAJOR_VERSION}" ]]; then
+                rm -f "${LIB_NAME}.so.${MAJOR_VERSION}"
+            fi
+            if [[ -e "${LIB_NAME}.so" && ! -L "${LIB_NAME}.so" ]]; then
+                rm -f "${LIB_NAME}.so"
+            fi
+            ln -sf "$VERSIONED_SO" "${LIB_NAME}.so.${MAJOR_VERSION}"
+            ln -sf "${LIB_NAME}.so.${MAJOR_VERSION}" "${LIB_NAME}.so"
+            echo "Symlinks created for $LIB_NAME:"
+            ls -l ${LIB_NAME}.so*
+        done
 - task: Bash@3
   displayName: 'List downloaded ROCm files'
   inputs:
     targetType: inline
-    ${{ if eq(parameters.extractToMnt, true) }}:
-      script: ls -1R /mnt/rocm
-    ${{ else }}:
-      script: ls -1R $(Agent.BuildDirectory)/rocm
+    script: ls -1R $(Agent.BuildDirectory)/rocm
 - ${{ if eq(parameters.skipLibraryLinking, false) }}:
   - task: Bash@3
     displayName: 'Link ROCm shared libraries'
     inputs:
       targetType: inline
 # OS ignores if the ROCm lib folder shows up more than once
-      ${{ if eq(parameters.extractToMnt, true) }}:
-        script: |
-          echo /mnt/rocm/lib | sudo tee /etc/ld.so.conf.d/rocm-ci.conf
-          echo /mnt/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
-          sudo cat /etc/ld.so.conf.d/rocm-ci.conf
-          sudo ldconfig -v
-          ldconfig -p
-      ${{ else }}:
-        script: |
-          echo $(Agent.BuildDirectory)/rocm/lib | sudo tee /etc/ld.so.conf.d/rocm-ci.conf
-          echo $(Agent.BuildDirectory)/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
-          sudo cat /etc/ld.so.conf.d/rocm-ci.conf
-          sudo ldconfig -v
-          ldconfig -p
+      script: |
+        echo $(Agent.BuildDirectory)/rocm/lib | sudo tee /etc/ld.so.conf.d/rocm-ci.conf
+        echo $(Agent.BuildDirectory)/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
+        sudo cat /etc/ld.so.conf.d/rocm-ci.conf
+        sudo ldconfig -v
+        ldconfig -p

.azuredevops/templates/steps/docker-container.yml

@@ -0,0 +1,348 @@
+parameters:
+# this base set of packages should not be changed by calling script
+# should be common across all pipelines
+- name: baseAptPackages
+  type: object
+  default:
+    - build-essential
+    - ca-certificates
+    - curl
+    - file
+    - git
+    - gcc
+    - g++
+    - gpg
+    - kmod
+    - libdrm-dev
+    - libelf-dev
+    - libgtest-dev
+    - libhsakmt-dev
+    - libhwloc-dev
+    - libnuma-dev
+    - libstdc++-12-dev
+    - libtbb-dev
+    - lsb-release
+    - lsof
+    - ninja-build
+    - pkg-config
+    - python3-dev
+    - python3-pip
+    - wget
+    - zip
+# optional array of additional apt packages to install
+- name: aptPackages
+  type: object
+  default: []
+# optional array of python modules to install
+- name: pipModules
+  type: object
+  default: []
+# optional array of workspace directories to install
+# sources, binaries, and rocm directories are copied by default
+- name: extraCopyDirectories
+  type: object
+  default: []
+# optional string to specify gpuTarget for the docker image string
+- name: gpuTarget
+  type: string
+  default: ''
+# test environment involves gpu-related steps
+# some jobs combine both build and test
+# some jobs differentiate based on gpu vendor
+- name: environment
+  type: string
+  default: build
+  values:
+    - build
+    - test
+    - combined
+    - amd
+    - nvidia
+# optional boolean prerequisites before install extra apt packages
+- name: registerROCmPackages
+  type: boolean
+  default: false
+- name: registerCUDAPackages
+  type: boolean
+  default: false
+- name: registerJPEGPackages
+  type: boolean
+  default: false
+# optional boolean for special setup steps to accomodate some components
+- name: installLatestCMake
+  type: boolean
+  default: false
+- name: installAOCL
+  type: boolean
+  default: false
+- name: aoclRepositoryUrl
+  type: string
+  default: https://download.amd.com/developer/eula/aocl/aocl-4-2
+- name: aoclPackageName
+  type: string
+  default: aocl-linux-gcc-4.2.0_1_amd64.deb
+- name: optSymLink
+  type: boolean
+  default: false
+- name: pythonEnvVars
+  type: boolean
+  default: false
+# optional string to add to PATH
+- name: extraPaths
+  type: string
+  default: ''
+# optional array of environment variables to set
+# each array element expected to be in format of
+# key:value
+- name: extraEnvVars
+  type: object
+  default: []
+# force the docker to be created, regardless of failure condition
+- name: forceDockerCreation
+  type: boolean
+  default: false
+
+steps:
+# these steps should only be run if there was a failure or warning
+# dynamically write to a Dockerfile
+# first is to do base setup of users, groups
+  - task: Bash@3
+    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    displayName: Create start of Dockerfile
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: |
+        echo "FROM ubuntu:22.04" > Dockerfile
+        echo "ARG USERNAME=user" >> Dockerfile
+        echo "ARG USER_UID=1000" >> Dockerfile
+        echo "ARG USER_GID=\$USER_UID" >> Dockerfile
+        echo "RUN groupadd --gid \$USER_GID \$USERNAME" >> Dockerfile
+        echo "RUN useradd --uid \$USER_UID --gid \$USER_GID -m \$USERNAME" >> Dockerfile
+        echo "RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update" >> Dockerfile
+        echo "RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install sudo" >> Dockerfile
+        echo "RUN echo \$USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/\$USERNAME" >> Dockerfile
+        echo "RUN chmod 0440 /etc/sudoers.d/\$USERNAME" >> Dockerfile
+# for test jobs, setup GPU-related users and group
+  - ${{ if eq(parameters.environment, 'test') }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: GPU setup of Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: |
+          echo "RUN groupadd render" >> Dockerfile
+          echo "RUN usermod -aG render,video \$USERNAME" >> Dockerfile
+# now install a common set of packages through apt
+  - ${{ if gt(length(parameters.baseAptPackages), 0) }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: Base Apt Packages to Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: echo "RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install ${{ join(' ', parameters.baseAptPackages) }}" >> Dockerfile
+# iterate through possible apt repos that might need to be added to the docker container
+  - ${{ if eq(parameters.registerROCmPackages, true) }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: Register ROCm packages to Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: |
+          echo "RUN mkdir --parents --mode=0755 /etc/apt/keyrings" >> Dockerfile
+          echo "RUN wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | tee /etc/apt/keyrings/rocm.gpg > /dev/null" >> Dockerfile
+          echo "RUN echo \"deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/$(REPO_RADEON_VERSION)/ubuntu jammy main\" | tee /etc/apt/sources.list.d/amdgpu.list" >> Dockerfile
+          echo "RUN echo \"deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$(REPO_RADEON_VERSION) jammy main\" | tee --append /etc/apt/sources.list.d/rocm.list" >> Dockerfile
+          echo "RUN printf 'Package: *\\nPin: release o=repo.radeon.com\\nPin-Priority: 600' > /etc/apt/preferences.d/rocm-pin-600" >> Dockerfile
+          echo "RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update" >> Dockerfile
+  - ${{ if eq(parameters.registerCUDAPackages, true) }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: Register CUDA packages to Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: |
+          echo "RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb" >> Dockerfile
+          echo "RUN dpkg -i cuda-keyring_1.1-1_all.deb" >> Dockerfile
+          echo "RUN rm -f cuda-keyring_1.1-1_all.deb" >> Dockerfile
+          echo 'RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update' >> Dockerfile
+  - ${{ if eq(parameters.registerJPEGPackages, true) }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: Register libjpeg-turbo packages to Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: |
+          echo "RUN mkdir --parents --mode=0755 /etc/apt/keyrings" >> Dockerfile
+          echo "RUN wget https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey -O - | gpg --dearmor | tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null" >> Dockerfile
+          echo "RUN echo \"deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main\" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list" >> Dockerfile
+          echo "RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update" >> Dockerfile
+# install AOCL to docker container, if needed
+  - ${{ if eq(parameters.installAOCL, true) }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: aocl install to Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: |
+          echo "RUN wget -nv ${{ parameters.aoclRepositoryUrl }}/${{ parameters.aoclPackageName }}" >> Dockerfile
+          echo "RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install ./${{ parameters.aoclPackageName }}" >> Dockerfile
+          echo "RUN rm -f ${{ parameters.aoclPackageName }}" >> Dockerfile
+# since apt repo list is updated, install the extra apt packages
+  - ${{ if gt(length(parameters.aptPackages), 0) }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: Extra Apt Packages to Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: echo "RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install ${{ join(' ', parameters.aptPackages) }}" >> Dockerfile
+# install latest cmake to docker container, if needed
+  - ${{ if eq(parameters.installLatestCMake, true) }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: latest cmake install to Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: |
+            echo "RUN DEBIAN_FRONTEND=noninteractive apt-get --yes purge cmake" >> Dockerfile
+            echo "RUN pip install cmake --upgrade" >> Dockerfile
+# setup workspace where binaries, sources, and dependencies from the job will be copied to
+  - task: Bash@3
+    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    displayName: Workspace setup of Dockerfile
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: |
+        echo "USER \$USERNAME" >> Dockerfile
+        echo "WORKDIR /home/user" >> Dockerfile
+        echo "RUN mkdir -p /home/user/workspace" >> Dockerfile
+# pip install is done here as non-root
+  - ${{ if gt(length(parameters.pipModules), 0) }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: Extra Python Modules to Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: echo "RUN pip install -v ${{ join(' ', parameters.pipModules) }}" >> Dockerfile
+# copy common directories
+  - task: Bash@3
+    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    displayName: Copy base directories to Dockerfile
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: |
+        if [ -d "$(Agent.BuildDirectory)/rocm" ]; then
+          echo "COPY rocm /home/user/workspace/rocm" >> Dockerfile
+        fi
+        if [ -d "$(Build.SourcesDirectory)" ] && [ "$(Build.SourcesDirectory)" != "" ]; then
+          echo "COPY s /home/user/workspace/src" >> Dockerfile
+        fi
+        if [ -d "$(Build.BinariesDirectory)" ] && [ "$(Build.BinariesDirectory)" != "" ]; then
+          echo "COPY b /home/user/workspace/bin" >> Dockerfile
+        fi
+# copy extra directories, if applicable to the job
+  - ${{ each extraCopyDirectory in parameters.extraCopyDirectories }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: Copy ${{ extraCopyDirectory }} to Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: |
+          if [ -d "${{ extraCopyDirectory }}" ]; then
+            echo "COPY ${{ extraCopyDirectory }} /home/user/workspace/${{ extraCopyDirectory }}" >> Dockerfile
+          fi
+# setup ldconfig
+  - task: Bash@3
+    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    displayName: ldconfig to Dockerfile
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: |
+        echo "USER root" >> Dockerfile
+        echo "RUN echo /home/user/workspace/rocm/lib | tee /etc/ld.so.conf.d/rocm-ci.conf" >> Dockerfile
+        echo "RUN echo /home/user/workspace/rocm/llvm/lib | tee -a /etc/ld.so.conf.d/rocm-ci.conf" >> Dockerfile
+        echo "RUN cat /etc/ld.so.conf.d/rocm-ci.conf" >> Dockerfile
+        echo "RUN ldconfig -v" >> Dockerfile
+# create /opt/rocm symbolic link, if needed
+  - ${{ if eq(parameters.optSymLink, true) }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: /opt/rocm symbolic link to Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: |
+          echo "USER root" >> Dockerfile
+          echo "RUN ln -s /home/user/workspace/rocm /opt/rocm" >> Dockerfile
+# set environment variables needed for some python-based components
+  - ${{ if eq(parameters.pythonEnvVars, true) }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: python environment variables
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: |
+          echo "USER root" >> Dockerfile
+          echo "ENV PYTHON_USER_SITE=$(python3 -m site --user-site)" >> Dockerfile
+          echo "ENV PYTHON_DIST_PACKAGES=$(python3 -c 'import sysconfig; print(sysconfig.get_paths()[\"purelib\"])')" >> Dockerfile
+          echo "ENV PYBIND11_PATH=$(python3 -c 'import pybind11; print(pybind11.get_cmake_dir())')" >> Dockerfile
+# add to PATH environment variable
+  - ${{ if ne(parameters.extraPaths, '') }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: Add to PATH in Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: echo "ENV PATH='${{ parameters.extraPaths }}:\$PATH'" >> Dockerfile
+# set extra environment variables, if applicable to the job
+# use ::: as delimiter to allow for colons to be in the environment variable values
+  - ${{ each extraEnvVar in parameters.extraEnvVars }}:
+    - task: Bash@3
+      condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+      displayName: Set ${{ extraEnvVar }} to Dockerfile
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: echo "ENV ${{ split(extraEnvVar, ':::')[0] }}='${{ split(extraEnvVar, ':::')[1] }}'" >> Dockerfile
+  - task: Bash@3
+    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    displayName: Print Dockerfile
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: cat Dockerfile
+  - task: Docker@2
+    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    inputs:
+      containerRegistry: 'ContainerService'
+      ${{ if ne(parameters.gpuTarget, '') }}:
+        repository: '$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}'
+      ${{ else }}:
+        repository: '$(Build.DefinitionName)-${{ parameters.environment }}'
+      Dockerfile: '$(Pipeline.Workspace)/Dockerfile'
+      buildContext: '$(Pipeline.Workspace)'
+  - task: Bash@3
+    condition: or(failed(), ${{ eq(parameters.forceDockerCreation, true) }})
+    displayName: "!! Docker Image URL !!"
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      ${{ if ne(parameters.gpuTarget, '') }}:
+        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}:$(Build.BuildId)"
+      ${{ else }}:
+        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}:$(Build.BuildId)"

.azuredevops/templates/steps/gpu-diagnostics.yml

@@ -0,0 +1,54 @@
+# Diagnostics for GPU-enabled systems
+parameters:
+- name: runRocminfo
+  type: boolean
+  default: true
+
+steps:
+- ${{ if eq(parameters.runRocminfo, true) }}:
+  - task: Bash@3
+    displayName: 'rocminfo'
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: $(Agent.BuildDirectory)/rocm/bin/rocminfo || true
+  - task: Bash@3
+    displayName: 'rocm_agent_enumerator'
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: $(Agent.BuildDirectory)/rocm/bin/rocm_agent_enumerator || true
+- task: Bash@3
+  displayName: 'List DRI devices'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: ls -la /dev/dri/ || true
+- task: Bash@3
+  displayName: 'List amdgpu/rocm/mesa packages'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: apt list --installed | grep -E 'amdgpu|rocm|mesa' || true
+- task: Bash@3
+  displayName: 'List GPU processes'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      ls /sys/class/kfd/kfd/proc/ || true
+      sudo lsof | grep amdgpu || true
+- task: Bash@3
+  displayName: 'System snapshot'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: top -bn1 || true
+- task: Bash@3
+  displayName: 'List dmesg'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      echo 'rocm-ci: $(Build.DefinitionName) $(System.DefinitionId)' | sudo tee /dev/kmsg || true
+      sudo dmesg || true

.azuredevops/templates/steps/local-artifact-download.yml

@@ -29,17 +29,17 @@ steps:
         definition: ${{ parameters.definitionId }}
         buildId: ${{ parameters.buildId }}
       itemPattern: '**/*${{ parameters.gpuTarget }}*'
-      targetPath: $(System.ArtifactsDirectory)
+      targetPath: $(Pipeline.Workspace)/d
   - task: ExtractFiles@1
     displayName: 'Extract Pipeline Build'
     inputs:
-      archiveFilePatterns: '$(System.ArtifactsDirectory)/**/*.tar.gz'
+      archiveFilePatterns: '$(Pipeline.Workspace)/d/**/*.tar.gz'
       destinationFolder: '$(Agent.BuildDirectory)/rocm'
       cleanDestinationFolder: false
       overwriteExistingFiles: true
   - task: DeleteFiles@1
     displayName: 'Clean up Compressed Pipeline Build'
     inputs:
-      SourceFolder: '$(System.ArtifactsDirectory)'
+      SourceFolder: '$(Pipeline.Workspace)/d'
       Contents: '/**/*.tar.xz'
       RemoveDotFiles: true

.azuredevops/templates/steps/manifest.yml

@@ -0,0 +1,150 @@
+parameters:
+- name: artifactName
+  type: string
+  default: 'drop'
+- name: gpuTarget
+  type: string
+  default: ''
+
+steps:
+- task: Bash@3
+  displayName: Set up current_repo values
+  condition: always()
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      sudo apt-get install -y jq
+
+      # RESOURCES_REPOSITORIES is a runtime variable (not an env var!) that contains quotations and newlines
+      # So we need to save it to a file to properly preserve its formatting and contents
+      cat <<EOF > resources.repositories
+      $(RESOURCES_REPOSITORIES)
+      EOF
+      cat resources.repositories
+
+      IS_TAG_BUILD=$(jq 'has("release_repo")' resources.repositories)
+      if [ "$IS_TAG_BUILD" = "true" ]; then
+        REPO_TYPE="release_repo" # Triggered by a ROCm/ROCm tag-builds file
+      else
+        REPO_TYPE="self" # Triggered by component repo's rocm-ci.yml file
+      fi
+
+      echo "##vso[task.setvariable variable=current_repo.id;]$(jq .$REPO_TYPE.id resources.repositories | tr -d '"')"
+      echo "##vso[task.setvariable variable=current_repo.name;]$(jq .$REPO_TYPE.name resources.repositories | tr -d '"')"
+      echo "##vso[task.setvariable variable=current_repo.ref;]$(jq .$REPO_TYPE.ref resources.repositories | tr -d '"')"
+      echo "##vso[task.setvariable variable=current_repo.url;]$(jq .$REPO_TYPE.url resources.repositories | tr -d '"')"
+      echo "##vso[task.setvariable variable=current_repo.version;]$(jq .$REPO_TYPE.version resources.repositories | tr -d '"')"
+- task: Bash@3
+  displayName: Create manifest.json
+  condition: always()
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      manifest_json=$(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.json
+
+      dependencies=()
+      for manifest_file in $(Pipeline.Workspace)/d/**/manifest_*.json; do
+        echo "Processing $manifest_file"
+        cat $manifest_file
+        current=$(jq '.current' "$manifest_file")
+        dependencies+=("$current")
+      done
+      dependencies_json=$(printf '%s\n' "${dependencies[@]}" | jq -s '.')
+
+      jq -n \
+        --arg buildNumber "$(Build.BuildNumber)" \
+        --arg buildId "$(Build.BuildId)" \
+        --arg repoId "$(current_repo.id)" \
+        --arg repoName "$(current_repo.name)" \
+        --arg repoRef "$(current_repo.ref)" \
+        --arg repoUrl "$(current_repo.url)" \
+        --arg repoVersion "$(current_repo.version)" \
+        --argjson dependencies "$dependencies_json" \
+        '{
+          current: {
+            buildNumber: $buildNumber,
+            buildId: $buildId,
+            repoId: $repoId,
+            repoName: $repoName,
+            repoRef: $repoRef,
+            repoUrl: $repoUrl,
+            repoVersion: $repoVersion
+          },
+          dependencies: $dependencies
+        }' > $manifest_json
+
+      dependencies_rows=$(cat $manifest_json | \
+        jq -r '
+          .dependencies[] | 
+          "<tr><td>" + .buildNumber + "</td>" + 
+          "<td><a href=\"https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=" + .buildId + "\">" + .buildId + "</a></td>" + 
+          "<td><a href=\"" + .repoUrl + "\">" + .repoName + "</a></td>" + 
+          "<td><a href=\"" + .repoUrl + "/tree/" + .repoRef + "\">" + .repoRef + "</a></td>" + 
+          "<td><a href=\"" + .repoUrl + "/commit/" + .repoVersion + "\">" + .repoVersion + "</a></td></tr>"
+        ')
+      dependencies_rows=$(echo $dependencies_rows)
+      echo "##vso[task.setvariable variable=dependencies_rows;]$dependencies_rows"
+
+      cat $manifest_json
+- task: Bash@3
+  displayName: Create manifest.html
+  condition: always()
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      manifest_html=$(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.html
+      cat <<EOF > $manifest_html
+      <html>
+      <h1>Manifest</h1>
+      <h2>Current</h2>
+      <table border="1">
+      <tr>
+        <th>Build Number</th>
+        <th>Build ID</th>
+        <th>Repo Name</th>
+        <th>Repo Ref</th>
+        <th>Repo Version</th>
+      </tr>
+      <tr>
+        <td>$(Build.BuildNumber)</td>
+        <td><a href="https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$(Build.BuildId)">$(Build.BuildId)</a></td>
+        <td><a href="$(current_repo.url)">$(current_repo.name)</a></td>
+        <td><a href="$(current_repo.url)/tree/$(current_repo.ref)">$(current_repo.ref)</a></td>
+        <td><a href="$(current_repo.url)/commit/$(current_repo.version)">$(current_repo.version)</a></td>
+      </tr>
+      </table>
+      <h2>Dependencies</h2>
+      <table border="1">
+      <tr>
+        <th>Build Number</th>
+        <th>Build ID</th>
+        <th>Repo Name</th>
+        <th>Repo Ref</th>
+        <th>Repo Version</th>
+      </tr>
+      $(dependencies_rows)
+      </table>
+      </html>
+      EOF
+
+      cat $manifest_html
+- task: PublishHtmlReport@1
+  displayName: Publish manifest.html
+  condition: always()
+  continueOnError: true
+  inputs:
+    tabName: Manifest
+    reportDir: $(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.html
+- task: Bash@3
+  displayName: Save manifest artifact file name
+  condition: always()
+  continueOnError: true
+  inputs:
+    workingDirectory: $(Pipeline.Workspace)
+    targetType: inline
+    script: |
+      echo "manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.html" >> pipelineArtifacts.txt
+      echo "manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.json" >> pipelineArtifacts.txt

.azuredevops/templates/steps/miopen-get-ck-build.yml

@@ -0,0 +1,72 @@
+parameters:
+- name: gpuTarget
+  type: string
+  default: ''
+
+steps:
+- task: Bash@3
+  name: downloadCKBuild
+  displayName: Download specific CK build
+  continueOnError: true
+  env:
+    CXX: $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+    CC: $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+  inputs:
+    targetType: inline
+    workingDirectory: $(Build.SourcesDirectory)
+    script: |
+      AZ_API="https://dev.azure.com/ROCm-CI/ROCm-CI/_apis"
+      GH_API="https://api.github.com/repos/ROCm"
+      ARTIFACT_NAME="composablekernel.${{ parameters.gpuTarget }}"
+      EXIT_CODE=0
+
+      # Try to find an Azure build for the specific CK commit called out in MIOpen's requirements.txt
+      CK_COMMIT=$(grep 'ROCm/composable_kernel' requirements.txt | sed -E 's/.*@([a-f0-9]{40}).*/\1/')
+      echo "Fetching CK build ID for commit $CK_COMMIT"
+      CK_CHECKS_URL="$GH_API/composable_kernel/commits/${CK_COMMIT}/check-runs"
+      CK_BUILD_ID=$(curl -s $CK_CHECKS_URL | \
+        jq '.check_runs[] | select(.name == "composable_kernel" and .app.slug == "azure-pipelines") | .details_url' | \
+        tr -d '"' | grep -oP 'buildId=\K\d+')
+
+      # If none found, use latest successful CK build instead
+      if [[ -z "$CK_BUILD_ID" ]]; then
+        echo "Did not find specific CK build ID"
+        LATEST_BUILD_URL="$AZ_API/build/builds?definitions=$(COMPOSABLE_KERNEL_PIPELINE_ID)&statusFilter=completed&resultFilter=succeeded&\$top=1&api-version=7.1"
+        CK_BUILD_ID=$(curl -s $LATEST_BUILD_URL | jq '.value[0].id')
+        echo "Found latest CK build ID: $CK_BUILD_ID"
+        EXIT_CODE=1
+      else
+        echo "Found specific CK build ID: $CK_BUILD_ID"
+      fi
+
+      AZURE_URL="$AZ_API/build/builds/$CK_BUILD_ID/artifacts?artifactName=$ARTIFACT_NAME&api-version=7.1"
+      ARTIFACT_URL=$(curl -s $AZURE_URL | jq '.resource.downloadUrl' | tr -d '"')
+
+      # If using the specific CK commit and it doesn't have any valid artifacts, use latest successful CK build instead
+      if { [[ -z "$ARTIFACT_URL" ]] || [[ "$ARTIFACT_URL" == "null" ]]; } && [[ $EXIT_CODE -eq 0 ]]; then
+        echo "Did not find valid specific CK build artifact"
+        LATEST_BUILD_URL="$AZ_API/build/builds?definitions=$(COMPOSABLE_KERNEL_PIPELINE_ID)&statusFilter=completed&resultFilter=succeeded&\$top=1&api-version=7.1"
+        CK_BUILD_ID=$(curl -s $LATEST_BUILD_URL | jq '.value[0].id')
+        echo "Found latest CK build ID: $CK_BUILD_ID"
+        AZURE_URL="$AZ_API/build/builds/$CK_BUILD_ID/artifacts?artifactName=$ARTIFACT_NAME&api-version=7.1"
+        ARTIFACT_URL=$(curl -s $AZURE_URL | jq '.resource.downloadUrl' | tr -d '"')
+        EXIT_CODE=2
+      fi
+
+      echo "Downloading CK artifact from $ARTIFACT_URL"
+      wget -nv $ARTIFACT_URL -O $(System.ArtifactsDirectory)/ck.zip
+      unzip $(System.ArtifactsDirectory)/ck.zip -d $(System.ArtifactsDirectory)
+      mkdir -p $(Agent.BuildDirectory)/rocm
+      tar -zxvf $(System.ArtifactsDirectory)/$ARTIFACT_NAME/*.tar.gz -C $(Agent.BuildDirectory)/rocm
+      rm -r $(System.ArtifactsDirectory)/ck.zip $(System.ArtifactsDirectory)/$ARTIFACT_NAME
+
+      if [[ $EXIT_CODE -ne 0 ]]; then
+        BUILD_COMMIT=$(curl -s $AZ_API/build/builds/$CK_BUILD_ID | jq '.sourceVersion' | tr -d '"')
+        if [[ $EXIT_CODE -eq 1 ]]; then
+          echo "WARNING: couldn't find a CK build for commit $CK_COMMIT"
+        elif [[ $EXIT_CODE -eq 2 ]]; then
+          echo "WARNING: couldn't find a valid CK artifact for commit $CK_COMMIT"
+        fi
+        echo "Instead used latest CK build $CK_BUILD_ID for commit $BUILD_COMMIT"
+      fi
+      exit $EXIT_CODE

.azuredevops/templates/steps/test.yml

@@ -26,25 +26,24 @@ parameters:
 - name: testPublishResults
   type: boolean
   default: true
-- name: reloadAMDGPU
-  type: boolean
-  default: false
+- name: allowPartiallySucceededBuilds
+  type: object
+  default:
+    - amdsmi
+    - aomp
+    - HIPIFY
+    - MIVisionX
+    - rocm-cmake
+    - rocm_smi_lib
+    - rocprofiler-sdk
+    - roctracer
 
 steps:
-# Avoids occasional AMDGPU driver issues with opening /dev/kfd
-- ${{ if parameters.reloadAMDGPU }}:
-  - task: Bash@3
-    displayName: Unload and reload AMDGPU
-    inputs:
-      targetType: inline
-      script: |
-        sudo modprobe -r amdgpu
-        sudo modprobe amdgpu
 # run test, continue on failure to publish results
 # and to publish build artifacts
 - task: Bash@3
   displayName: '${{ parameters.componentName }} Test'
-  continueOnError: true
+  continueOnError: ${{ containsValue(parameters.allowPartiallySucceededBuilds, parameters.componentName) }}
   inputs:
     targetType: inline
     script: ${{ parameters.testExecutable }} ${{ parameters.testParameters }}
@@ -52,8 +51,8 @@ steps:
 - ${{ if parameters.testPublishResults }}:
   - task: PublishTestResults@2
     displayName: '${{ parameters.componentName }} Publish Results'
+    condition: succeededOrFailed()
     inputs:
       searchFolder: ${{ parameters.testDir }}
       testResultsFormat: ${{ parameters.testOutputFormat }}
       testResultsFiles: '**/${{ parameters.testOutputFile }}'
-    condition: succeededOrFailed()

.azuredevops/variables-global.yml

@@ -1,6 +1,8 @@
 # specify non-secret global variables reused across pipelines here
 
 variables:
+- name: RESOURCES_REPOSITORIES
+  value: $[ convertToJson(resources.repositories) ]
 - name: CI_ROOT_PATH
   value: /.azuredevops
 - name: CI_COMPONENT_PATH
@@ -25,14 +27,14 @@ variables:
   value: rocm-ci_larger_base_disk_pool
 - name: GFX942_TEST_POOL
   value: gfx942_test_pool
+- name: LATEST_RELEASE_VERSION
+  value: 6.3.3
+- name: REPO_RADEON_VERSION
+  value: 6.3.3
+- name: NEXT_RELEASE_VERSION
+  value: 6.4.0
 - name: LATEST_RELEASE_TAG
-  value: rocm-6.1.0
-- name: DOCKER_IMAGE_NAME
-  value: rocm/dev-ubuntu-22.04
-- name: LATEST_DOCKER_VERSION
-  value: 6.1
-- name: KEYRING_VERSION
-  value: 6.1
+  value: rocm-6.3.3
 - name: AMDMIGRAPHX_GFX942_TEST_PIPELINE_ID
   value: 197
 - name: AMDMIGRAPHX_PIPELINE_ID
@@ -72,13 +74,17 @@ variables:
 - name: HALF_TAGGED_PIPELINE_ID
   value: 11
 - name: HALF560_PIPELINE_ID
-  value: 66
-- name: HALF560_TAGGED_PIPELINE_ID
-  value: 66
+  value: 68
+- name: HALF560_BUILD_ID
+  value: 621
 - name: HIP_PIPELINE_ID
   value: 93
 - name: HIP_TAGGED_PIPELINE_ID
   value: 31
+- name: HIP_TESTS_PIPELINE_ID
+  value: 233
+- name: HIP_TESTS_TAGGED_PIPELINE_ID
+  value: 220
 - name: HIPBLAS_COMMON_PIPELINE_ID
   value: 223
 - name: HIPBLAS_COMMON_TAGGED_PIPELINE_ID
@@ -145,8 +151,6 @@ variables:
   value: 105
 - name: HIPTENSOR_TAGGED_PIPELINE_ID
   value: 56
-- name: LAST_RELEASE
-  value: 6.1.0
 - name: LLVM_PROJECT_PIPELINE_ID
   value: 2
 - name: LLVM_PROJECT_TAGGED_PIPELINE_ID
@@ -159,6 +163,14 @@ variables:
   value: 80
 - name: MIVISIONX_TAGGED_PIPELINE_ID
   value: 18
+- name: OMNIPERF_PIPELINE_ID
+  value: 241
+- name: OMNIPERF_TAGGED_PIPELINE_ID
+  value: 242
+- name: OMNITRACE_PIPELINE_ID
+  value: 253
+- name: OMNITRACE_TAGGED_PIPELINE_ID
+  value: 252
 - name: RCCL_GFX942_TEST_PIPELINE_ID
   value: 184
 - name: RCCL_PIPELINE_ID
@@ -169,10 +181,6 @@ variables:
   value: 100
 - name: RDC_TAGGED_PIPELINE_ID
   value: 59
-- name: REIMAGE_ORG
-  value: AGS-ROCm-CI
-- name: REIMAGE_REPO
-  value: cirrascale-reimage-automation
 - name: ROCAL_PIPELINE_ID
   value: 151
 - name: ROCALUTION_GFX942_TEST_PIPELINE_ID
@@ -205,6 +213,10 @@ variables:
   value: 134
 - name: ROCGDB_TAGGED_PIPELINE_ID
   value: 50
+- name: ROCJPEG_PIPELINE_ID
+  value: 262
+- name: ROCJPEG_TAGGED_PIPELINE_ID
+  value: 263
 - name: ROCM_BANDWIDTH_TEST_PIPELINE_ID
   value: 88
 - name: ROCM_BANDWIDTH_TEST_TAGGED_PIPELINE_ID
@@ -219,6 +231,10 @@ variables:
   value: 22
 - name: ROCM_EXAMPLES_GFX942_TEST_PIPELINE_ID
   value: 204
+- name: ROCM_EXAMPLES_PIPELINE_ID
+  value: 216
+- name: ROCM_EXAMPLES_TAGGED_PIPELINE_ID
+  value: 245
 - name: ROCM_SMI_LIB_PIPELINE_ID
   value: 96
 - name: ROCM_SMI_LIB_TAGGED_PIPELINE_ID
@@ -243,14 +259,30 @@ variables:
   value: 20
 - name: ROCPROFILER_GFX942_TEST_PIPELINE_ID
   value: 190
-- name: ROCPROFILER_PIPELINE_ID
-  value: 143
+- name: ROCPROFILER_COMPUTE_PIPELINE_ID
+  value: 257
+- name: ROCPROFILER_COMPUTE_TAGGED_PIPELINE_ID
+  value: 258
 - name: ROCPROFILER_REGISTER_PIPELINE_ID
   value: 1
 - name: ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID
   value: 25
+- name: ROCPROFILER_SDK_PIPELINE_ID
+  value: 246
+- name: ROCPROFILER_SDK_TAGGED_PIPELINE_ID
+  value: 234
+- name: ROCPROFILER_SYSTEMS_PIPELINE_ID
+  value: 255
+- name: ROCPROFILER_SYSTEMS_TAGGED_PIPELINE_ID
+  value: 254
+- name: ROCPROFILER_PIPELINE_ID
+  value: 143
 - name: ROCPROFILER_TAGGED_PIPELINE_ID
   value: 28
+- name: ROCPYDECODE_PIPELINE_ID
+  value: 239
+- name: ROCPYDECODE_TAGGED_PIPELINE_ID
+  value: 232
 - name: ROCR_DEBUG_AGENT_PIPELINE_ID
   value: 136
 - name: ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID
@@ -305,3 +337,9 @@ variables:
   value: 78
 - name: RPP_TAGGED_PIPELINE_ID
   value: 39
+- name: TRANSFERBENCH_PIPELINE_ID
+  value: 265
+- name: TRANSFERBENCH_TAGGED_PIPELINE_ID
+  value: 266
+- name: BOOST_DEPENDENCY_PIPELINE_ID
+  value: 250

.github/CODEOWNERS

@@ -5,3 +5,4 @@ docs/ @amd-aakash @jlgreathouse @samjwu @yhuiYH @ROCm/rocm-documentation
 *.rst @amd-aakash @jlgreathouse @samjwu @yhuiYH @ROCm/rocm-documentation
 # External CI
 /.azuredevops/ @ROCm/external-ci
+tools/rocm-build/ @ROCm/rocm-devops

.github/workflows/issue_retrieval.yml

@@ -2,7 +2,7 @@ name: Issue retrieval
 
 on:
   issues:
-    types: [opened]
+    types: [opened, edited]
 
 jobs:
   auto-retrieve:
@@ -15,7 +15,7 @@ jobs:
           app_id: ${{ secrets.ACTION_APP_ID }}
           private_key: ${{ secrets.ACTION_PEM }}
       - name: 'Retrieve Issue'
-        uses: abhimeda/rocm_issue_management@main
+        uses: harkgill-amd/rocm_issue_management@main
         with:
           authentication-token: ${{ steps.generate_token.outputs.token }}
           github-organization: 'ROCm'

.github/workflows/linting.yml

@@ -2,13 +2,13 @@ name: Linting
 
 on:
   push:
-    branches: 
+    branches:
     - develop
     - main
     - 'docs/*'
     - 'roc**'
   pull_request:
-    branches: 
+    branches:
     - develop
     - main
     - 'docs/*'

.spellcheck.local.yaml

@@ -0,0 +1,10 @@
+matrix:
+- name: Markdown
+  sources:
+  - ['tools/autotag/templates/**/*.md', '!tools/autotag/templates/**/5*.md', '!tools/autotag/templates/**/6.0*.md', '!tools/autotag/templates/**/6.1*.md']
+- name: reST
+  sources:
+  - []
+- name: Cpp
+  sources:
+  - []

.wordlist.txt

@@ -13,6 +13,7 @@ AMDMIGraphX
 AMI
 AOCC
 AOMP
+AOTriton
 APBDIS
 APIC
 APIs
@@ -25,6 +26,7 @@ ASm
 ATI
 AddressSanitizer
 AlexNet
+Andrej
 Arb
 Autocast
 BARs
@@ -36,6 +38,7 @@ Bluefield
 Bootloader
 CCD
 CDNA
+CHTML
 CIFAR
 CLI
 CLion
@@ -53,6 +56,7 @@ CSC
 CSE
 CSV
 CSn
+CTest
 CTests
 CU
 CUDA
@@ -69,6 +73,8 @@ Concretized
 Conda
 ConnectX
 CuPy
+Dashboarding
+DBRX
 DDR
 DF
 DGEMM
@@ -86,6 +92,8 @@ Dask
 DataFrame
 DataLoader
 DataParallel
+Debian
+DeepSeek
 DeepSpeed
 Dependabot
 Deprecations
@@ -102,12 +110,14 @@ FFT
 FFTs
 FFmpeg
 FHS
+FIXME
 FMA
 FP
 FX
 Filesystem
 FindDb
 Flang
+FluxBenchmark
 Fortran
 Fuyu
 GALB
@@ -122,10 +132,13 @@ GDS
 GEMM
 GEMMs
 GFortran
+GFXIP
+Gemma
 GiB
 GIM
 GL
 GLXT
+Gloo
 GMI
 GPG
 GPR
@@ -144,6 +157,8 @@ HGX
 HIPCC
 HIPExtension
 HIPIFY
+HIPification
+HIPify
 HPC
 HPCG
 HPE
@@ -155,6 +170,8 @@ HWS
 Haswell
 Higgs
 Hyperparameters
+Huggingface
+ICD
 ICV
 IDE
 IDEs
@@ -177,14 +194,17 @@ Interop
 Intersphinx
 Intra
 Ioffe
+JAX's
 Jinja
 JSON
 Jupyter
 KFD
 KFDTest
-KiB
+KMD
 KV
 KVM
+Karpathy's
+KiB
 Keras
 Khronos
 LAPACK
@@ -205,11 +225,13 @@ MiB
 MIGraphX
 MIOpen
 MIOpenGEMM
+MIOpen's
 MIVisionX
 MLM
 MMA
 MMIO
 MMIOH
+MMU
 MNIST
 MPI
 MSVC
@@ -226,12 +248,15 @@ Mellanox's
 Meta's
 Miniconda
 MirroredStrategy
+Mixtral
 Multicore
 Multithreaded
 MyEnvironment
 MyST
 NBIO
 NBIOs
+NCCL
+NCF
 NIC
 NICs
 NLI
@@ -271,10 +296,14 @@ OpenMPI
 OpenSSL
 OpenVX
 OpenXLA
+Oversubscription
+PagedAttention
+Pallas
 PCC
 PCI
 PCIe
 PEFT
+PEQT
 PIL
 PILImage
 POR
@@ -289,21 +318,29 @@ PipelineParallel
 PnP
 PowerEdge
 PowerShell
+Pretraining
+Profiler's
 PyPi
+Pytest
 PyTorch
 Qcycles
+Qwen
 RAII
 RAS
 RCCL
 RDC
+RDC's
 RDMA
 RDNA
 README
 RHEL
+RMW
 RNN
 RNNs
 ROC
 ROCProfiler
+ROCT
+ROCTx
 ROCTracer
 ROCclr
 ROCdbgapi
@@ -315,6 +352,7 @@ ROCmSoftwarePlatform
 ROCmValidationSuite
 ROCprofiler
 ROCr
+RPP
 RST
 RW
 Radeon
@@ -322,6 +360,7 @@ RelWithDebInfo
 Req
 Rickle
 RoCE
+Runfile
 Ryzen
 SALU
 SBIOS
@@ -334,6 +373,7 @@ SENDMSG
 SGPR
 SGPRs
 SHA
+SHARK's
 SIGQUIT
 SIMD
 SIMDs
@@ -367,6 +407,7 @@ TCR
 TF
 TFLOPS
 TP
+TPS
 TPU
 TPUs
 TSME
@@ -377,9 +418,14 @@ TensorFlow
 TensorParallel
 ToC
 TorchAudio
+torchaudio
+TorchElastic
 TorchMIGraphX
+torchrec
 TorchScript
 TorchServe
+torchserve
+torchtext
 TorchVision
 TransferBench
 TrapStatus
@@ -387,6 +433,7 @@ UAC
 UC
 UCC
 UCX
+UE
 UIF
 UMC
 USM
@@ -397,6 +444,7 @@ Unittests
 Unhandled
 VALU
 VBIOS
+VCN
 VGPR
 VGPRs
 VM
@@ -442,10 +490,12 @@ api
 atmi
 atomics
 autogenerated
+autotune
 avx
 awk
 backend
 backends
+benchmarked
 benchmarking
 bfloat
 bilinear
@@ -455,6 +505,7 @@ blit
 bootloader
 boson
 bosons
+br
 buildable
 bursty
 bzip
@@ -482,11 +533,15 @@ copyable
 cpp
 csn
 cuBLAS
+cuda
+cuDNN
+cudnn
 cuFFT
 cuLIB
 cuRAND
 cuSOLVER
 cuSPARSE
+customizations
 cTDP
 dataset
 datasets
@@ -496,6 +551,8 @@ datatypes
 dbgapi
 de
 deallocation
+debuggability
+debian
 denoise
 denoised
 denoises
@@ -510,6 +567,9 @@ devsel
 dimensionality
 disambiguates
 distro
+distros
+dkms
+dtype
 el
 embeddings
 enablement
@@ -530,6 +590,7 @@ gRPC
 galb
 gcc
 gdb
+gemm
 gfortran
 gfx
 githooks
@@ -542,8 +603,10 @@ heterogenous
 hipBLAS
 hipBLASLt
 hipBLASLt's
+hipblaslt
 hipCUB
 hipFFT
+hipFORT
 hipLIB
 hipRAND
 hipSOLVER
@@ -558,12 +621,14 @@ hipfort
 hipify
 hipsolver
 hipsparse
+hlist
 hotspotting
 hpc
 hpp
 hsa
 hsakmt
 hyperparameter
+hyperparameters
 iDRAC
 ib_core
 inband
@@ -581,11 +646,17 @@ intra
 invariants
 invocating
 ipo
+jax
 kdb
+kfd
+kv
+lang
 latencies
+len
 libfabric
 libjpeg
 libs
+linalg
 linearized
 linter
 linux
@@ -600,6 +671,7 @@ migraphx
 miopen
 miopengemm
 mivisionx
+mjx
 mkdir
 mlirmiopen
 mtypes
@@ -607,6 +679,8 @@ mutex
 mvffr
 namespace
 namespaces
+nanoGPT
+num
 numref
 ocl
 opencl
@@ -615,8 +689,11 @@ openmp
 openssl
 optimizers
 os
+oversubscription
 pageable
+pallas
 parallelization
+parallelizing
 parameterization
 passthrough
 perfcounter
@@ -629,6 +706,7 @@ prebuilt
 precompiled
 preconditioner
 preconfigured
+preemptible
 prefetch
 prefetchable
 prefill
@@ -640,17 +718,22 @@ preprocessing
 preprocessor
 prequantized
 prerequisites
+pretraining
 profiler
 profilers
 protobuf
 pseudorandom
 py
+recommender
+recommenders
 quantile
 quantizer
 quasirandom
 queueing
+radeon
 rccl
 rdc
+rdma
 reStructuredText
 redirections
 refactorization
@@ -669,6 +752,8 @@ rocALUTION
 rocBLAS
 rocDecode
 rocFFT
+rocHPCG
+rocJPEG
 rocLIB
 rocMLIR
 rocPRIM
@@ -686,6 +771,7 @@ rocm
 rocminfo
 rocprim
 rocprof
+rocprofv
 rocprofiler
 rocr
 rocrand
@@ -693,12 +779,16 @@ rocsolver
 rocsparse
 rocthrust
 roctracer
+rst
 runtime
 runtimes
 sL
 scalability
 scalable
+scipy
+seealso
 sendmsg
+seqs
 serializers
 shader
 sharding
@@ -720,6 +810,7 @@ submodules
 supercomputing
 symlink
 symlinks
+sys
 td
 tensorfloat
 th
@@ -739,8 +830,10 @@ txt
 uarch
 uncached
 uncorrectable
+underoptimized
 unhandled
 uninstallation
+unmapped
 unsqueeze
 unstacking
 unswitching
@@ -760,6 +853,8 @@ vectorize
 vectorized
 vectorizer
 vectorizes
+virtualize
+virtualized
 vjxb
 voxel
 walkthrough

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 - 2024 Advanced Micro Devices, Inc. All rights reserved.
+Copyright (c) 2023 - 2025 Advanced Micro Devices, Inc. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal