Update pipeline-debug.yml

Fixed greater than to be less than in rocFFT changes

.azuredevops/ci-builds/aomp.yml

@@ -4,11 +4,21 @@ variables:
 
 resources:
   repositories:
-  - repository: release_repo
+  - repository: aomp_repo
     type: github
     endpoint: ROCm
     name: ROCm/aomp
-    ref: aomp-dev
+    ref: amd-staging
+  - repository: aomp-extras_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp-extras
+    ref: amd-staging
+  - repository: flang_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/flang
+    ref: amd-staging
   - repository: llvm-project_repo
     type: github
     endpoint: ROCm
@@ -26,4 +36,4 @@ pr: none
 jobs:
   - template: ${{ variables.CI_COMPONENT_PATH }}/aomp.yml
     parameters:
-      checkoutRepo: release_repo
+      checkoutRepo: aomp_repo

.azuredevops/components/AMDMIGraphX.yml

@@ -11,6 +11,7 @@ parameters:
     - git
     - cmake
     - ninja-build
+    - libdrm-dev
     - libnuma-dev
     - python3-pip
     - python3-venv
@@ -34,17 +35,21 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
-    - clr
-    - rocminfo
-    - rocMLIR
-    - MIOpen
     - aomp
     - aomp-extras
-    - rocBLAS
+    - clr
     - composable_kernel
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - MIOpen
+    - rocm-cmake
+    - ROCR-Runtime
+    - rocBLAS
+    - rocminfo
+    - rocMLIR
+    - rocprofiler-register
 
 jobs:
 - job: AMDMIGraphX
@@ -54,6 +59,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -68,18 +79,18 @@ jobs:
     parameters:
       dependencySource: fixed
       fixedComponentName: half
-      fixedPipelineIdentifier: $(half560-pipeline-id)
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      fixedPipelineIdentifier: ${{ variables.HALF560_PIPELINE_ID }}
+      skipLibraryLinking: true
+      skipLlvmSymlink: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -87,7 +98,8 @@ jobs:
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
         -DCMAKE_BUILD_TYPE=Release
-        -DGPU_TARGETS=gfx942
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
         -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
         -DMIGRAPHX_USE_COMPOSABLEKERNEL=OFF
@@ -95,3 +107,5 @@ jobs:
         -GNinja
 # REFERENCE: https://github.com/ROCm/composable_kernel/issues/782
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/HIP.yml

@@ -10,6 +10,9 @@ parameters:
   default:
     - libnuma-dev
     - mesa-common-dev
+    - ocl-icd-libopencl1
+    - ocl-icd-opencl-dev
+    - opencl-headers
 - name: pipModules
   type: object
   default:
@@ -18,7 +21,6 @@ parameters:
   type: object
   default:
     - llvm-project
-    - ROCT-Thunk-Interface
     - ROCR-Runtime
     - rocprofiler-register
 - name: rocmDependenciesNvidia
@@ -55,17 +57,17 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: matching_repo
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependenciesAMD }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: hipother_repo
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependenciesAMD }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependenciesAMD }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
 # compile clr
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
@@ -98,6 +100,7 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
       pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
 # checkout triggering repo (either HIP or clr)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
@@ -110,17 +113,14 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: hipother_repo
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependenciesNvidia }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependenciesNvidia }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependenciesNvidia }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - script: 'ls -1R $(Agent.BuildDirectory)/rocm'
     displayName: 'Artifact listing'

.azuredevops/components/HIPIFY.yml

@@ -11,45 +11,97 @@ parameters:
     - cmake
     - ninja-build
     - libnuma-dev
-- name: rocmDependencies
-  type: object
-  default:
-    - llvm-project
+    - python3-dev
+    - python3-pip
+    - python-is-python3
+    - mesa-common-dev
+    - ccache
+    - cuda-toolkit
+    - cudnn
 
 jobs:
 - job: HIPIFY
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  - name: UPSTREAM_LLVM_GIT_URL
+    value: https://github.com/llvm/llvm-project.git
+  - name: UPSTREAM_LLVM_TAG
+    value: llvmorg-18.1.2
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
   steps:
+  - task: Bash@3
+    displayName: 'Register CUDA packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-archive-keyring.gpg | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/cuda-archive-keyring.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" | sudo tee /etc/apt/sources.list.d/cuda-ubuntu2204-x86_64.list
+        sudo apt update
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: Bash@3
+    displayName: git clone upstream llvm-project
+    inputs:
+      targetType: inline
+      script: git clone $(UPSTREAM_LLVM_GIT_URL) --depth=1 --branch $(UPSTREAM_LLVM_TAG) --recurse-submodules
+      workingDirectory: $(Pipeline.Workspace)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
-        dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
-        dependencySource: tag-builds
+  - script: |
+      mkdir -p $(CCACHE_DIR)
+      echo "##vso[task.prependpath]/usr/lib/ccache:/usr/local/cuda/bin"
+    displayName: Update path for cuda and ccache
+  - task: Cache@2
+    displayName: Ccache caching
+    inputs:
+      key: HIPIFY | $(Agent.OS) | "$(UPSTREAM_LLVM_TAG)"
+      path: $(CCACHE_DIR)
+      restoreKeys: HIPIFY | $(Agent.OS)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
+      componentName: upstream-llvm
+      cmakeBuildDir: $(Pipeline.Workspace)/llvm-project/llvm/build
+      installDir: $(Pipeline.Workspace)/llvm
       extraBuildFlags: >-
         -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DLLVM_ENABLE_PROJECTS=clang
+        -DLLVM_INCLUDE_TESTS=OFF
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
         -GNinja
+  - task: Bash@3
+    displayName: python install lit
+    inputs:
+      targetType: inline
+      script: sudo python3 $(Pipeline.Workspace)/llvm-project/llvm/utils/lit/setup.py install
+  - task: Bash@3
+    displayName: install FileCheck
+    inputs:
+      targetType: inline
+      script: cp $(Pipeline.Workspace)/llvm-project/llvm/build/bin/FileCheck $(Pipeline.Workspace)/llvm/bin
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      componentName: HIPIFY
+      extraBuildFlags: >-
+        -DHIPIFY_CLANG_TESTS=ON
+        -DCMAKE_BUILD_TYPE=Release
+        -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda/targets/x86_64-linux
+        -DCUDA_DNN_ROOT_DIR=/usr/local/cuda/targets/x86_64-linux
+        -DCMAKE_PREFIX_PATH=$(Pipeline.Workspace)/llvm;/usr/local/cuda/targets/x86_64-linux/lib
+        -DLLVM_EXTERNAL_LIT=$(Pipeline.Workspace)/llvm-project/llvm/build/bin/llvm-lit
+      multithreadFlag: -- -j32
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: HIPIFY
+      testDir: $(Build.SourcesDirectory)/build
+      testExecutable: make
+      testParameters: test-hipify
+      testPublishResults: false

.azuredevops/components/MIOpen.yml

@@ -25,6 +25,7 @@ parameters:
     - rocBLAS
     - hipBLAS
     - hipBLASLt
+    - hipBLAS-common
     - half
     - composable_kernel
     - rocm-cmake
@@ -40,9 +41,15 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LARGE_DISK_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -63,17 +70,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -87,3 +92,5 @@ jobs:
         -DBUILD_TESTING=ON
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/MIVisionX.yml

@@ -58,6 +58,20 @@ parameters:
     - AMDMIGraphX
     - rpp
     - rocDecode
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - half
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - MIOpen
+    - rocBLAS
+    - rocprofiler-register
+    - ROCR-Runtime
+    - roctracer
+    - rpp
 
 jobs:
 - job: MIVisionX
@@ -68,6 +82,12 @@ jobs:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -77,17 +97,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -97,3 +115,54 @@ jobs:
         -DROCM_DEP_ROCMCORE=ON
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: MIVisionX_testing
+  dependsOn: MIVisionX
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  # anything in /opt may be persistent across runs
+  # so we need to remove the symlink if it already exists
+  - script: |
+      sudo rm -f /opt/rocm
+      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      mkdir mivisionx-tests
+      cd mivisionx-tests
+      cmake /opt/rocm/share/mivisionx/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: MIVisionX
+      testDir: 'mivisionx-tests'

.azuredevops/components/ROCR-Runtime.yml

@@ -16,7 +16,6 @@ parameters:
   type: object
   default:
     - llvm-project
-    - ROCT-Thunk-Interface
     - rocprofiler-register
 
 jobs:
@@ -36,21 +35,18 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-      cmakeBuildDir: 'src/build'
+        -DBUILD_SHARED_LIBS=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/ROCdbgapi.yml

@@ -35,17 +35,14 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:

.azuredevops/components/ROCgdb.yml

@@ -5,24 +5,46 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# reference: https://github.com/ROCm/ROCgdb/blob/amd-staging/README-ROCM.md
 - name: aptPackages
   type: object
   default:
-    - libgmp-dev
-    - libmpfr-dev
-    - texinfo
     - bison
+    - dejagnu
     - flex
+    - libbabeltrace-dev
+    - libexpat-dev
+    - libgmp-dev
+    - liblzma-dev
+    - libmpfr-dev
+    - ncurses-dev
+    - texinfo
+    - zlib1g-dev
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - ROCdbgapi
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
-- job: rocgdb
+- job: ROCgdb
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  - name: PKG_CONFIG_PATH
+    value: $(Agent.BuildDirectory)/rocm/share/pkgconfig
+  pool: $(JOB_TEST_POOL)
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -31,5 +53,63 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-autotools.yml
+    parameters:
+      configureFlags: >-
+        --program-prefix=roc
+        --enable-64-bit-bfd
+        --enable-targets="x86_64-linux-gnu,amdgcn-amd-amdhsa"
+        --disable-ld
+        --disable-gas
+        --disable-gdbserver
+        --disable-sim
+        --enable-tui
+        --disable-gdbtk
+        --disable-shared
+        --disable-gprofng
+        --with-expat
+        --with-system-zlib
+        --without-guile
+        --with-babeltrace
+        --with-lzma
+        --with-python=python3
+        --with-rocm-dbgapi=$(Agent.BuildDirectory)/rocm
+        LDFLAGS="-Wl,--enable-new-dtags,-rpath=$(Agent.BuildDirectory)/rocm/lib"
+      makeCallPrefix: LD_RUN_PATH='${ORIGIN}/../lib'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        echo "##vso[task.prependpath]/opt/rocm/bin"
+  - task: Bash@3
+    displayName: check-gdb
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: make check-gdb TESTS=gdb.rocm/simple.exp
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: print gdb log
+    inputs:
+      targetType: inline
+      script: find -name gdb.log -exec cat {} \;
+      workingDirectory: $(Build.SourcesDirectory)

.azuredevops/components/ROCmValidationSuite.yml

@@ -19,6 +19,8 @@ parameters:
   type: object
   default:
     - clr
+    - hipBLAS-common
+    - hipBLASLt
     - hipRAND
     - llvm-project
     - rocBLAS
@@ -28,7 +30,19 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocRAND
-    - ROCT-Thunk-Interface
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - hipRAND
+    - llvm-project
+    - rocBLAS
+    - rocm_smi_lib
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
 
 jobs:
 - job: ROCmValidationSuite
@@ -45,6 +59,12 @@ jobs:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -53,24 +73,16 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
-# Set link to redirect llvm folder
-  - task: Bash@3
-    displayName: create symlink
-    inputs:
-      targetType: inline
-      script: ln -s $(Agent.BuildDirectory)/rocm/llvm $(Agent.BuildDirectory)/rocm/lib/llvm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -80,3 +92,49 @@ jobs:
         -DCPACK_PACKAGING_INSTALL_PREFIX=$(Build.BinariesDirectory)
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: ROCmValidationSuite_testing
+  dependsOn: ROCmValidationSuite
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: ROCmValidationSuite
+      testExecutable: $(Agent.BuildDirectory)/rocm/bin/rvs -c $(Agent.BuildDirectory)/rocm/share/rocm-validation-suite/conf/MI300X/gst_single.conf
+      testParameters: ''
+      testDir: $(Agent.BuildDirectory)
+      testPublishResults: false

.azuredevops/components/amdsmi.yml

@@ -32,3 +32,30 @@ jobs:
       extraBuildFlags: >-
         -DBUILD_TESTS=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: amdsmi_testing
+  dependsOn: amdsmi
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: amdsmi
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/share/amd_smi/tests/amdsmitst'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/aomp-extras.yml

@@ -35,17 +35,14 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:

.azuredevops/components/aomp.yml

@@ -5,9 +5,8 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
-- name: offloadEnabled
-  type: boolean
-  default: false
+# reference:
+# https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL_PREREQUISITE.md
 - name: aptPackages
   type: object
   default:
@@ -30,21 +29,52 @@ parameters:
     - libsystemd-dev
     - libssl-dev
     - libstdc++-12-dev
+    - ccache
+    - libgmp-dev
+    - libmpfr-dev
+    - texinfo
+    - libbison-dev
+    - bison
+    - flex
+    - libbabeltrace-dev
+    - libncurses5-dev
+    - liblzma-dev
+    - python3-setuptools
+    - python3-dev
+    - libudev-dev
+    - parallel
+  # Referencing comment snippet below but excluding rocprofiler and roctracer.
+  # This is to remove need for separate build per gpu target.
+  # With our selected build flags, compilation and installation work fine without these two.
+  #
+  # snippet from https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L131-L134
+  #
+  # For ROCM build (AOMP_STANDALONE_BUILD=0) the components roct, rocr,
+  # libdevice, project, comgr, rocminfo, hipamd, rocdbgapi, rocgdb,
+  # roctracer, rocprofiler, rocm_smi_lib, and amdsmi should be found
+  # in ROCM in /opt/rocm.  The ROCM build only needs these components:
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
+    - amdsmi
+    - clr
     - llvm-project
+    - ROCdbgapi
+    - ROCgdb
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - rocm_smi_lib
+    - rocprofiler-register
     - ROCR-Runtime
-    - ROCT-Thunk-Interface
+    - roctracer
 
 jobs:
 - job: aomp
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -52,74 +82,407 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+# checkout the repos tied to openmp-extras, plus llvm-project
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
-      checkoutRepo: llvm-project_repo
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
-        dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
-        dependencySource: tag-builds
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+      checkoutRepo: aomp-extras_repo
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
-      componentName: llvm-openmp
-      extraBuildFlags: >-
-        -DOPENMP_ENABLE_LIBOMPTARGET=1
-        -DOPENMP_TEST_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-        -DOPENMP_TEST_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-        -DLIBOMPTARGET_AMDGCN_GFXLIST=gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906;gfx908;gfx90a;gfx90c;gfx940;gfx941;gfx942;gfx1030;gfx1031;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103
-        -DDEVICELIBS_ROOT=$(Build.SourcesDirectory)/llvm-project/amd/device-libs
-        -DLIBOMP_COPY_EXPORTS=OFF
-        -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm
-        -DLLVM_MAIN_INCLUDE_DIR=$(Build.SourcesDirectory)/llvm-project/llvm/include
-        -DLIBOMPTARGET_LLVM_INCLUDE_DIRS=$(Build.SourcesDirectory)/llvm-project/llvm/include
-        -DCUDA_TOOLKIT_ROOT_DIR=OFF
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/lib/cmake
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_SKIP_BUILD_RPATH=TRUE
-        -DCMAKE_SKIP_INSTALL_RPATH=TRUE
-        -DCMAKE_SHARED_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-        -DCMAKE_EXE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-        -DCMAKE_MODULE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-        -GNinja
-      cmakeBuildDir: $(Build.SourcesDirectory)/llvm-project/openmp/build
-      installDir: $(Build.BinariesDirectory)/llvm
-# offload does not exist for recent releases, so use CI conditional
-  - ${{ if eq(parameters.offloadEnabled, true) }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      parameters:
-        componentName: llvm-offload
-        extraBuildFlags: >-
-          -DOPENMP_ENABLE_LIBOMPTARGET=1
-          -DOPENMP_TEST_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-          -DOPENMP_TEST_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-          -DLIBOMPTARGET_AMDGCN_GFXLIST=gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906;gfx908;gfx90a;gfx90c;gfx940;gfx941;gfx942;gfx1030;gfx1031;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103
-          -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm
-          -DLLVM_MAIN_INCLUDE_DIR=$(Build.SourcesDirectory)/llvm-project/llvm/include
-          -DLIBOMPTARGET_LLVM_INCLUDE_DIRS=$(Build.SourcesDirectory)/llvm-project/llvm/include
-          -DCUDA_TOOLKIT_ROOT_DIR=OFF
-          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/lib/cmake
-          -DCMAKE_BUILD_TYPE=Release
-          -DCMAKE_SKIP_BUILD_RPATH=TRUE
-          -DCMAKE_SKIP_INSTALL_RPATH=TRUE
-          -DCMAKE_SHARED_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-          -DCMAKE_EXE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-          -DCMAKE_MODULE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-          -GNinja
-        cmakeBuildDir: $(Build.SourcesDirectory)/llvm-project/offload/build
-        installDir: $(Build.BinariesDirectory)/llvm
+      checkoutRepo: flang_repo
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: llvm-project_repo
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+# Because clang is not being rebuilt and to separate downloaded ROCm
+# dependencies from the new artifacts, we use temporary symbolic links
+# for the compilation and installation to go through.
+  - script: |
+      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      mkdir -p $(Build.BinariesDirectory)/bin
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang $(Build.BinariesDirectory)/bin/clang
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang++ $(Build.BinariesDirectory)/bin/clang++
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/llvm-config $(Build.BinariesDirectory)/bin/llvm-config
+      ln -s $(Agent.BuildDirectory)/rocm/llvm $(Build.BinariesDirectory)/llvm
+    displayName: Extra build environment setup
+# We follow the sequence described in the aomp repo instructions
+# https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL.md
+# But instead of calling build_aomp.sh directly, we have to split up the calls to build each component
+# in order for the omp.h header file to be both be published as an artifact,
+# and be found when building subsequent components.
+# Environment variables specified per bash script were determined from looking through the
+# individual scripts and iteratively testing compilation.
+# Splitting up the build_aomp.sh call is also easier for debugging within Azure, as the former
+# method leads to a giant build log compared to separate logs per script call.
+#
+# Components compiled and the order for non-standalone build found at
+# https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L135-L143
+  - task: Bash@3
+    displayName: Build Prereq
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_prereq.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Build extras
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_extras.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_EXTRAS: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install extras
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_extras.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_EXTRAS: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build openmp
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_openmp.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install openmp
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_openmp.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - script: ln -s $(Build.BinariesDirectory)/include/omp.h $(Build.SourcesDirectory)/llvm-project/llvm/include/omp.h
+    displayName: Link omp header
+  - task: Bash@3
+    displayName: Build offload
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_offload.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install offload
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_offload.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build flang-legacy
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-legacy.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install flang-legacy
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-legacy.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build pgmath
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_pgmath.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install pgmath
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_pgmath.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build flang
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install flang
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build flang_runtime
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang_runtime.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install flang_runtime
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang_runtime.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+# Clean up build environment before publish artifact
+  - script: |
+      rm $(Build.BinariesDirectory)/bin/clang
+      rm $(Build.BinariesDirectory)/bin/clang++
+      rm $(Build.BinariesDirectory)/bin/llvm-config
+      rm $(Build.BinariesDirectory)/llvm
+    displayName: Remove temporary symbolic links
+# Copy the files to artifact staging temporarily to clean up binaries directory
+# and then copy files back to llvm subdirectory in the cleaned up binaries directory
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.BinariesDirectory)
+      targetDir: $(Build.ArtifactStagingDirectory)
+  - task: DeleteFiles@1
+    displayName: 'Cleanup Binaries Directory'
+    inputs:
+      SourceFolder: '$(Build.BinariesDirectory)'
+      Contents: '/**/*'
+      RemoveDotFiles: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.ArtifactStagingDirectory)
+      targetDir: $(Build.BinariesDirectory)/llvm
+  - task: DeleteFiles@1
+    displayName: 'Cleanup Staging Directory'
+    inputs:
+      SourceFolder: $(Build.ArtifactStagingDirectory)
+      Contents: '/**/*'
+      RemoveDotFiles: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: aomp_testing
+  dependsOn: aomp
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: aomp-extras_repo
+# these copy steps are from the aomp prototype script for test prep
+  - task: CopyFiles@2
+    displayName: 'Copy AOMP contents'
+    inputs:
+      CleanTargetFolder: false
+      SourceFolder: $(Build.SourcesDirectory)/aomp
+      Contents: |
+        **
+        !**/.git/**
+        !**/.github/**
+        !**/.gitignore
+      TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests
+      retryCount: 3
+  - task: CopyFiles@2
+    displayName: 'Copy FileCheck'
+    inputs:
+      CleanTargetFolder: false
+      SourceFolder: $(Agent.BuildDirectory)/rocm/llvm/bin
+      Contents: FileCheck
+      TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests/bin
+      retryCount: 3
+  - task: Bash@3
+    displayName: Test AOMP
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: ./run_rocm_test.sh
+      workingDirectory: $(Build.SourcesDirectory)/aomp/bin
+    env:
+      AOMP: $(Agent.BuildDirectory)/rocm/llvm
+      AOMP_REPOS_TEST: $(Build.SourcesDirectory)/aomp-test
+      AOMP_TEST_DIR: $(Build.SourcesDirectory)/aomp-test
+      SKIP_TEST_PACKAGE: 1
+      MAINLINE_BUILD: 1
+      SUITE_LIST: smoke

.azuredevops/components/composable_kernel.yml

@@ -13,6 +13,7 @@ parameters:
     - git
     - python3-pip
     - libdrm-dev
+    - ccache
 - name: rocmDependencies
   type: object
   default:
@@ -22,16 +23,33 @@ parameters:
     - clr
     - rocminfo
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - composable_kernel
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: composable_kernel
-  timeoutInMinutes: 100
+  timeoutInMinutes: 240
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
+  - name: DAY_STRING
+    value: $[format('{0:ddMMyyyy}', pipeline.startTime)]
   pool: ${{ variables.ULTRA_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -40,25 +58,89 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - script: |
+      mkdir -p $(CCACHE_DIR)
+      echo "##vso[task.prependpath]/usr/lib/ccache"
+    displayName: Update path for ccache
+  - task: Cache@2
+    displayName: Ccache caching
+    inputs:
+      key: composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING) | $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+      path: $(CCACHE_DIR)
+      restoreKeys: |
+        composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING)
+        composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET)
+        composable_kernel | $(Agent.OS)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
+        -DCMAKE_HIP_FLAGS="-Wno-missing-include-dirs"
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
-        -DGPU_TARGETS=gfx942
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: composable_kernel_testing
+  dependsOn: composable_kernel
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: TEST_LOG_FILE
+    value: $(Pipeline.Workspace)/ckTestLog.log
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Iterate through test scripts
+    inputs:
+      targetType: inline
+      script: |
+        for file in ./test_*; do
+          ./$file | tee -a $(TEST_LOG_FILE)
+        done
+      workingDirectory: $(Agent.BuildDirectory)/rocm/bin

.azuredevops/components/copyHIP.yml

@@ -24,9 +24,8 @@ jobs:
   - checkout: none
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-download.yml
     parameters:
-      useDefaultBranch: false
       componentName: HIP
-      branchName: develop
+      pipelineId: $(hip-pipeline-id)
   - template:  ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
       sourceDir: $(Agent.BuildDirectory)/rocm

.azuredevops/components/half.yml

@@ -37,17 +37,14 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  # CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-  # manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -56,3 +53,13 @@ jobs:
         -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  # only run test03 because test11 has too many test cases, taking way too long
+  - task: Bash@3
+    displayName: test03
+    inputs:
+      targetType: inline
+      script: |
+        make clean
+        make test03
+        ./bin/test
+      workingDirectory: $(Build.SourcesDirectory)/test

.azuredevops/components/hip-tests.yml

@@ -0,0 +1,144 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - ninja-build
+    - git
+    - libnuma-dev
+    - mesa-common-dev
+    - ocl-icd-libopencl1
+    - ocl-icd-opencl-dev
+    - opencl-headers
+    - libglu1-mesa-dev
+    - freeglut3-dev
+    - libboost-all-dev
+    - python3-pip
+- name: rocmDependencies
+  type: object
+  default:
+    - llvm-project
+    - ROCR-Runtime
+    - clr
+    - rocminfo
+    - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+
+# HIP with AMD backend
+jobs:
+- job: hip_tests
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: HIP_ROCCLR_HOME
+    value: $(Agent.BuildDirectory)/rocm
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  # compile hip-tests
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      componentName: hip-tests
+      cmakeSourceDir: '../catch'
+      customBuildTarget: build_tests
+      extraBuildFlags: >-
+        -DHIP_PLATFORM=amd
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DHIP_PATH=$(Agent.BuildDirectory)/rocm
+        -DOFFLOAD_ARCH_STR="--offload-arch=$(JOB_GPU_TARGET)"
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hip_tests_testing
+  timeoutInMinutes: 180
+  dependsOn: hip_tests
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Symlink rocm_agent_enumerator
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir -p /opt/rocm/bin
+        sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rocm_agent_enumerator /opt/rocm/bin/rocm_agent_enumerator
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hip_tests
+      testDir: $(Agent.BuildDirectory)/rocm/share/hip
+      reloadAMDGPU: true
+  - task: Bash@3
+    displayName: Clean up symlink
+    inputs:
+      targetType: inline
+      script: sudo rm -rf /opt/rocm

.azuredevops/components/hipBLAS-common.yml

@@ -0,0 +1,62 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - ninja-build
+    - git
+    - wget
+    - python3-pip
+- name: rocmDependencies
+  type: object
+  default:
+    - rocm-cmake
+    - llvm-project
+    - ROCR-Runtime
+    - clr
+    - rocminfo
+
+jobs:
+- job: hipBLAS_common
+  variables:
+  - group: common
+  - name: ROCM_PATH
+    value: $(Agent.BuildDirectory)/rocm
+  - template: /.azuredevops/variables-global.yml
+  pool: 
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+# CI case: download latest default branch build
+  - ${{ if eq(parameters.checkoutRef, '') }}:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
+        dependencySource: staging
+# manual build case: triggered by ROCm/ROCm repo
+  - ${{ if ne(parameters.checkoutRef, '') }}:
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/hipBLAS.yml

@@ -16,20 +16,36 @@ parameters:
     - libgtest-dev
     - wget
     - python3-pip
-    - libomp-dev
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
+    - aomp
     - clr
+    - hipBLAS-common
+    - llvm-project
+    - rocm-cmake
+    - ROCR-Runtime
     - rocminfo
     - rocprofiler-register
     - rocBLAS
     - rocSPARSE
     - rocSOLVER
+- name: rocmTestDependencies
+  type: object
+  default:
     - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocm-cmake
+    - ROCR-Runtime
+    - rocminfo
+    - rocprofiler-register
+    - rocBLAS
+    - rocSPARSE
+    - rocSOLVER
+    - roctracer
 
 jobs:
 - job: hipBLAS
@@ -39,6 +55,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -47,29 +69,16 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - task: Bash@3
-    displayName: 'Download AOCL'
-    inputs:
-      targetType: inline
-      script: wget -nv https://download.amd.com/developer/eula/aocl/aocl-4-1/aocl-linux-aocc-4.1.0_1_amd64.deb
-      workingDirectory: '$(Pipeline.Workspace)'
-  - task: Bash@3
-    displayName: 'Install AOCL'
-    inputs:
-      targetType: inline
-      script: sudo apt install --yes ./aocl-linux-aocc-4.1.0_1_amd64.deb
-      workingDirectory: '$(Pipeline.Workspace)'
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -84,3 +93,48 @@ jobs:
         -DCPACK_SET_DESTDIR=OFF
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipBLAS_testing
+  dependsOn: hipBLAS
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipBLAS
+      testExecutable: $(Agent.BuildDirectory)/rocm/bin/hipblas-test
+      testParameters: '--yaml hipblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'

.azuredevops/components/hipBLASLt.yml

@@ -8,13 +8,16 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - gfortran
     - git
     - libdrm-dev
     - libmsgpack-dev
+    - libnuma-dev
     - ninja-build
     - python3-pip
     - python3-venv
+    - gfortran
+    - libblas-dev
+    - ccache
 - name: pipModules
   type: object
   default:
@@ -22,12 +25,27 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
+    - aomp
     - clr
-    - hipBLAS
+    - hipBLAS-common
     - llvm-project
     - rocminfo
+    - rocm_smi_lib
     - rocprofiler-register
     - ROCR-Runtime
+    - roctracer
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - llvm-project
+    - rocminfo
+    - rocm_smi_lib
+    - rocprofiler-register
+    - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: hipBLASLt
@@ -47,9 +65,17 @@ jobs:
     value: $(Agent.BuildDirectory)/rocm/bin/hipcc
   - name: PATH
     value: $(Agent.BuildDirectory)/rocm/llvm/bin:$(Agent.BuildDirectory)/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  - name: DAY_STRING
+    value: $[format('{0:ddMMyyyy}', pipeline.startTime)]
+  pool: ${{ variables.ULTRA_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -60,17 +86,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
     displayName: ROCm symbolic link
@@ -78,6 +102,7 @@ jobs:
 # $(Pipeline.Workspace)/deps is a temporary folder for the build process
 # $(Pipeline.Workspace)/s/deps is part of the hipBLASLt repo
   - script: mkdir $(Pipeline.Workspace)/deps
+    displayName: Create temp folder for external dependencies
 # hipBLASLt already has a CMake script for external deps, so we can just run that
 # https://github.com/ROCm/hipBLASLt/blob/develop/deps/CMakeLists.txt
   - script: cmake $(Pipeline.Workspace)/s/deps
@@ -89,23 +114,79 @@ jobs:
   - script: sudo make install
     displayName: Install hipBLASLt external dependencies
     workingDirectory: $(Pipeline.Workspace)/deps
-# Set link to redirect llvm folder
-  - task: Bash@3
-    displayName: Symlink to rocm/lib/llvm
+  - script: |
+      mkdir -p $(CCACHE_DIR)
+      echo "##vso[task.prependpath]/usr/lib/ccache"
+    displayName: Update path for ccache
+  - task: Cache@2
+    displayName: Ccache caching
     inputs:
-      targetType: inline
-      script: ln -s $(Agent.BuildDirectory)/rocm/llvm $(Agent.BuildDirectory)/rocm/lib/llvm
+      key: hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING) | $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+      path: $(CCACHE_DIR)
+      restoreKeys: |
+        hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING)
+        hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET)
+        hipBLASLt | $(Agent.OS)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_BUILD_TYPE=Release
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DAMDGPU_TARGETS=gfx942
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DTensile_LOGIC=
         -DTensile_CPU_THREADS=
         -DTensile_CODE_OBJECT_VERSION=default
         -DTensile_LIBRARY_FORMAT=msgpack
         -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
+        -DBUILD_CLIENTS_TESTS=ON
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+
+- job: hipBLASLt_testing
+  dependsOn: hipBLASLt
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipBLASLt
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipblaslt-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'

.azuredevops/components/hipCUB.yml

@@ -10,7 +10,7 @@ parameters:
   default:
     - cmake
     - ninja-build
-    - googletest
+    - libgtest-dev
     - git
     - python3-pip
 - name: rocmDependencies
@@ -21,6 +21,15 @@ parameters:
     - rocminfo
     - rocPRIM
     - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocPRIM
+    - ROCR-Runtime
+    - rocprofiler-register
 
 jobs:
 - job: hipCUB
@@ -30,6 +39,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -38,17 +53,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -57,6 +70,49 @@ jobs:
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_TEST=ON
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipCUB_testing
+  dependsOn: hipCUB
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipCUB
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/hipcub'

.azuredevops/components/hipFFT.yml

@@ -14,11 +14,11 @@ parameters:
     - libgtest-dev
     - libfftw3-dev
     - python3-pip
-    - libomp-14-dev
 # rocm dependencies should match dependencies-rocm.yml
 - name: rocmDependencies
   type: object
   default:
+    - aomp
     - rocRAND
     - hipRAND
     - llvm-project
@@ -27,6 +27,19 @@ parameters:
     - rocminfo
     - rocFFT
     - aomp
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipRAND
+    - llvm-project
+    - rocFFT
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+
 jobs:
 - job: hipFFT
   variables:
@@ -38,6 +51,12 @@ jobs:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -46,17 +65,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  # CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -66,7 +83,7 @@ jobs:
         -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DUSE_HIP_CLANG=ON
         -DHIP_COMPILER=clang
         -DBUILD_CLIENTS_TESTS=ON
@@ -74,3 +91,48 @@ jobs:
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipFFT_testing
+  dependsOn: hipFFT
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipFFT
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipfft-test'
+      testParameters: '--test_prob 0.002 --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/hipRAND.yml

@@ -20,6 +20,15 @@ parameters:
     - clr
     - rocminfo
     - rocRAND
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
 
 jobs:
 - job: hipRAND
@@ -32,6 +41,12 @@ jobs:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -40,17 +55,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -61,6 +74,49 @@ jobs:
         -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipRAND_testing
+  dependsOn: hipRAND
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipRAND
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/hipRAND'

.azuredevops/components/hipSOLVER.yml

@@ -27,6 +27,21 @@ parameters:
     - ROCR-Runtime
     - rocSPARSE
     - rocSOLVER
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - hipBLAS-common
+    - hipBLASLt
+    - hipSPARSE
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocSOLVER
+    - rocSPARSE
 
 jobs:
 - job: hipSOLVER
@@ -37,6 +52,12 @@ jobs:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -45,17 +66,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
 # build external gtest and lapack
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
@@ -78,3 +97,48 @@ jobs:
         -DUSE_CUDA=OFF
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipSOLVER_testing
+  dependsOn: hipSOLVER
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipSOLVER
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipsolver-test'
+      testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/hipSPARSE.yml

@@ -25,6 +25,18 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocSPARSE
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - hipBLAS-common
+    - hipBLASLt
+    - rocBLAS
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocSPARSE
 
 jobs:
 - job: hipSPARSE
@@ -35,6 +47,12 @@ jobs:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -43,17 +61,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -67,6 +83,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: hipSPARSE
+      gpuTarget: $(JOB_GPU_TARGET)
       publish: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
@@ -75,3 +92,47 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: testMatrices
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipSPARSE_testing
+  dependsOn: hipSPARSE
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipSPARSE
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipsparse-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/hipSPARSELt.yml

@@ -13,21 +13,36 @@ parameters:
     - libmsgpack-dev
     - git
     - python3-pip
+    - gfortran
+    - libgfortran5
+    - libopenblas-dev
 - name: pipModules
   type: object
   default:
     - joblib
-# rocm dependencies should match dependencies-rocm.yml
 - name: rocmDependencies
   type: object
   default:
-    - llvm-project
-    - ROCR-Runtime
+    - aomp
     - clr
+    - hipSPARSE
+    - llvm-project
+    - rocBLAS
     - rocminfo
     - rocprofiler-register
-    - hipSPARSE
+    - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - llvm-project
+    - hipBLAS-common
+    - hipBLASLt
     - rocBLAS
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: hipSPARSELt
@@ -47,6 +62,10 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -57,30 +76,84 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+# Build and install gtest and lapack
+# $(Pipeline.Workspace)/deps is a temporary folder for the build process
+# $(Pipeline.Workspace)/s/deps is part of the hipSPARSELt repo
+  - script: mkdir $(Pipeline.Workspace)/deps
+    displayName: Create temp folder for external dependencies
+# hipSPARSELt already has a CMake script for external deps, so we can just run that
+# https://github.com/ROCm/hipSPARSELt/blob/develop/deps/CMakeLists.txt
+  - script: cmake $(Pipeline.Workspace)/s/deps
+    displayName: Configure hipSPARSELt external dependencies
+    workingDirectory: $(Pipeline.Workspace)/deps
+  - script: make
+    displayName: Build hipSPARSELt external dependencies
+    workingDirectory: $(Pipeline.Workspace)/deps
+  - script: sudo make install
+    displayName: Install hipSPARSELt external dependencies
+    workingDirectory: $(Pipeline.Workspace)/deps
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_BUILD_TYPE=Release
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DTensile_LOGIC=
         -DTensile_CPU_THREADS=
         -DTensile_CODE_OBJECT_VERSION=default
         -DTensile_LIBRARY_FORMAT=msgpack
         -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DBUILD_CLIENTS_TESTS=ON
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipSPARSELt_testing
+  dependsOn: hipSPARSELt
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipSPARSELt
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipsparselt-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'

.azuredevops/components/hipTensor.yml

@@ -21,6 +21,14 @@ parameters:
     - clr
     - rocminfo
     - composable_kernel
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: hipTensor
@@ -30,6 +38,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -38,17 +52,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -58,6 +70,51 @@ jobs:
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
         -DHIPTENSOR_BUILD_TESTS=ON
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
       multithreadFlag: -- -j32
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipTensor_testing
+  timeoutInMinutes: 90
+  dependsOn: hipTensor
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipTensor
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/hiptensor'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/hipfort.yml

@@ -11,26 +11,137 @@ parameters:
     - gfortran
     - git
     - cmake
+    - python3-pip
+    - python3-dev
+    - ninja-build
+- name: rocmDependencies
+  type: object
+  default:
+    - aomp
+    - llvm-project
+    - clr
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - hipFFT
+    - hipRAND
+    - hipSOLVER
+    - hipSPARSE
+    - rocBLAS
+    - rocFFT
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler
+    - rocprofiler-register
+    - rocRAND
+    - ROCR-Runtime
+    - rocSOLVER
+    - rocSPARSE
+    - roctracer
 
 jobs:
 - job: hipfort
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/llvm
+        -DHIPFORT_INSTALL_DIR=$(Build.BinariesDirectory)
+        -DHIPFORT_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/flang
+        -DCMAKE_Fortran_FLAGS="-Mfree -fPIC"
+        -DCMAKE_Fortran_FLAGS_DEBUG=""
+        -DHIPFORT_COMPILER_FLAGS="-cpp"
+        -DHIPFORT_AR=$(Agent.BuildDirectory)/rocm/llvm/bin/llvm-ar
+        -DHIPFORT_RANLIB=$(Agent.BuildDirectory)/rocm/llvm/bin/llvm-ranlib
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DBUILD_TESTING=ON
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipfort_testing
+  dependsOn: hipfort
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DHIPFORT_INSTALL_DIR=/tmp/hipfort
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - task: Bash@3
+    displayName: 'ROCm symbolic link'
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: 'Test hipfort'
+    inputs:
+      targetType: inline
+      script: PATH=$(Agent.BuildDirectory)/rocm/bin:$PATH make run_all
+      workingDirectory: $(Build.SourcesDirectory)/test

.azuredevops/components/llvm-project.yml

@@ -38,17 +38,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      skipLlvmSymlink: true
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -106,6 +104,14 @@ jobs:
       testExecutable: './bin/llvm-lit'
       testParameters: '-q --xunit-xml-output=lld_test_output.xml ./tools/lld/test'
       testOutputFile: lld_test_output.xml
+  - task: CopyFiles@2
+    displayName: Copy FileCheck for Publishing
+    inputs:
+      CleanTargetFolder: false
+      SourceFolder: llvm/build/bin
+      Contents: FileCheck
+      TargetFolder: $(Build.BinariesDirectory)/llvm/bin
+      retryCount: 3
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: device-libs

.azuredevops/components/omniperf.yml

@@ -0,0 +1,167 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - python3-pip
+- name: pipModules
+  type: object
+  default:
+    - astunparse==1.6.2
+    - colorlover
+    - dash>=1.12.0
+    - matplotlib
+    - numpy>=1.17.5
+    - pandas>=1.4.3
+    - pymongo
+    - pyyaml
+    - tabulate
+    - tqdm
+    - dash-svg
+    - dash-bootstrap-components
+    - kaleido
+    - setuptools
+    - plotille
+    - mock
+    - pytest
+    - pytest-cov
+    - pytest-xdist
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler
+    - rocprofiler-register
+    - roctracer
+
+jobs:
+- job: omniperf
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: omniperf_testing
+  dependsOn: omniperf
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: PYTHON_VERSION
+    value: 3.10
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -DENABLE_TESTS=ON
+        -DINSTALL_TESTS=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: omniperf
+      testDir: $(Build.BinariesDirectory)/libexec/omniperf
+      testExecutable: export OMNIPERF_ARCH_OVERRIDE="MI300X"; ctest
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"

.azuredevops/components/pipeline-debug.yml

@@ -0,0 +1,75 @@
+parameters:
+# ubuntu near-equivalent list of yum installs in https://github.com/ROCm/ROCm-docker/blob/master/dev/Dockerfile-centos-7-complete
+# plus additional packages found through iterative testing of pipeline
+- name: aptPackages
+  type: object
+  default:
+    - python3-dev
+    - python3-pip
+- name: pipModules
+  type: object
+  default:
+    - tox
+# list from https://github.com/pytorch/builder/blob/main/manywheel/build_rocm.sh
+- name: rocmDependencies
+  type: object
+  default:
+    - rocm-cmake
+    - llvm-project
+    - ROCR-Runtime
+    - clr
+    - rocminfo
+    - rocprofiler-register
+    - rocm_smi_lib
+    - rocm-core
+    - aomp
+    - aomp-extras
+    - hipBLAS-common
+    - hipBLASLt
+
+trigger: none
+pr: none
+
+jobs:
+- deployment: pipeline_debug_approval  # Deployment job (this is required for approval gates)
+  displayName: "Requires approval"
+  environment: rccl
+- job: pipeline_debug_job
+  displayName: 'Pipeline Debug Deployment'
+  timeoutInMinutes: 120
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: HIP_ROCCLR_HOME
+    value: $(Build.BinariesDirectory)/rocm
+  - name: TENSILE_ROCM_ASSEMBLER_PATH
+    value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang
+  - name: CMAKE_CXX_COMPILER
+    value: $(Agent.BuildDirectory)/rocm/bin/hipcc
+  - name: TENSILE_ROCM_OFFLOAD_BUNDLER_PATH
+    value: $(Agent.BuildDirectory)/rocm/llvm/bin/clang-offload-bundler
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  steps:
+    # Manual Approval Gate Before Continuing All Steps
+    - script: echo "Waiting for manual approval before continuing."
+      displayName: "Waiting for approval"
+    # After approval, the following steps will run
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+      parameters:
+        aptPackages: ${{ parameters.aptPackages }}
+        pipModules: ${{ parameters.pipModules }}
+    - task: Bash@3
+      displayName: 'git clone Tensile'
+      inputs:
+        targetType: inline
+        script: git clone https://github.com/ROCm/Tensile --depth=1 --shallow-submodules
+        workingDirectory: $(Build.SourcesDirectory)
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+      parameters:
+        dependencyList: ${{ parameters.rocmDependencies }}
+        gpuTarget: gfx942
+        dependencySource: staging

.azuredevops/components/rccl.yml

@@ -16,6 +16,7 @@ parameters:
     - git
     - ninja-build
     - libstdc++-12-dev
+    - libnuma-dev
 - name: rocmDependencies
   type: object
   default:
@@ -30,6 +31,20 @@ parameters:
     - HIPIFY
     - aomp
     - aomp-extras
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - aomp-extras
+    - clr
+    - HIPIFY
+    - llvm-project
+    - rocm-cmake
+    - rocm-core
+    - rocm_smi_lib
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rccl
@@ -41,25 +56,31 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  # CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      submoduleBehaviour: recursive
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-  # manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - script: chmod +x $(Agent.BuildDirectory)/rocm/bin/hipify-perl
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
@@ -72,6 +93,52 @@ jobs:
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_TESTS=ON
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/share/rocm/cmake/
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rccl_testing
+  timeoutInMinutes: 120
+  dependsOn: rccl
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rccl
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rccl-UnitTests'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rdc.yml

@@ -18,16 +18,35 @@ parameters:
     - autoconf
     - libtool
     - pkg-config
+    - libdrm-dev
+    - libyaml-cpp-dev
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
+    - amdsmi
     - clr
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
     - rocminfo
     - rocm_smi_lib
+    - ROCmValidationSuite
+    - rocprofiler
+    - rocprofiler-register
+    - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
     - amdsmi
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocm_smi_lib
+    - ROCmValidationSuite
+    - rocprofiler
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rdc
@@ -37,6 +56,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -45,17 +70,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  # CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-  # manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
 # Build grpc
   - task: Bash@3
@@ -74,11 +97,71 @@ jobs:
         -DBUILD_SHARED_LIBS=ON
         -DCMAKE_INSTALL_LIBDIR=lib
         -DCMAKE_BUILD_TYPE=Release
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DGRPC_ROOT="$(Build.SourcesDirectory)/bin"
+        -DBUILD_RVS=ON
+        -DBUILD_PROFILER=ON
         -DBUILD_TESTS=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rdc_testing
+  dependsOn: rdc
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo rm -rf /usr/sbin/rdcd
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rdcd /usr/sbin/rdcd
+        echo $(Agent.BuildDirectory)/rocm/lib/rdc/grpc/lib | sudo tee /etc/ld.so.conf.d/grpc.conf
+        sudo ldconfig -v
+  - task: Bash@3
+    displayName: Test rdc
+    inputs:
+      targetType: inline
+      script: >-
+        $(Agent.BuildDirectory)/rocm/share/rdc/rdctst_tests/rdctst
+        --batch_mode
+        --start_rdcd
+        --unauth_comm

.azuredevops/components/rocAL.yml

@@ -25,6 +25,7 @@ parameters:
     - libturbojpeg-dev
     - libjpeg-turbo-official=3.0.2-20240124
     - libopencv-dev
+    - --allow-downgrades # for libjpeg-turbo
 - name: pipModules
   type: object
   default:
@@ -44,6 +45,17 @@ parameters:
     - rpp
     - MIVisionX
     - aomp
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - half
+    - llvm-project
+    - MIVisionX
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rpp
 
 jobs:
 - job: rocAL
@@ -54,6 +66,12 @@ jobs:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - task: Bash@3
     displayName: 'Register libjpeg-turbo packages'
@@ -115,17 +133,15 @@ jobs:
       targetType: inline
       script: sudo cmake --build . --target install
       workingDirectory: '$(Build.SourcesDirectory)/rapidjson/build'
-  # CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-  # manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -136,3 +152,83 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocAL_testing
+  dependsOn: rocAL
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: ROCM_PATH
+    value: $(Agent.BuildDirectory)/rocm
+  - name: CMAKE_INCLUDE_PATH
+    value: $(Agent.BuildDirectory)/rocm/include/rocal
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - task: Bash@3
+    displayName: 'Register libjpeg-turbo packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
+        sudo apt update
+        apt-cache show libjpeg-turbo-official | grep Version
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Link libjpeg-turbo
+    inputs:
+      targetType: inline
+      script: |
+        echo /opt/libjpeg-turbo/lib64 | sudo tee /etc/ld.so.conf.d/libjpeg-turbo.conf
+        sudo ldconfig -v
+  - task: Bash@3
+    displayName: Build rocAL tests
+    inputs:
+      targetType: inline
+      script: |
+        mkdir rocAL-tests
+        cd rocAL-tests
+        cmake $(Agent.BuildDirectory)/rocm/share/rocal/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocAL
+      testDir: rocAL-tests
+      reloadAMDGPU: true
+  - task: Bash@3
+    displayName: Clean up libjpeg-turbo
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm /etc/ld.so.conf.d/libjpeg-turbo.conf
+        sudo ldconfig -v

.azuredevops/components/rocALUTION.yml

@@ -28,6 +28,21 @@ parameters:
     - ROCR-Runtime
     - rocRAND
     - rocSPARSE
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocminfo
+    - rocPRIM
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+    - rocSPARSE
 
 jobs:
 - job: rocALUTION
@@ -40,6 +55,12 @@ jobs:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -48,17 +69,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -67,9 +86,54 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/share/rocm/cmake/
         -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/lib/cmake/hip
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DBUILD_CLIENTS_TESTS=ON
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocALUTION_testing
+  dependsOn: rocALUTION
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocALUTION
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocalution-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocBLAS.yml

@@ -24,6 +24,7 @@ parameters:
   type: object
   default:
     - joblib
+    - pyyaml
 - name: rocmDependencies
   type: object
   default:
@@ -37,6 +38,20 @@ parameters:
     - rocm-core
     - aomp
     - aomp-extras
+    - hipBLAS-common
+    - hipBLASLt
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - rocm_smi_lib
+    - ROCR-Runtime
 
 jobs:
 - job: rocBLAS
@@ -54,6 +69,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -63,43 +84,17 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - task: Bash@3
-      displayName: 'Download AOCL'
-      inputs:
-        targetType: inline
-        script: wget -nv https://download.amd.com/developer/eula/aocl/aocl-4-2/aocl-linux-gcc-4.2.0_1_amd64.deb
-        workingDirectory: '$(Pipeline.Workspace)'
-    - task: Bash@3
-      displayName: 'Install AOCL'
-      inputs:
-        targetType: inline
-        script: sudo apt install --yes ./aocl-linux-gcc-4.2.0_1_amd64.deb
-        workingDirectory: '$(Pipeline.Workspace)'
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - task: Bash@3
-      displayName: 'Download AOCL'
-      inputs:
-        targetType: inline
-        script: wget -nv https://download.amd.com/developer/eula/aocl/aocl-4-1/aocl-linux-aocc-4.1.0_1_amd64.deb
-        workingDirectory: '$(Pipeline.Workspace)'
-    - task: Bash@3
-      displayName: 'Install AOCL'
-      inputs:
-        targetType: inline
-        script: sudo apt install --yes ./aocl-linux-aocc-4.1.0_1_amd64.deb
-        workingDirectory: '$(Pipeline.Workspace)'
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
-  - script: echo $PATH
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -108,7 +103,7 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DTensile_CODE_OBJECT_VERSION=default
         -DTensile_LOGIC=asm_full
         -DTensile_SEPARATE_ARCHITECTURES=ON
@@ -120,3 +115,49 @@ jobs:
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocBLAS_testing
+  dependsOn: rocBLAS
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocBLAS
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocblas-test'
+      testParameters: '--yaml rocblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocDecode.yml

@@ -29,6 +29,14 @@ parameters:
     - clr
     - rocminfo
     - rocm-core
+    - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocDecode
@@ -59,24 +67,69 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  # CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-  # manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocDecode_testing
+  dependsOn: rocDecode
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  # anything in /opt may be persistent across runs
+  # so we need to remove the symlink if it already exists
+  - script: |
+      sudo rm -rf /opt/rocm
+      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      mkdir rocDecode-tests
+      cd rocDecode-tests
+      cmake /opt/rocm/share/rocdecode/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocDecode
+      testDir: 'rocDecode-tests'
+  - script: sudo rm /opt/rocm

.azuredevops/components/rocFFT.yml

@@ -19,15 +19,28 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
-    - llvm-project
-    - ROCR-Runtime
+    - aomp
     - clr
+    - hipRAND
+    - llvm-project
+    - rocm-cmake
     - rocminfo
     - rocprofiler-register
-    - hipRAND
+    - ROCR-Runtime
     - rocRAND
-    - rocm-cmake
+- name: rocmTestDependencies
+  type: object
+  default:
     - aomp
+    - clr
+    - hipRAND
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+
 jobs:
 - job: rocFFT
   variables:
@@ -38,6 +51,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -46,17 +65,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  # CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -65,7 +82,7 @@ jobs:
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DUSE_HIP_CLANG=ON
         -DHIP_COMPILER=clang
         -DBUILD_CLIENTS_TESTS=ON
@@ -73,3 +90,48 @@ jobs:
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocFFT_testing
+  dependsOn: rocFFT
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocFFT
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocfft-test'
+      testParameters: '--test_prob 0.004 --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocMLIR.yml

@@ -12,14 +12,23 @@ parameters:
     - ninja-build
     - git
     - python3-pip
+    - libdrm-dev
+- name: pipModules
+  type: object
+  default:
+    - tomli
 - name: rocmDependencies
   type: object
   default:
     - llvm-project
     - rocm-cmake
+    - clr
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
-- job: rocMLIR
+- job: rocMLIR_library
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -30,21 +39,19 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -56,3 +63,63 @@ jobs:
         -DBUILD_FAT_LIBROCKCOMPILER=1
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+# compiling and running test on the test system together
+- job: rocMLIR_testing
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.GFX942_TEST_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      installEnabled: false
+      extraBuildFlags: >-
+        -DCMAKE_BUILD_TYPE=Release
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DROCM_TEST_CHIPSET=$(JOB_GPU_TARGET)
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocMLIR
+      testDir: $(Build.SourcesDirectory)/build
+      testExecutable: ninja
+      testParameters: check-rocmlir

.azuredevops/components/rocPRIM.yml

@@ -21,6 +21,14 @@ parameters:
     - ROCR-Runtime
     - clr
     - rocminfo
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler-register
 
 jobs:
 - job: rocPRIM
@@ -30,6 +38,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -38,20 +52,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# ${{ }} are resolved during compile-time
-# so this next step is skipped completely until
-# we define explicit aptPackages needed to install
-  # CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-  # manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -59,7 +68,50 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_BENCHMARK=ON
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DBUILD_TEST=ON
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocPRIM_testing
+  dependsOn: rocPRIM
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocPRIM
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocprim'

.azuredevops/components/rocPyDecode.yml

@@ -0,0 +1,171 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - python3-pip
+    - cmake
+    - ninja-build
+    - pkg-config
+    - ffmpeg
+    - libavcodec-dev
+    - libavformat-dev
+    - libavutil-dev
+    - libva-dev
+    - libdrm-dev
+    - pybind11-dev
+    - python3-pybind11
+    - libdlpack-dev
+- name: pipModules
+  type: object
+  default:
+    - -i
+    - https://test.pypi.org/simple
+    - hip-python
+- name: rocmDependencies
+  type: object
+  default:
+    - rocm-cmake
+    - llvm-project
+    - ROCR-Runtime
+    - clr
+    - rocminfo
+    - rocm-core
+    - rocprofiler-register
+    - rocDecode
+
+jobs:
+- job: rocPyDecode
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: 'ROCm symbolic link'
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_INSTALL_PREFIX_PYTHON=$(Build.BinariesDirectory)
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+      publish: false
+  - task: Bash@3
+    displayName: Create wheel file
+    inputs:
+      targetType: inline
+      script: python setup.py bdist_wheel
+      workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.SourcesDirectory)/dist
+      contentsString: '*.whl'
+      targetDir: $(Build.ArtifactStagingDirectory)
+      clean: false
+  - task: PublishPipelineArtifact@1
+    displayName: 'wheel file Publish'
+    retryCountOnTaskFailure: 3
+    inputs:
+      targetPath: $(Build.ArtifactStagingDirectory)
+
+- job: rocPyDecode_testing
+  dependsOn: rocPyDecode
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        cd $(Build.SourcesDirectory)
+        sudo pip install .
+        cmake -DAMDGPU_TARGETS=$(JOB_GPU_TARGET) .
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocPyDecode
+      testDir: $(Build.SourcesDirectory)
+# sudo required for pip install but screws up permissions for next pipeline run
+  - task: Bash@3
+    displayName: Clean up test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf $(Build.SourcesDirectory)/*

.azuredevops/components/rocRAND.yml

@@ -21,6 +21,14 @@ parameters:
     - ROCR-Runtime
     - clr
     - rocminfo
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocRAND
@@ -33,6 +41,12 @@ jobs:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -41,17 +55,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -59,6 +71,49 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_TEST=ON
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocRAND_testing
+  dependsOn: rocRAND
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocRAND
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocRAND'

.azuredevops/components/rocSOLVER.yml

@@ -20,15 +20,30 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
     - clr
-    - rocminfo
-    - rocBLAS
-    - rocPRIM
-    - rocSPARSE
     - hipSPARSE
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - rocPRIM
+    - ROCR-Runtime
+    - rocSPARSE
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - hipSPARSE
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - rocPRIM
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocSPARSE
 
 jobs:
 - job: rocSOLVER
@@ -38,6 +53,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -52,17 +73,15 @@ jobs:
       targetType: inline
       script: git clone --depth 1 --branch v3.9.1 https://github.com/Reference-LAPACK/lapack
       workingDirectory: '$(Build.SourcesDirectory)'
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -82,9 +101,54 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Pipeline.Workspace)/deps-install
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DBUILD_CLIENTS_TESTS=ON
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocSOLVER_testing
+  dependsOn: rocSOLVER
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocSOLVER
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocsolver-test'
+      testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocSPARSE.yml

@@ -29,6 +29,17 @@ parameters:
     - rocminfo
     - rocPRIM
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocSPARSE
@@ -40,6 +51,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -48,17 +65,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  # CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-  # manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -68,7 +83,7 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DBUILD_CLIENTS_SAMPLES=OFF
         -DBUILD_CLIENTS_TESTS=ON
         -DBUILD_CLIENTS_BENCHMARKS=OFF
@@ -77,6 +92,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: rocSPARSE
+      gpuTarget: $(JOB_GPU_TARGET)
       publish: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
@@ -85,3 +101,49 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: testMatrices
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocSPARSE_testing
+  timeoutInMinutes: 90
+  dependsOn: rocSPARSE
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocSPARSE
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocsparse-test'
+      testParameters: '--gtest_filter="*quick*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+

.azuredevops/components/rocThrust.yml

@@ -24,6 +24,16 @@ parameters:
     - rocminfo
     - rocPRIM
     - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocPRIM
+    - ROCR-Runtime
+    - hipRAND
+    - rocprofiler-register
 
 jobs:
 - job: rocThrust
@@ -33,6 +43,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -41,17 +57,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -60,7 +74,49 @@ jobs:
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DBUILD_TEST=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
 
+- job: rocThrust_testing
+  dependsOn: rocThrust
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocThrust
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocthrust'

.azuredevops/components/rocWMMA.yml

@@ -16,10 +16,10 @@ parameters:
     - googletest
     - libfftw3-dev
     - git
-    - libomp-dev
 - name: rocmDependencies
   type: object
   default:
+    - aomp
     - rocm-cmake
     - llvm-project
     - ROCR-Runtime
@@ -28,6 +28,21 @@ parameters:
     - rocBLAS
     - aomp
     - rocm_smi_lib
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocm_smi_lib
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: rocWMMA
@@ -37,6 +52,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -45,17 +66,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  # CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-  # manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -65,7 +84,51 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DROCWMMA_BUILD_TESTS=ON
         -DROCWMMA_BUILD_SAMPLES=OFF
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
 # gfx1030 not supported in documentation
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocWMMA_testing
+  timeoutInMinutes: 90
+  dependsOn: rocWMMA
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocWMMA
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocwmma'

.azuredevops/components/rocm-cmake.yml

@@ -44,5 +44,7 @@ jobs:
       git config --global user.email "you@example.com"
       git config --global user.name "Your Name"
     displayName: "ctest setup"
-#  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm-cmake
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/rocm-core.yml

@@ -29,5 +29,5 @@ jobs:
         -DCPACK_GENERATOR=DEB
         -DCPACK_DEBIAN_PACKAGE_RELEASE="local.9999~99.99"
         -DCPACK_RPM_PACKAGE_RELEASE="local.9999"
-        -DROCM_VERSION="$(last-release)"
+        -DROCM_VERSION="$(next-release)"
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/rocm-examples.yml

@@ -8,20 +8,51 @@ parameters:
 - name: aptPackages
   type: object
   default:
+    - cmake
     - libglfw3-dev
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
     - AMDMIGraphX
     - clr
     - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
     - hipCUB
+    - hipFFT
     - HIPIFY
     - hipRAND
     - hipSOLVER
     - hipSPARSE
     - llvm-project
     - rocBLAS
+    - rocFFT
+    - rocPRIM
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+    - rocSOLVER
+    - rocSPARSE
+    - rocThrust
+- name: rocmTestDependencies
+  type: object
+  default:
+    - AMDMIGraphX
+    - clr
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - hipCUB
+    - hipFFT
+    - HIPIFY
+    - hipRAND
+    - hipSOLVER
+    - hipSPARSE
+    - llvm-project
+    - rocBLAS
+    - rocFFT
+    - rocminfo
     - rocPRIM
     - rocprofiler-register
     - ROCR-Runtime
@@ -35,29 +66,32 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
-      aptPackages: ${{ parameters.aptPackages }}
-  # CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-  # manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -66,6 +100,72 @@ jobs:
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCM_ROOT=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_HIP_ARCHITECTURES=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
         -DCMAKE_EXE_LINKER_FLAGS=-fgpu-rdc
+  - task: Bash@3
+    displayName: Move rocm-examples binaries to rocm/examples
+    inputs:
+      targetType: inline
+      script: |
+        mkdir -p $(Build.BinariesDirectory)/examples
+        mv $(Build.BinariesDirectory)/bin/* $(Build.BinariesDirectory)/examples
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocm_examples_testing
+  dependsOn: rocm_examples
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: TEST_LOG_FILE
+    value: $(Pipeline.Workspace)/rocm-examplesTestLog.log
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Unload and reload AMDGPU
+    inputs:
+      targetType: inline
+      script: |
+        sudo modprobe -r amdgpu
+        sudo modprobe amdgpu
+  - task: Bash@3
+    displayName: Iterate through examples
+    inputs:
+      targetType: inline
+      script: |
+        for file in *; do
+          echo Now running: $file
+          ./$file | tee -a $(TEST_LOG_FILE)
+        done
+      workingDirectory: $(Agent.BuildDirectory)/rocm/examples

.azuredevops/components/rocm_bandwidth_test.yml

@@ -24,7 +24,11 @@ parameters:
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
-    - ROCT-Thunk-Interface
+- name: rocmTestDependencies
+  type: object
+  default:
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocm_bandwidth_test
@@ -48,17 +52,14 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -68,3 +69,40 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/include;$(Agent.BuildDirectory)/rocm/include/hsa
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocm_bandwidth_test_testing
+  dependsOn: rocm_bandwidth_test
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm_bandwidth_test
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/bin/rocm-bandwidth-test'
+      testParameters: ''
+      testPublishResults: false

.azuredevops/components/rocm_smi_lib.yml

@@ -26,3 +26,27 @@ jobs:
         -DBUILD_TESTS=ON
         -DROCM_DEP_ROCMCORE=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocm_smi_lib_testing
+  dependsOn: rocm_smi_lib
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm_smi_lib
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/share/rocm_smi/rsmitst_tests/rsmitst'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocminfo.yml

@@ -10,6 +10,11 @@ parameters:
   default:
     - ROCR-Runtime
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - ROCR-Runtime
+    - rocprofiler-register
 
 jobs:
 - job: rocminfo
@@ -25,17 +30,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      skipLlvmSymlink: true
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -43,3 +46,43 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCRTST_BLD_TYPE=release
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocminfo_testing
+  dependsOn: rocminfo
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocminfo
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/bin/rocminfo'
+      testParameters: ''
+      testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm_agent_enumerator
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/bin/rocm_agent_enumerator'
+      testParameters: ''
+      testPublishResults: false

.azuredevops/components/rocprofiler.yml

@@ -39,7 +39,6 @@ parameters:
     - rocminfo
     - ROCR-Runtime
     - rocprofiler-register
-    - ROCT-Thunk-Interface
     - roctracer
 
 jobs:
@@ -54,6 +53,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -63,40 +68,21 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# Manually download aqlprofile, hard-coded 6.1.0 version
-  - task: Bash@3
-    displayName: 'Download aqlprofile'
-    inputs:
-      targetType: inline
-      script: wget -nv https://repo.radeon.com/rocm/misc/aqlprofile/ubuntu-22.04/hsa-amd-aqlprofile_1.0.0.60200.60200-crdnnh.14213~22.04_amd64.deb
-      workingDirectory: '$(Pipeline.Workspace)'
-  - task: Bash@3
-    displayName: 'Extract aqlprofile'
-    inputs:
-      targetType: inline
-      script: |
-        mkdir hsa-amd-aqlprofile
-        dpkg-deb -R hsa-amd-aqlprofile_1.0.0.60200.60200-crdnnh.14213~22.04_amd64.deb hsa-amd-aqlprofile
-      workingDirectory: '$(Pipeline.Workspace)'
-  - task: Bash@3
-    displayName: 'Move aqlprofile'
-    inputs:
-      targetType: inline
-      script: |
-        mkdir -p $(Agent.BuildDirectory)/rocm
-        cp -R hsa-amd-aqlprofile/opt/rocm-6.2.0-14213/* $(Agent.BuildDirectory)/rocm
-      workingDirectory: '$(Pipeline.Workspace)'
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -105,5 +91,66 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DENABLE_LDCONFIG=OFF
         -DUSE_PROF_API=1
-        -DGPU_TARGETS=gfx942
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j32
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocprofiler_testing
+  dependsOn: rocprofiler
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofilerV1
+      testExecutable: LD_LIBRARY_PATH="$(Agent.BuildDirectory)/rocm/lib/rocprofiler:$(Agent.BuildDirectory)/rocm/share/rocprofiler/tests-v1/test" ./run.sh
+      testDir: $(Agent.BuildDirectory)/rocm/share/rocprofiler/tests-v1
+      testParameters: ''
+      testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofilerV2
+      testExecutable: LD_LIBRARY_PATH="$(Agent.BuildDirectory)/rocm/lib/rocprofiler:$(Agent.BuildDirectory)/rocm/share/rocprofiler/tests" share/rocprofiler/tests/runUnitTests
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: $(Agent.BuildDirectory)/rocm

.azuredevops/components/rocr_debug_agent.yml

@@ -12,6 +12,8 @@ parameters:
     - ninja-build
     - libelf-dev
     - libdw-dev
+    - libstdc++-12-dev
+    - python-is-python3
 - name: rocmDependencies
   type: object
   default:
@@ -22,6 +24,15 @@ parameters:
     - rocminfo
     - ROCR-Runtime
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - ROCdbgapi
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocr_debug_agent
@@ -40,17 +51,14 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -61,3 +69,52 @@ jobs:
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocr_debug_agent_testing
+  dependsOn: rocr_debug_agent
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      componentName: rocr_debug_agent-tests
+      extraBuildFlags: >-
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake;$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+      cmakeBuildDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'
+      cmakeSourceDir: '.'
+      installEnabled: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocr_debug_agent
+      testDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'

.azuredevops/components/roctracer.yml

@@ -25,6 +25,13 @@ parameters:
     - rocprofiler-register
     - clr
     - rocminfo
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: roctracer
@@ -52,17 +59,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -76,3 +81,47 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: roctracer_testing
+  dependsOn: roctracer
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: roctracer
+      testExecutable: $(Agent.BuildDirectory)/rocm/share/roctracer/run_tests.sh
+      testParameters: ''
+      testDir: $(Agent.BuildDirectory)
+      testPublishResults: false

.azuredevops/components/rpp.yml

@@ -9,13 +9,12 @@ parameters:
   type: object
   default:
     - cmake
-    - libomp-dev # needed to pass flag step
     - ninja-build
     - clang
 - name: rocmDependencies
   type: object
   default:
-    - aomp # needed to pass build step
+    - aomp
     - clr
     - half
     - llvm-project
@@ -31,6 +30,12 @@ jobs:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -39,17 +44,15 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# CI case: download latest default branch build
-  - ${{ if eq(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
-# manual build case: triggered by ROCm/ROCm repo
-  - ${{ if ne(parameters.checkoutRef, '') }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-      parameters:
-        dependencyList: ${{ parameters.rocmDependencies }}
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -60,6 +63,8 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DHALF_INCLUDE_DIRS=$(Agent.BuildDirectory)/rocm/include
         -DCMAKE_BUILD_TYPE=Release
-        -DAMDGPU_TARGETS=gfx942
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/nightly/pytorch.yml

@@ -54,6 +54,14 @@ parameters:
     - libdrm-amdgpu1
     - ccache
     - zip
+    - libjpeg-turbo-official
+    - libjpeg-dev
+    - libwebp-dev
+    - libfreetype-dev
+    - gnutls-bin
+    - ffmpeg
+    - libopenblas-dev
+    - liblapack-dev
 - name: pipModules
   type: object
   default:
@@ -76,6 +84,9 @@ parameters:
     - ninja
     - packaging
     - optree>=0.12.0
+  # list for vision
+    - auditwheel
+    - future
 # list from https://github.com/pytorch/builder/blob/main/manywheel/build_rocm.sh
 - name: rocmDependencies
   type: object
@@ -105,6 +116,7 @@ parameters:
     # below are additional dependencies not called out by build script, but throw errors during cmake
     - hipCUB
     - rocThrust
+    - hipBLAS-common
 
 trigger: none
 pr: none
@@ -119,16 +131,20 @@ schedules:
 jobs:
 - job: pytorch
   timeoutInMinutes: 120
+  strategy:
+    matrix:
+      amd-staging-gfx942:
+        ROCM_BRANCH: amd-staging
+        JOB_GPU_TARGET: gfx942
+      amd-staging-gfx90a:
+        ROCM_BRANCH: amd-staging
+        JOB_GPU_TARGET: gfx90a
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
 # various flags/parameters expected by bash scripts in pytorch builder repo
   - name: ROCM_VERSION
     value: 6.3.0
-  - name: PYTORCH_ROCM_ARCH
-    value: gfx942
-  - name: GPU_TARGET
-    value: gfx942
   - name: ROCM_PATH
     value: /opt/rocm
   - name: DESIRED_CUDA
@@ -141,8 +157,6 @@ jobs:
     value: 3.10
   - name: PYTORCH_ROOT
     value: $(Build.SourcesDirectory)/pytorch
-  - name: CMAKE_ARGS
-    value: -GNinja
   - name: DESIRED_DEVTOOLSET
     value: cxx11-abi
   pool: ${{ variables.ULTRA_BUILD_POOL }}
@@ -152,6 +166,16 @@ jobs:
 # copy environment setup from https://github.com/pytorch/builder/blob/main/manywheel/Dockerfile
 # but instead of centos, use ubuntu environment
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
+  - task: Bash@3
+    displayName: 'Register libjpeg-turbo packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
+        sudo apt update
+        apt-cache show libjpeg-turbo-official | grep Version
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
@@ -169,13 +193,12 @@ jobs:
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
       dependencySource: staging
+      gpuTarget: $(JOB_GPU_TARGET)
   - task: Bash@3
-    displayName: ROCm symbolic links
+    displayName: ROCm symbolic link
     inputs:
       targetType: inline
-      script: |
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-        sudo ln -s $(Agent.BuildDirectory)/rocm/llvm $(Agent.BuildDirectory)/rocm/lib/llvm
+      script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
   - checkout: self
   - task: Bash@3
     displayName: git clone pytorch builder
@@ -189,12 +212,6 @@ jobs:
       targetType: inline
       script: git clone https://github.com/pytorch/pytorch.git --depth=1 --recurse-submodules
       workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: Patch out forced GPU testing block in pytorch build script
-    inputs:
-      targetType: inline
-      script: git apply $(Build.SourcesDirectory)/.azuredevops/patches/pytorch/0001-ROCm-CI-patches.patch
-      workingDirectory: $(Build.SourcesDirectory)/builder
   - task: Bash@3
     displayName: Install patchelf
     inputs:
@@ -221,15 +238,14 @@ jobs:
     inputs:
       targetType: inline
       script: |
-        sudo PYTORCH_ROCM_ARCH=$(PYTORCH_ROCM_ARCH) MKLROOT=$(MKLROOT) bash pytorch/.ci/docker/common/install_rocm_magma.sh
+        sudo PYTORCH_ROCM_ARCH=$(JOB_GPU_TARGET) MKLROOT=$(MKLROOT) bash pytorch/.ci/docker/common/install_rocm_magma.sh
       workingDirectory: $(Build.SourcesDirectory)
   - task: Bash@3
     displayName: Install AOTriton Shared Library
     inputs:
       targetType: inline
-      script: |
-        sudo bash ./install_aotriton.sh /opt/rocm
-      workingDirectory: $(Build.SourcesDirectory)/pytorch/.ci/docker/common
+      script: sudo bash ./common/install_aotriton.sh /opt/rocm
+      workingDirectory: $(Build.SourcesDirectory)/pytorch/.ci/docker
   - task: Bash@3
     displayName: Run ROCm Build Script
     inputs:
@@ -237,16 +253,69 @@ jobs:
       script: >-
         sudo
         DESIRED_CUDA=$(DESIRED_CUDA)
-        PYTORCH_ROCM_ARCH=$(PYTORCH_ROCM_ARCH)
+        PYTORCH_ROCM_ARCH=$(JOB_GPU_TARGET)
+        GPU_TARGET=$(JOB_GPU_TARGET)
         DESIRED_PYTHON=$(DESIRED_PYTHON)
         PYTORCH_ROOT=$(PYTORCH_ROOT)
-        CMAKE_ARGS=$(CMAKE_ARGS)
+        CMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         AOTRITON_INSTALLED_PREFIX=$(AOTRITON_INSTALLED_PREFIX)
         DESIRED_DEVTOOLSET=$(DESIRED_DEVTOOLSET)
+        TORCH_PACKAGE_NAME=torch.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
+        PYTORCH_BUILD_VERSION=$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)
+        PYTORCH_BUILD_NUMBER=$(date -u +%Y%m%d)
+        SKIP_ALL_TESTS=1
         bash ./manywheel/build_rocm.sh
       workingDirectory: $(Build.SourcesDirectory)/builder
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: /remote/wheelhouserocm$(ROCM_VERSION)
+      contentsString: '*.whl'
+# common helper source for pytorch vision and audio
+  - task: Bash@3
+    displayName: git clone pytorch test-infra
+    inputs:
+      targetType: inline
+      script: git clone https://github.com/pytorch/test-infra.git --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: install package helper
+    inputs:
+      targetType: inline
+      script: python3 -m pip install test-infra/tools/pkg-helpers
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: pytorch pkg helpers
+    inputs:
+      targetType: inline
+      script: CU_VERSION=${CU_VERSION} CHANNEL=${CHANNEL} python -m pytorch_pkg_helpers
+# get torch vision source and build
+  - task: Bash@3
+    displayName: git clone pytorch vision
+    inputs:
+      targetType: inline
+      script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Build vision
+    inputs:
+      targetType: inline
+      script: >-
+        BUILD_VERSION=$(ROCM_BRANCH).$(JOB_GPU_TARGET)-$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)$(date -u +%Y%m%d)
+        python3 setup.py bdist_wheel
+      workingDirectory: $(Build.SourcesDirectory)/vision
+  - task: Bash@3
+    displayName: Relocate vision
+    inputs:
+      targetType: inline
+      script: python3 packaging/wheel/relocate.py
+      workingDirectory: $(Build.SourcesDirectory)/vision
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.SourcesDirectory)/vision/dist
+      contentsString: '*.whl'
+      clean: false
   - task: PublishPipelineArtifact@1
-    displayName: 'ROCm pytorch wheel file Publish'
+    displayName: 'wheel file Publish'
     retryCountOnTaskFailure: 3
     inputs:
-      targetPath: /remote/wheelhouserocm$(ROCM_VERSION)
+      targetPath: $(Build.BinariesDirectory)

.azuredevops/nightly/rocm-nightly.yml

@@ -1,5 +1,5 @@
 parameters:
-# currently excludes clr and rocm-examples
+# currently excludes clr
 - name: rocmDependencies
   type: object
   default:
@@ -10,7 +10,9 @@ parameters:
     - composable_kernel
     - half
     - HIP
+    - hip-tests
     - hipBLAS
+    - hipBLAS-common
     - hipBLASLt
     - hipCUB
     - hipFFT
@@ -24,6 +26,7 @@ parameters:
     - llvm-project
     - MIOpen
     - MIVisionX
+    - omniperf
     - rccl
     - rdc
     - rocAL
@@ -35,6 +38,7 @@ parameters:
     - ROCgdb
     - rocm-cmake
     - rocm-core
+    - rocm-examples
     - rocminfo
     - rocMLIR
     - ROCmValidationSuite
@@ -43,12 +47,12 @@ parameters:
     - rocPRIM
     - rocprofiler-register
     - rocprofiler
+    - rocPyDecode
     - ROCR-Runtime
     - rocRAND
     - rocr_debug_agent
     - rocSOLVER
     - rocSPARSE
-    - ROCT-Thunk-Interface
     - rocThrust
     - roctracer
     - rocWMMA
@@ -72,6 +76,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - task: DeleteFiles@1
     displayName: 'Cleanup checkout space'
@@ -84,28 +94,26 @@ jobs:
       SourceFolder: '$(Build.ArtifactStagingDirectory)'
       Contents: '/**/*'
       RemoveDotFiles: true
-  - script: sudo chmod 777 /mnt
-    displayName: 'Set permissions for /mnt'
   - script: df -h
     displayName: System disk space before ROCm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
       dependencySource: staging
-      extractToMnt: true
       skipLibraryLinking: true
+      gpuTarget: $(JOB_GPU_TARGET)
   - script: df -h
     displayName: System disk space after ROCm
-  - script: du -sh /mnt/rocm
+  - script: du -sh $(Agent.BuildDirectory)/rocm
     displayName: Uncompressed ROCm size
   - task: ArchiveFiles@2
     displayName: Compress rocm-nightly
     inputs:
-      rootFolderOrFile: /mnt/rocm
+      rootFolderOrFile: $(Agent.BuildDirectory)/rocm
       includeRootFolder: false
       archiveType: tar
       tarCompression: gz
-      archiveFile: $(Build.ArtifactStagingDirectory)/$(Build.DefinitionName)_$(Build.BuildNumber)_ubuntu2204.tar.gz
+      archiveFile: $(Build.ArtifactStagingDirectory)/$(Build.DefinitionName)_$(Build.BuildNumber)_ubuntu2204_$(JOB_GPU_TARGET).tar.gz
   - script: du -sh $(Build.ArtifactStagingDirectory)
     displayName: Compressed ROCm size
   - task: PublishPipelineArtifact@1

.azuredevops/patches/pytorch/0001-ROCm-CI-patches.patch

@@ -1,40 +0,0 @@
-From b2d3c88f7a8b179e814e72f76f27e25c82659200 Mon Sep 17 00:00:00 2001
-From: Joseph Macaranas <Joseph.Macaranas@amd.com>
-Date: Tue, 30 Jul 2024 05:43:06 -0400
-Subject: [PATCH] ROCm CI patches
-
----
- manywheel/build_common.sh | 9 ---------
- 1 file changed, 9 deletions(-)
-
-diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh
-index 08ca924..52c468f 100644
---- a/manywheel/build_common.sh
-+++ b/manywheel/build_common.sh
-@@ -475,11 +475,9 @@ if [[ -z "$BUILD_PYTHONLESS" ]]; then
-   fi
- 
-   pip uninstall -y "$TORCH_PACKAGE_NAME"
--  
-   if [[ "$USE_SPLIT_BUILD" == "true" ]]; then
-     pip install "$TORCH_NO_PYTHON_PACKAGE_NAME" --no-index -f /$WHEELHOUSE_DIR --no-dependencies -v
-   fi
--  
-   pip install "$TORCH_PACKAGE_NAME" --no-index -f /$WHEELHOUSE_DIR --no-dependencies -v
- 
-   # Print info on the libraries installed in this wheel
-@@ -491,11 +489,4 @@ if [[ -z "$BUILD_PYTHONLESS" ]]; then
-       ldd "$installed_lib" || true
-   done
- 
--  # Run the tests
--  echo "$(date) :: Running tests"
--  pushd "$PYTORCH_ROOT"
--  LD_LIBRARY_PATH=/usr/local/nvidia/lib64 \
--          "${SOURCE_DIR}/../run_tests.sh" manywheel "${py_majmin}" "$DESIRED_CUDA"
--  popd
--  echo "$(date) :: Finished tests"
- fi
--- 
-2.44.0.windows.1
-

.azuredevops/tag-builds/aomp.yml

@@ -9,16 +9,26 @@ parameters:
 
 resources:
   repositories:
-  - repository: release_repo
+  - repository: aomp_repo
     type: github
     endpoint: ROCm
     name: ROCm/aomp
     ref: ${{ parameters.checkoutRef }}
+  - repository: aomp-extras_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp-extras
+    ref: ${{ parameters.checkoutRef }}
+  - repository: flang_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/flang
+    ref: ${{ parameters.checkoutRef }}
   - repository: llvm-project_repo
     type: github
     endpoint: ROCm
     name: ROCm/llvm-project
-    ref: ${{ parameters.checkoutRef }}
+    ref: amd-staging
 
 trigger: none
 pr: none
@@ -26,5 +36,5 @@ pr: none
 jobs:
   - template: ${{ variables.CI_COMPONENT_PATH }}/aomp.yml
     parameters:
-      checkoutRepo: release_repo
+      checkoutRepo: aomp_repo
       checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/hip-tests.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/hip-tests
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/hip-tests.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/hipBLAS-common.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: develop
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/hipBLAS-common
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/hipBLAS-common.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/omniperf.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/omniperf
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/omniperf.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocPyDecode.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocPyDecode
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocPyDecode.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/templates/steps/artifact-download.yml

@@ -25,12 +25,13 @@ parameters:
     amdsmi: develop
     aomp-extras: aomp-dev
     aomp: aomp-dev
-    clr: develop
+    clr: amd-staging
     composable_kernel: develop
     half: rocm
-    HIP: develop
+    HIP: amd-staging
     hipBLAS: develop
     hipBLASLt: develop
+    hipBLAS-common: develop
     hipCUB: develop
     hipFFT: develop
     hipfort: develop
@@ -62,17 +63,28 @@ parameters:
     rocm_smi_lib: develop
     rocPRIM: develop
     rocprofiler-register: amd-mainline
-    rocprofiler: amd-master
-    ROCR-Runtime: master
+    rocprofiler: amd-staging
+    ROCR-Runtime: amd-staging
     rocRAND: develop
     rocr_debug_agent: amd-staging
     rocSOLVER: develop
     rocSPARSE: develop
-    ROCT-Thunk-Interface: master
     rocThrust: develop
-    roctracer: amd-master
+    roctracer: amd-staging
     rocWMMA: develop
-    rpp: master
+    rpp: develop
+- name: allowPartiallySucceededBuilds
+  type: object
+  default:
+    - amdsmi
+    - aomp
+    - HIPIFY
+    - rccl
+    - rdc
+    - rocm-cmake
+    - rocm_smi_lib
+    - rocFFT
+    - MIVisionX
 # BELOW REQUIRED IF useDefaultBranch false
 - name: branchName
   type: string
@@ -88,13 +100,13 @@ steps:
     specificBuildWithTriggering: true
     itemPattern: '**/*${{ parameters.fileFilter }}*'
     ${{ if eq(parameters.latestFromBranch, true) }}:
-      ${{ if notIn(parameters.componentName, 'aomp', 'clr', 'rocMLIR') }}: # remove this once these pipelines are functional + up-to-date
+      ${{ if notIn(parameters.componentName, 'aomp') }}: # remove this once these pipelines are functional + up-to-date
         buildVersionToDownload: latestFromBranch # default is 'latest'
     ${{ if eq(parameters.useDefaultBranch, true) }}:
       branchName: refs/heads/${{ parameters.defaultBranchList[parameters.componentName] }}
     ${{ else }}:
       branchName: ${{ parameters.branchName }}
-    ${{ if in(parameters.componentName, 'rocm-cmake') }}:
+    ${{ if containsValue(parameters.allowPartiallySucceededBuilds, parameters.componentName) }}:
       allowPartiallySucceededBuilds: true
     targetPath: '$(Pipeline.Workspace)/d'
 - task: ExtractFiles@1

.azuredevops/templates/steps/build-autotools.yml

@@ -11,6 +11,9 @@ parameters:
 - name: installDir
   type: string
   default: '$(Build.BinariesDirectory)'
+- name: makeCallPrefix
+  type: string
+  default: ''
 
 steps:
 - task: Bash@3
@@ -23,7 +26,7 @@ steps:
   displayName: '${{ parameters.componentName }} make'
   inputs:
     targetType: inline
-    script: make -j$(nproc)
+    script: ${{ parameters.makeCallPrefix }} make -j$(nproc)
     workingDirectory: ${{ parameters.buildDir }}
 - task: Bash@3
   displayName: '${{ parameters.componentName }} make install'

.azuredevops/templates/steps/build-cmake.yml

@@ -14,6 +14,9 @@ parameters:
 - name: cmakeSourceDir
   type: string
   default: '..'
+- name: customBuildTarget
+  type: string
+  default: ''
 - name: cmakeTarget
   type: string
   default: 'install'
@@ -49,7 +52,10 @@ steps:
   displayName: '${{parameters.componentName }} Build'
   inputs:
     workingDirectory: ${{ parameters.cmakeBuildDir }}
-    cmakeArgs: '--build ${{ parameters.cmakeTargetDir }} ${{ parameters.multithreadFlag }}'
+    ${{ if eq(parameters.customBuildTarget, '') }}:
+      cmakeArgs: '--build ${{ parameters.cmakeTargetDir }} ${{ parameters.multithreadFlag }}'
+    ${{ else }}:
+      cmakeArgs: '--build ${{ parameters.cmakeTargetDir }} --target ${{ parameters.customBuildTarget }} ${{ parameters.multithreadFlag }}'
     retryCountOnTaskFailure: 10
 - script: df -h
   displayName: Disk space after build

.azuredevops/templates/steps/ccache.yml

@@ -1,11 +1,14 @@
 # to use ccache, run this template step before build-cmake.yml
 # and add this cmake flag to extraBuildFlags
 # -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
 steps:
 - task: Cache@2
   displayName: Ccache caching
   inputs:
     key: 'ccache | "$(Agent.OS)" | $(Build.SourceVersion)'
-    path: $(CCACHE_DIR)
+    path: ${{ variables.CCACHE_DIR }}
     restoreKeys: |
       ccache | "$(Agent.OS)"

.azuredevops/templates/steps/dependencies-aocl.yml

@@ -0,0 +1,27 @@
+parameters:
+- name: repositoryUrl
+  type: string
+  default: https://download.amd.com/developer/eula/aocl/aocl-4-2
+- name: packageName
+  type: string
+  default: aocl-linux-gcc-4.2.0_1_amd64.deb
+
+steps:
+- task: Bash@3
+  displayName: Download AOCL
+  inputs:
+    targetType: inline
+    workingDirectory: $(Pipeline.Workspace)
+    script: wget -nv ${{ parameters.repositoryUrl }}/${{ parameters.packageName }}
+- task: Bash@3
+  displayName: Install AOCL
+  inputs:
+    targetType: inline
+    workingDirectory: $(Pipeline.Workspace)
+    script: sudo apt install -y ./${{ parameters.packageName }}
+- task: Bash@3
+  displayName: Clean up AOCL
+  inputs:
+    targetType: inline
+    workingDirectory: $(Pipeline.Workspace)
+    script: rm -f ${{ parameters.packageName }}

.azuredevops/templates/steps/dependencies-aqlprofile.yml

@@ -0,0 +1,47 @@
+parameters:
+- name: dependencySource
+  type: string
+  default: staging
+  values:
+    - staging
+    - tag-builds
+- name: repositoryUrl
+  type: object
+  default: 
+    staging: https://repo.radeon.com/rocm/apt/6.2/pool/main/h/hsa-amd-aqlprofile
+    tag-builds: https://repo.radeon.com/rocm/apt/6.2/pool/main/h/hsa-amd-aqlprofile
+- name: packageName
+  type: object
+  default:
+    staging: hsa-amd-aqlprofile_1.0.0.60200.60200-66~22.04_amd64.deb
+    tag-builds: hsa-amd-aqlprofile_1.0.0.60200.60200-66~22.04_amd64.deb
+
+steps:
+- task: Bash@3
+  displayName: 'Download aqlprofile'
+  inputs:
+    targetType: inline
+    script: wget -nv ${{ parameters.repositoryUrl[parameters.dependencySource] }}/${{ parameters.packageName[parameters.dependencySource] }}
+    workingDirectory: '$(Pipeline.Workspace)'
+- task: Bash@3
+  displayName: 'Extract aqlprofile'
+  inputs:
+    targetType: inline
+    script: |
+      mkdir hsa-amd-aqlprofile
+      dpkg-deb -R ${{ parameters.packageName[parameters.dependencySource] }} hsa-amd-aqlprofile
+    workingDirectory: '$(Pipeline.Workspace)'
+- task: Bash@3
+  displayName: 'Copy aqlprofile files'
+  inputs:
+    targetType: inline
+    script: |
+      mkdir -p $(Agent.BuildDirectory)/rocm
+      cp -R hsa-amd-aqlprofile/opt/rocm-*/* $(Agent.BuildDirectory)/rocm
+    workingDirectory: '$(Pipeline.Workspace)'
+- task: Bash@3
+  displayName: 'Clean up aqlprofile'
+  inputs:
+    targetType: inline
+    script: rm -rf hsa-amd-aqlprofile ${{ parameters.packageName[parameters.dependencySource] }}
+    workingDirectory: '$(Pipeline.Workspace)'

.azuredevops/templates/steps/dependencies-cmake-latest.yml

@@ -5,6 +5,6 @@ steps:
   inputs:
     targetType: inline
     script: |
-      sudo apt purge cmake
+      sudo apt purge cmake -y
       sudo snap install cmake --classic
       hash -r

.azuredevops/templates/steps/dependencies-other.yml

@@ -12,34 +12,26 @@ steps:
   displayName: 'sudo apt-get update'
   inputs:
     targetType: inline
-    script: sudo apt-get --yes update
-  env:
-    DEBIAN_FRONTEND: noninteractive
+    script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes update
 - task: Bash@3
   displayName: 'sudo apt-get upgrade'
   inputs:
     targetType: inline
-    script: sudo apt-get --yes upgrade
-  env:
-    DEBIAN_FRONTEND: noninteractive
+    script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes upgrade
 - task: Bash@3
   displayName: 'sudo apt-get fix'
   inputs:
     targetType: inline
-    script: sudo apt --yes --fix-broken install
-  env:
-    DEBIAN_FRONTEND: noninteractive
+    script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes --fix-broken install
 - ${{ if gt(length(parameters.aptPackages), 0) }}:
   - task: Bash@3
     displayName: 'sudo apt-get install ...'
     inputs:
       targetType: inline
-      script: sudo apt-get --yes install ${{ join(' ', parameters.aptPackages) }}
-    env:
-      DEBIAN_FRONTEND: noninteractive
+      script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install ${{ join(' ', parameters.aptPackages) }}
 - ${{ if gt(length(parameters.pipModules), 0) }}:
   - task: Bash@3
     displayName: 'pip install  ...'
     inputs:
       targetType: inline
-      script: pip install ${{ join(' ', parameters.pipModules) }}
+      script: pip install -v ${{ join(' ', parameters.pipModules) }}

.azuredevops/templates/steps/dependencies-rocm.yml

@@ -29,159 +29,206 @@ parameters:
 - name: stagingPipelineIdentifiers
   type: object
   default:
-    AMDMIGraphX: $(amdmigraphx-pipeline-id)
-    amdsmi: $(amdsmi-pipeline-id)
-    aomp-extras: $(aomp-extras-pipeline-id)
-    aomp: $(aomp-pipeline-id)
-    clr: $(clr-pipeline-id)
-    composable_kernel: $(composable-kernel-pipeline-id)
-    half: $(half-pipeline-id)
-    HIP: $(hip-pipeline-id)
-    hipBLAS: $(hipblas-pipeline-id)
-    hipBLASLt: $(hipblaslt-pipeline-id)
-    hipCUB: $(hipcub-pipeline-id)
-    hipFFT: $(hipfft-pipeline-id)
-    hipfort: $(hipfort-pipeline-id)
-    HIPIFY: $(hipify-pipeline-id)
-    hipRAND: $(hiprand-pipeline-id)
-    hipSOLVER: $(hipsolver-pipeline-id)
-    hipSPARSE: $(hipsparse-pipeline-id)
-    hipSPARSELt: $(hipsparselt-pipeline-id)
-    hipTensor: $(hiptensor-pipeline-id)
-    llvm-project: $(llvm-project-pipeline-id)
-    MIOpen: $(miopen-pipeline-id)
-    MIVisionX: $(mivisionx-pipeline-id)
-    rccl: $(rccl-pipeline-id)
-    rdc: $(rdc-pipeline-id)
-    rocAL: $(rocal-pipeline-id)
-    rocALUTION: $(rocalution-pipeline-id)
-    rocBLAS: $(rocblas-pipeline-id)
-    ROCdbgapi : $(rocdbgapi-pipeline-id)
-    rocDecode: $(rocdecode-pipeline-id)
-    rocFFT: $(rocfft-pipeline-id)
-    ROCgdb: $(rocgdb-pipeline-id)
-    rocm-cmake: $(rocm-cmake-pipeline-id)
-    rocm-core: $(rocm-core-pipeline-id)
-    rocm-examples: $(rocm-examples-pipeline-id)
-    rocminfo: $(rocminfo-pipeline-id)
-    rocMLIR: $(rocmlir-pipeline-id)
-    ROCmValidationSuite: $(rocmvalidationsuite-pipeline-id)
-    rocm_bandwidth_test: $(rocm-bandwidth-test-pipeline-id)
-    rocm_smi_lib: $(rocm-smi-lib-pipeline-id)
-    rocPRIM: $(rocprim-pipeline-id)
-    rocprofiler-register: $(rocprofiler-register-pipeline-id)
-    rocprofiler: $(rocprofiler-pipeline-id)
-    ROCR-Runtime: $(rocr-runtime-pipeline-id)
-    rocRAND: $(rocrand-pipeline-id)
-    rocr_debug_agent: $(rocr-debug-agent-pipeline-id)
-    rocSOLVER: $(rocsolver-pipeline-id)
-    rocSPARSE: $(rocsparse-pipeline-id)
-    ROCT-Thunk-Interface: $(roct-thunk-interface-pipeline-id)
-    rocThrust: $(rocthrust-pipeline-id)
-    roctracer: $(roctracer-pipeline-id)
-    rocWMMA: $(rocwmma-pipeline-id)
-    rpp: $(rpp-pipeline-id)
+    AMDMIGraphX: $(AMDMIGRAPHX_PIPELINE_ID)
+    amdsmi: $(AMDSMI_PIPELINE_ID)
+    aomp-extras: $(AOMP_EXTRAS_PIPELINE_ID)
+    aomp: $(AOMP_PIPELINE_ID)
+    clr: $(CLR_PIPELINE_ID)
+    composable_kernel: $(COMPOSABLE_KERNEL_PIPELINE_ID)
+    half: $(HALF_PIPELINE_ID)
+    HIP: $(HIP_PIPELINE_ID)
+    hipBLAS: $(HIPBLAS_PIPELINE_ID)
+    hipBLAS-common: $(HIPBLAS_COMMON_PIPELINE_ID)
+    hipBLASLt: $(HIPBLASLT_PIPELINE_ID)
+    hipCUB: $(HIPCUB_PIPELINE_ID)
+    hipFFT: $(HIPFFT_PIPELINE_ID)
+    hipfort: $(HIPFORT_PIPELINE_ID)
+    HIPIFY: $(HIPIFY_PIPELINE_ID)
+    hipRAND: $(HIPRAND_PIPELINE_ID)
+    hipSOLVER: $(HIPSOLVER_PIPELINE_ID)
+    hipSPARSE: $(HIPSPARSE_PIPELINE_ID)
+    hipSPARSELt: $(HIPSPARSELT_PIPELINE_ID)
+    hipTensor: $(HIPTENSOR_PIPELINE_ID)
+    llvm-project: $(LLVM_PROJECT_PIPELINE_ID)
+    MIOpen: $(MIOpen_PIPELINE_ID)
+    MIVisionX: $(MIVISIONX_PIPELINE_ID)
+    omniperf: $(OMNIPERF_PIPELINE_ID)
+    rccl: $(RCCL_PIPELINE_ID)
+    rdc: $(RDC_PIPELINE_ID)
+    rocAL: $(ROCAL_PIPELINE_ID)
+    rocALUTION: $(ROCALUTION_PIPELINE_ID)
+    rocBLAS: $(ROCBLAS_PIPELINE_ID)
+    ROCdbgapi: $(ROCDBGAPI_PIPELINE_ID)
+    rocDecode: $(ROCDECODE_PIPELINE_ID)
+    rocFFT: $(ROCFFT_PIPELINE_ID)
+    ROCgdb: $(ROCGDB_PIPELINE_ID)
+    rocm-cmake: $(ROCM_CMAKE_PIPELINE_ID)
+    rocm-core: $(ROCM_CORE_PIPELINE_ID)
+    rocm-examples: $(ROCM_EXAMPLES_PIPELINE_ID)
+    rocminfo: $(ROCMINFO_PIPELINE_ID)
+    rocMLIR: $(ROCMLIR_PIPELINE_ID)
+    ROCmValidationSuite: $(ROCMVALIDATIONSUITE_PIPELINE_ID)
+    rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_PIPELINE_ID)
+    rocm_smi_lib: $(ROCM_SMI_LIB_PIPELINE_ID)
+    rocPRIM: $(ROCPRIM_PIPELINE_ID)
+    rocprofiler-register: $(ROCPROFILER_REGISTER_PIPELINE_ID)
+    rocprofiler: $(ROCPROFILER_PIPELINE_ID)
+    ROCR-Runtime: $(ROCR_RUNTIME_PIPELINE_ID)
+    rocRAND: $(ROCRAND_PIPELINE_ID)
+    rocr_debug_agent: $(ROCR_DEBUG_AGENT_PIPELINE_ID)
+    rocSOLVER: $(ROCSOLVER_PIPELINE_ID)
+    rocSPARSE: $(ROCSPARSE_PIPELINE_ID)
+    ROCT-Thunk-Interface: $(ROCT_THUNK_INTERFACE_PIPELINE_ID)
+    rocThrust: $(ROCTHRUST_PIPELINE_ID)
+    roctracer: $(ROCTRACER_PIPELINE_ID)
+    rocWMMA: $(ROCWMMA_PIPELINE_ID)
+    rpp: $(RPP_PIPELINE_ID)
 - name: taggedPipelineIdentifiers
   type: object
   default:
-    AMDMIGraphX: $(amdmigraphx-tagged-pipeline-id)
-    amdsmi: $(amdsmi-tagged-pipeline-id)
-    aomp-extras: $(aomp-extras-tagged-pipeline-id)
-    aomp: $(aomp-tagged-pipeline-id)
-    clr: $(clr-tagged-pipeline-id)
-    composable_kernel: $(composable-kernel-tagged-pipeline-id)
-    half: $(half-tagged-pipeline-id)
-    HIP: $(hip-tagged-pipeline-id)
-    hipBLAS: $(hipblas-tagged-pipeline-id)
-    hipBLASLt: $(hipblaslt-tagged-pipeline-id)
-    hipCUB: $(hipcub-tagged-pipeline-id)
-    hipFFT: $(hipfft-tagged-pipeline-id)
-    hipfort: $(hipfort-tagged-pipeline-id)
-    HIPIFY: $(hipify-tagged-pipeline-id)
-    hipRAND: $(hiprand-tagged-pipeline-id)
-    hipSOLVER: $(hipsolver-tagged-pipeline-id)
-    hipSPARSE: $(hipsparse-tagged-pipeline-id)
-    hipSPARSELt: $(hipsparselt-tagged-pipeline-id)
-    hipTensor: $(hiptensor-tagged-pipeline-id)
-    llvm-project: $(llvm-project-tagged-pipeline-id)
-    MIOpen: $(miopen-tagged-pipeline-id)
-    MIVisionX: $(mivisionx-tagged-pipeline-id)
-    rccl: $(rccl-tagged-pipeline-id)
-    rdc: $(rdc-tagged-pipeline-id)
-    rocAL: $(rocal-tagged-pipeline-id)
-    rocALUTION: $(rocalution-tagged-pipeline-id)
-    rocBLAS: $(rocblas-tagged-pipeline-id)
-    ROCdbgapi : $(rocdbgapi-tagged-pipeline-id)
-    rocDecode: $(rocdecode-tagged-pipeline-id)
-    rocFFT: $(rocfft-tagged-pipeline-id)
-    ROCgdb: $(rocgdb-tagged-pipeline-id)
-    rocm-cmake: $(rocm-cmake-tagged-pipeline-id)
-    rocm-core: $(rocm-core-tagged-pipeline-id)
-    rocm-examples: $(rocm-examples-tagged-pipeline-id)
-    rocminfo: $(rocminfo-tagged-pipeline-id)
-    rocMLIR: $(rocmlir-tagged-pipeline-id)
-    ROCmValidationSuite: $(rocmvalidationsuite-tagged-pipeline-id)
-    rocm_bandwidth_test: $(rocm-bandwidth-test-tagged-pipeline-id)
-    rocm_smi_lib: $(rocm-smi-lib-tagged-pipeline-id)
-    rocPRIM: $(rocprim-tagged-pipeline-id)
-    rocprofiler-register: $(rocprofiler-register-tagged-pipeline-id)
-    rocprofiler: $(rocprofiler-tagged-pipeline-id)
-    ROCR-Runtime: $(rocr-runtime-tagged-pipeline-id)
-    rocRAND: $(rocrand-tagged-pipeline-id)
-    rocr_debug_agent: $(rocr-debug-agent-tagged-pipeline-id)
-    rocSOLVER: $(rocsolver-tagged-pipeline-id)
-    rocSPARSE: $(rocsparse-tagged-pipeline-id)
-    ROCT-Thunk-Interface: $(roct-thunk-interface-tagged-pipeline-id)
-    rocThrust: $(rocthrust-tagged-pipeline-id)
-    roctracer: $(roctracer-tagged-pipeline-id)
-    rocWMMA: $(rocwmma-tagged-pipeline-id)
-    rpp: $(rpp-tagged-pipeline-id)
+    AMDMIGraphX: $(AMDMIGRAPHX_TAGGED_PIPELINE_ID)
+    amdsmi: $(AMDSMI_TAGGED_PIPELINE_ID)
+    aomp-extras: $(AOMP_EXTRAS_TAGGED_PIPELINE_ID)
+    aomp: $(AOMP_TAGGED_PIPELINE_ID)
+    clr: $(CLR_TAGGED_PIPELINE_ID)
+    composable_kernel: $(COMPOSABLE_KERNEL_TAGGED_PIPELINE_ID)
+    half: $(HALF_TAGGED_PIPELINE_ID)
+    HIP: $(HIP_TAGGED_PIPELINE_ID)
+    hipBLAS: $(HIPBLAS_TAGGED_PIPELINE_ID)
+    hipBLAS-common: $(HIPBLAS_COMMON_TAGGED_PIPELINE_ID)
+    hipBLASLt: $(HIPBLASLT_TAGGED_PIPELINE_ID)
+    hipCUB: $(HIPCUB_TAGGED_PIPELINE_ID)
+    hipFFT: $(HIPFFT_TAGGED_PIPELINE_ID)
+    hipfort: $(HIPFORT_TAGGED_PIPELINE_ID)
+    HIPIFY: $(HIPIFY_TAGGED_PIPELINE_ID)
+    hipRAND: $(HIPRAND_TAGGED_PIPELINE_ID)
+    hipSOLVER: $(HIPSOLVER_TAGGED_PIPELINE_ID)
+    hipSPARSE: $(HIPSPARSE_TAGGED_PIPELINE_ID)
+    hipSPARSELt: $(HIPSPARSELT_TAGGED_PIPELINE_ID)
+    hipTensor: $(HIPTENSOR_TAGGED_PIPELINE_ID)
+    llvm-project: $(LLVM_PROJECT_TAGGED_PIPELINE_ID)
+    MIOpen: $(MIOpen_TAGGED_PIPELINE_ID)
+    MIVisionX: $(MIVISIONX_TAGGED_PIPELINE_ID)
+    omniperf: $(OMNIPERF_TAGGED_PIPELINE_ID)
+    rccl: $(RCCL_TAGGED_PIPELINE_ID)
+    rdc: $(RDC_TAGGED_PIPELINE_ID)
+    rocAL: $(ROCAL_TAGGED_PIPELINE_ID)
+    rocALUTION: $(ROCALUTION_TAGGED_PIPELINE_ID)
+    rocBLAS: $(ROCBLAS_TAGGED_PIPELINE_ID)
+    ROCdbgapi: $(ROCDBGAPI_TAGGED_PIPELINE_ID)
+    rocDecode: $(ROCDECODE_TAGGED_PIPELINE_ID)
+    rocFFT: $(ROCFFT_TAGGED_PIPELINE_ID)
+    ROCgdb: $(ROCGDB_TAGGED_PIPELINE_ID)
+    rocm-cmake: $(ROCM_CMAKE_TAGGED_PIPELINE_ID)
+    rocm-core: $(ROCM_CORE_TAGGED_PIPELINE_ID)
+    rocm-examples: $(ROCM_EXAMPLES_TAGGED_PIPELINE_ID)
+    rocminfo: $(ROCMINFO_TAGGED_PIPELINE_ID)
+    rocMLIR: $(ROCMLIR_TAGGED_PIPELINE_ID)
+    ROCmValidationSuite: $(ROCMVALIDATIONSUITE_TAGGED_PIPELINE_ID)
+    rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_TAGGED_PIPELINE_ID)
+    rocm_smi_lib: $(ROCM_SMI_LIB_TAGGED_PIPELINE_ID)
+    rocPRIM: $(ROCPRIM_TAGGED_PIPELINE_ID)
+    rocprofiler-register: $(ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID)
+    rocprofiler: $(ROCPROFILER_TAGGED_PIPELINE_ID)
+    ROCR-Runtime: $(ROCR_RUNTIME_TAGGED_PIPELINE_ID)
+    rocRAND: $(ROCRAND_TAGGED_PIPELINE_ID)
+    rocr_debug_agent: $(ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID)
+    rocSOLVER: $(ROCSOLVER_TAGGED_PIPELINE_ID)
+    rocSPARSE: $(ROCSPARSE_TAGGED_PIPELINE_ID)
+    ROCT-Thunk-Interface: $(ROCT_THUNK_INTERFACE_TAGGED_PIPELINE_ID)
+    rocThrust: $(ROCTHRUST_TAGGED_PIPELINE_ID)
+    roctracer: $(ROCTRACER_TAGGED_PIPELINE_ID)
+    rocWMMA: $(ROCWMMA_TAGGED_PIPELINE_ID)
+    rpp: $(RPP_TAGGED_PIPELINE_ID)
 # set to true if you're calling this template file multiple files in same pipeline
 # only leave last call false to optimize sequence
 - name: skipLibraryLinking
   type: boolean
   default: false
+# set to true if llvm-project is not downloaded in a particular call
+# or if you just don't want the symlink
+- name: skipLlvmSymlink
+  type: boolean
+  default: false
+# some ROCm components can specify GPU target and this will affect downloads
+- name: gpuTarget
+  type: string
+  default: ''
+# array of ROCm components that can specify GPU target or have dependency of such a component
+# these components would have parallel build jobs per GPU target so they produce multiple artifacts
+# only download artifact tied to the GPU target
+- name: componentsWithGPUTarget
+  type: object
+  default:
+    - AMDMIGraphX
+    - composable_kernel
+    - hipBLASLt
+    - hipCUB
+    - hipFFT
+    - hipRAND
+    - hipSPARSELt
+    - hipTensor
+    - rccl
+    - rocALUTION
+    - rocBLAS
+    - rocFFT
+    - rocm-examples
+    - rocPRIM
+    - rocprofiler
+    - rocRAND
+    - rocSOLVER
+    - rocSPARSE
+    - rocThrust
+    - roctracer
+    - rocWMMA
+    - rpp
+# list below do not have flag for gpu target but have dependencies of components that do, so build separately per gpu target
+    - hipBLAS
+    - hipSOLVER
+    - hipSPARSE
+    - MIOpen
+    - MIVision
+    - omniperf
+    - rocAL
+    - ROCmValidationSuite
 
 steps:
 # assuming artifact-download.yml template file in same directory
 # for the case where rocm dependency item in list has a colon (:)
 # assume it is of the format of componentName:fileFilter
 # fileFilter could contain both a subcomponent name or gpu name separated by asterisks
+# gpu name will be specified by parameters.gpuTarget for components that are in componentsWithGPUTarget
 # e.g., gfx942 to only download artifacts from component for this gpu if applicable
 - ${{ each dependency in parameters.dependencyList }}:
   - ${{ if contains(dependency, ':') }}:
-    - ${{ if eq(parameters.dependencySource, 'staging') }}:
-      - template: artifact-download.yml
-        parameters:
-          componentName: ${{ split(dependency, ':')[0] }}
-          pipelineId: ${{ parameters.stagingPipelineIdentifiers[split(dependency, ':')[0]] }}
-          fileFilter: ${{ split(dependency, ':')[1] }}
+    - template: artifact-download.yml
+      parameters:
+        componentName: ${{ split(dependency, ':')[0] }}
+        extractToMnt: ${{ parameters.extractToMnt }}
+        ${{ if eq(parameters.dependencySource, 'staging') }}:
+          pipelineId: ${{ parameters.stagingPipelineIdentifiers[ split(dependency, ':')[0] ] }}
           latestFromBranch: ${{ parameters.latestFromBranch }}
-          extractToMnt: ${{ parameters.extractToMnt }}
-    - ${{ if eq(parameters.dependencySource, 'tag-builds') }}:
-      - template: artifact-download.yml
-        parameters:
-          componentName: ${{ split(dependency, ':')[0] }}
-          pipelineId: ${{ parameters.taggedPipelineIdentifiers[split(dependency, ':')[0]] }}
-          fileFilter: ${{ split(dependency, ':')[1] }}
+        ${{ elseif eq(parameters.dependencySource, 'tag-builds') }}:
+          pipelineId: ${{ parameters.taggedPipelineIdentifiers[ split(dependency, ':')[0] ] }}
           latestFromBranch: false
-          extractToMnt: ${{ parameters.extractToMnt }}
+        ${{ if containsValue( parameters.componentsWithGPUTarget, split(dependency, ':')[0] ) }}:
+          fileFilter: "${{ split(dependency, ':')[1] }}*${{ parameters.gpuTarget }}"
+        ${{ else }}:
+          fileFilter: ${{ split(dependency, ':')[1] }}
 # no colon (:) found in this item in the list
   - ${{ else }}:
-    - ${{ if eq(parameters.dependencySource, 'staging') }}:
-      - template: artifact-download.yml
-        parameters:
-          componentName: ${{ dependency }}
+    - template: artifact-download.yml
+      parameters:
+        componentName: ${{ dependency }}
+        extractToMnt: ${{ parameters.extractToMnt }}
+        ${{ if eq(parameters.dependencySource, 'staging') }}:
           pipelineId: ${{ parameters.stagingPipelineIdentifiers[dependency] }}
           latestFromBranch: ${{ parameters.latestFromBranch }}
-          extractToMnt: ${{ parameters.extractToMnt }}
-    - ${{ if eq(parameters.dependencySource, 'tag-builds') }}:
-      - template: artifact-download.yml
-        parameters:
-          componentName: ${{ dependency }}
+        ${{ elseif eq(parameters.dependencySource, 'tag-builds') }}:
           pipelineId: ${{ parameters.taggedPipelineIdentifiers[dependency] }}
           latestFromBranch: false
-          extractToMnt: ${{ parameters.extractToMnt }}
+        ${{ if containsValue( parameters.componentsWithGPUTarget, dependency ) }}:
+          fileFilter: ${{ parameters.gpuTarget }}
 # fixed case only accepts one component at a time, so no array input
 - ${{ if eq(parameters.dependencySource, 'fixed') }}:
   - template: artifact-download.yml
@@ -190,8 +237,23 @@ steps:
       pipelineId: ${{ parameters.fixedPipelineIdentifier }}
       latestFromBranch: false
       extractToMnt: ${{ parameters.extractToMnt }}
+# Set link to redirect llvm folder
+- ${{ if eq(parameters.skipLlvmSymlink, false) }}:
+  - task: Bash@3
+    displayName: Symlink from rocm/llvm to rocm/lib/llvm
+    inputs:
+      targetType: inline
+      script: sudo ln -s $(Agent.BuildDirectory)/rocm/llvm $(Agent.BuildDirectory)/rocm/lib/llvm
+  - task: Bash@3
+    displayName: Symlink executables from rocm/llvm/bin to rocm/bin
+    inputs:
+      targetType: inline
+      script: |
+        for file in amdclang amdclang++ amdclang-cl amdclang-cpp amdflang amdlld aompcc mygpu mycpu offload-arch; do
+          sudo ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/$file $(Agent.BuildDirectory)/rocm/bin/$file
+        done
 - task: Bash@3
-  displayName: 'list downloaded ROCm files'
+  displayName: 'List downloaded ROCm files'
   inputs:
     targetType: inline
     ${{ if eq(parameters.extractToMnt, true) }}:
@@ -200,21 +262,21 @@ steps:
       script: ls -1R $(Agent.BuildDirectory)/rocm
 - ${{ if eq(parameters.skipLibraryLinking, false) }}:
   - task: Bash@3
-    displayName: 'link ROCm shared libraries'
+    displayName: 'Link ROCm shared libraries'
     inputs:
       targetType: inline
 # OS ignores if the ROCm lib folder shows up more than once
       ${{ if eq(parameters.extractToMnt, true) }}:
         script: |
-          echo /mnt/rocm/lib | sudo tee -a /etc/ld.so.conf
-          echo /mnt/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf
-          sudo cat /etc/ld.so.conf
+          echo /mnt/rocm/lib | sudo tee /etc/ld.so.conf.d/rocm-ci.conf
+          echo /mnt/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
+          sudo cat /etc/ld.so.conf.d/rocm-ci.conf
           sudo ldconfig -v
           ldconfig -p
       ${{ else }}:
         script: |
-          echo $(Agent.BuildDirectory)/rocm/lib | sudo tee -a /etc/ld.so.conf
-          echo $(Agent.BuildDirectory)/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf
-          sudo cat /etc/ld.so.conf
+          echo $(Agent.BuildDirectory)/rocm/lib | sudo tee /etc/ld.so.conf.d/rocm-ci.conf
+          echo $(Agent.BuildDirectory)/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
+          sudo cat /etc/ld.so.conf.d/rocm-ci.conf
           sudo ldconfig -v
           ldconfig -p

.azuredevops/templates/steps/local-artifact-download.yml

@@ -0,0 +1,45 @@
+parameters:
+  - name: gpuTarget
+    type: string
+    default: ''
+  # Set buildType to specific to download artifacts from previous builds, useful for saving time when debugging
+  - name: buildType
+    type: string
+    default: current
+    values:
+      - current
+      - specific
+  # Must be set if buildType == specific
+  # Set definitionId to the pipeline ID and buildId to the specific build ID
+  - name: definitionId
+    type: string
+    default: 0
+  - name: buildId
+    type: string
+    default: 0
+
+steps:
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Build'
+    inputs:
+      ${{ if eq(parameters.buildType, 'specific') }}:
+        buildType: specific
+        buildVersionToDownload: specific
+        project: ROCm-CI
+        definition: ${{ parameters.definitionId }}
+        buildId: ${{ parameters.buildId }}
+      itemPattern: '**/*${{ parameters.gpuTarget }}*'
+      targetPath: $(System.ArtifactsDirectory)
+  - task: ExtractFiles@1
+    displayName: 'Extract Pipeline Build'
+    inputs:
+      archiveFilePatterns: '$(System.ArtifactsDirectory)/**/*.tar.gz'
+      destinationFolder: '$(Agent.BuildDirectory)/rocm'
+      cleanDestinationFolder: false
+      overwriteExistingFiles: true
+  - task: DeleteFiles@1
+    displayName: 'Clean up Compressed Pipeline Build'
+    inputs:
+      SourceFolder: '$(System.ArtifactsDirectory)'
+      Contents: '/**/*.tar.xz'
+      RemoveDotFiles: true

.azuredevops/templates/steps/test.yml

@@ -10,15 +10,36 @@ parameters:
   default: 'ctest'
 - name: testParameters
   type: string
-  default: '--output-on-failure --force-new-ctest-process --output-junit test_output.xml'
+  default: '-VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml'
 - name: testOutputFile
   type: string
   default: test_output.xml
 - name: testOutputFormat
   type: string
-  default: 'junit'
+  default: 'JUnit'
+  values:
+    - JUnit
+    - NUnit
+    - VSTest
+    - XUnit
+    - CTest
+- name: testPublishResults
+  type: boolean
+  default: true
+- name: reloadAMDGPU
+  type: boolean
+  default: false
 
 steps:
+# Avoids occasional AMDGPU driver issues with opening /dev/kfd
+- ${{ if parameters.reloadAMDGPU }}:
+  - task: Bash@3
+    displayName: Unload and reload AMDGPU
+    inputs:
+      targetType: inline
+      script: |
+        sudo modprobe -r amdgpu
+        sudo modprobe amdgpu
 # run test, continue on failure to publish results
 # and to publish build artifacts
 - task: Bash@3
@@ -28,9 +49,11 @@ steps:
     targetType: inline
     script: ${{ parameters.testExecutable }} ${{ parameters.testParameters }}
     workingDirectory: ${{ parameters.testDir }}
-- task: PublishTestResults@2
-  displayName: '${{ parameters.componentName }} Publish Results'
-  inputs:
-    testResultsFormat: ${{ parameters.testOutputFormat }}
-    testResultsFiles: '**/${{ parameters.testOutputFile }}'
-  condition: succeededOrFailed()
+- ${{ if parameters.testPublishResults }}:
+  - task: PublishTestResults@2
+    displayName: '${{ parameters.componentName }} Publish Results'
+    inputs:
+      searchFolder: ${{ parameters.testDir }}
+      testResultsFormat: ${{ parameters.testOutputFormat }}
+      testResultsFiles: '**/${{ parameters.testOutputFile }}'
+    condition: succeededOrFailed()

.azuredevops/variables-global.yml

@@ -23,6 +23,8 @@ variables:
   value: rocm-ci_build_pool
 - name: LARGE_DISK_BUILD_POOL
   value: rocm-ci_larger_base_disk_pool
+- name: GFX942_TEST_POOL
+  value: gfx942_test_pool
 - name: LATEST_RELEASE_TAG
   value: rocm-6.1.0
 - name: DOCKER_IMAGE_NAME
@@ -31,3 +33,279 @@ variables:
   value: 6.1
 - name: KEYRING_VERSION
   value: 6.1
+- name: AMDMIGRAPHX_GFX942_TEST_PIPELINE_ID
+  value: 197
+- name: AMDMIGRAPHX_PIPELINE_ID
+  value: 113
+- name: AMDMIGRAPHX_TAGGED_PIPELINE_ID
+  value: 60
+- name: AMDSMI_PIPELINE_ID
+  value: 99
+- name: AMDSMI_TAGGED_PIPELINE_ID
+  value: 33
+- name: AOMP_EXTRAS_PIPELINE_ID
+  value: 111
+- name: AOMP_EXTRAS_TAGGED_PIPELINE_ID
+  value: 75
+- name: AOMP_PIPELINE_ID
+  value: 115
+- name: AOMP_TAGGED_PIPELINE_ID
+  value: 76
+- name: CCACHE_DIR
+  value: $(Pipeline.Workspace)/ccache
+- name: CLR_PIPELINE_ID
+  value: 145
+- name: CLR_TAGGED_PIPELINE_ID
+  value: 71
+- name: COMPOSABLE_KERNEL_GFX942_TEST_PIPELINE_ID
+  value: 179
+- name: COMPOSABLE_KERNEL_PIPELINE_ID
+  value: 86
+- name: COMPOSABLE_KERNEL_TAGGED_PIPELINE_ID
+  value: 38
+- name: FLANG_LEGACY_PIPELINE_ID
+  value: 77
+- name: FLANG_LEGACY_TAGGED_PIPELINE_ID
+  value: 77
+- name: HALF_PIPELINE_ID
+  value: 101
+- name: HALF_TAGGED_PIPELINE_ID
+  value: 11
+- name: HALF560_PIPELINE_ID
+  value: 66
+- name: HALF560_TAGGED_PIPELINE_ID
+  value: 66
+- name: HIP_PIPELINE_ID
+  value: 93
+- name: HIP_TAGGED_PIPELINE_ID
+  value: 31
+- name: HIPBLAS_COMMON_PIPELINE_ID
+  value: 223
+- name: HIPBLAS_COMMON_TAGGED_PIPELINE_ID
+  value: 224
+- name: HIPBLAS_GFX942_TEST_PIPELINE_ID
+  value: 202
+- name: HIPBLAS_PIPELINE_ID
+  value: 87
+- name: HIPBLAS_TAGGED_PIPELINE_ID
+  value: 44
+- name: HIPBLASLT_GFX942_TEST_PIPELINE_ID
+  value: 187
+- name: HIPBLASLT_PIPELINE_ID
+  value: 112
+- name: HIPBLASLT_TAGGED_PIPELINE_ID
+  value: 45
+- name: HIPCUB_GFX942_TEST_PIPELINE_ID
+  value: 186
+- name: HIPCUB_PIPELINE_ID
+  value: 97
+- name: HIPCUB_TAGGED_PIPELINE_ID
+  value: 46
+- name: HIPFFT_GFX942_TEST_PIPELINE_ID
+  value: 198
+- name: HIPFFT_PIPELINE_ID
+  value: 121
+- name: HIPFFT_TAGGED_PIPELINE_ID
+  value: 12
+- name: HIPFORT_PIPELINE_ID
+  value: 102
+- name: HIPFORT_TAGGED_PIPELINE_ID
+  value: 34
+- name: HIPIFY_PIPELINE_ID
+  value: 92
+- name: HIPIFY_TAGGED_PIPELINE_ID
+  value: 13
+- name: HIPRAND_GFX942_TEST_PIPELINE_ID
+  value: 188
+- name: HIPRAND_PIPELINE_ID
+  value: 90
+- name: HIPRAND_TAGGED_PIPELINE_ID
+  value: 42
+- name: HIPSOLVER_GFX942_TEST_PIPELINE_ID
+  value: 201
+- name: HIPSOLVER_PIPELINE_ID
+  value: 84
+- name: HIPSOLVER_TAGGED_PIPELINE_ID
+  value: 52
+- name: HIPSPARSE_GFX942_TEST_PIPELINE_ID
+  value: 195
+- name: HIPSPARSE_PIPELINE_ID
+  value: 83
+- name: HIPSPARSE_TAGGED_PIPELINE_ID
+  value: 14
+- name: HIPSPARSELT_GFX942_TEST_PIPELINE_ID
+  value: 200
+- name: HIPSPARSELT_PIPELINE_ID
+  value: 104
+- name: HIPSPARSELT_TAGGED_PIPELINE_ID
+  value: 53
+- name: HIPTENSOR_GFX942_TEST_PIPELINE_ID
+  value: 192
+- name: HIPTENSOR_PIPELINE_ID
+  value: 105
+- name: HIPTENSOR_TAGGED_PIPELINE_ID
+  value: 56
+- name: LAST_RELEASE
+  value: 6.1.0
+- name: LLVM_PROJECT_PIPELINE_ID
+  value: 2
+- name: LLVM_PROJECT_TAGGED_PIPELINE_ID
+  value: 8
+- name: MIOPEN_PIPELINE_ID
+  value: 108
+- name: MIOPEN_TAGGED_PIPELINE_ID
+  value: 58
+- name: MIVISIONX_PIPELINE_ID
+  value: 80
+- name: MIVISIONX_TAGGED_PIPELINE_ID
+  value: 18
+- name: OMNIPERF_PIPELINE_ID
+  value: 241
+- name: OMNIPERF_TAGGED_PIPELINE_ID
+  value: 242
+- name: RCCL_GFX942_TEST_PIPELINE_ID
+  value: 184
+- name: RCCL_PIPELINE_ID
+  value: 107
+- name: RCCL_TAGGED_PIPELINE_ID
+  value: 15
+- name: RDC_PIPELINE_ID
+  value: 100
+- name: RDC_TAGGED_PIPELINE_ID
+  value: 59
+- name: REIMAGE_ORG
+  value: AGS-ROCm-CI
+- name: REIMAGE_REPO
+  value: cirrascale-reimage-automation
+- name: ROCAL_PIPELINE_ID
+  value: 151
+- name: ROCALUTION_GFX942_TEST_PIPELINE_ID
+  value: 196
+- name: ROCALUTION_PIPELINE_ID
+  value: 89
+- name: ROCALUTION_TAGGED_PIPELINE_ID
+  value: 16
+- name: ROCBLAS_GFX942_TEST_PIPELINE_ID
+  value: 185
+- name: ROCBLAS_PIPELINE_ID
+  value: 85
+- name: ROCBLAS_TAGGED_PIPELINE_ID
+  value: 32
+- name: ROCDBGAPI_PIPELINE_ID
+  value: 135
+- name: ROCDBGAPI_TAGGED_PIPELINE_ID
+  value: 17
+- name: ROCDECODE_PIPELINE_ID
+  value: 79
+- name: ROCDECODE_TAGGED_PIPELINE_ID
+  value: 21
+- name: ROCFFT_GFX942_TEST_PIPELINE_ID
+  value: 189
+- name: ROCFFT_PIPELINE_ID
+  value: 120
+- name: ROCFFT_TAGGED_PIPELINE_ID
+  value: 19
+- name: ROCGDB_PIPELINE_ID
+  value: 134
+- name: ROCGDB_TAGGED_PIPELINE_ID
+  value: 50
+- name: ROCM_BANDWIDTH_TEST_PIPELINE_ID
+  value: 88
+- name: ROCM_BANDWIDTH_TEST_TAGGED_PIPELINE_ID
+  value: 23
+- name: ROCM_CMAKE_PIPELINE_ID
+  value: 6
+- name: ROCM_CMAKE_TAGGED_PIPELINE_ID
+  value: 7
+- name: ROCM_CORE_PIPELINE_ID
+  value: 103
+- name: ROCM_CORE_TAGGED_PIPELINE_ID
+  value: 22
+- name: ROCM_EXAMPLES_GFX942_TEST_PIPELINE_ID
+  value: 204
+- name: ROCM_SMI_LIB_PIPELINE_ID
+  value: 96
+- name: ROCM_SMI_LIB_TAGGED_PIPELINE_ID
+  value: 47
+- name: ROCMINFO_PIPELINE_ID
+  value: 91
+- name: ROCMINFO_TAGGED_PIPELINE_ID
+  value: 27
+- name: ROCMLIR_PIPELINE_ID
+  value: 229
+- name: ROCMLIR_TAGGED_PIPELINE_ID
+  value: 62
+- name: ROCMVALIDATIONSUITE_PIPELINE_ID
+  value: 106
+- name: ROCMVALIDATIONSUITE_TAGGED_PIPELINE_ID
+  value: 43
+- name: ROCPRIM_GFX942_TEST_PIPELINE_ID
+  value: 180
+- name: ROCPRIM_PIPELINE_ID
+  value: 82
+- name: ROCPRIM_TAGGED_PIPELINE_ID
+  value: 20
+- name: ROCPROFILER_GFX942_TEST_PIPELINE_ID
+  value: 190
+- name: ROCPROFILER_PIPELINE_ID
+  value: 143
+- name: ROCPROFILER_REGISTER_PIPELINE_ID
+  value: 1
+- name: ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID
+  value: 25
+- name: ROCPROFILER_TAGGED_PIPELINE_ID
+  value: 28
+- name: ROCR_DEBUG_AGENT_PIPELINE_ID
+  value: 136
+- name: ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID
+  value: 29
+- name: ROCR_RUNTIME_PIPELINE_ID
+  value: 10
+- name: ROCR_RUNTIME_TAGGED_PIPELINE_ID
+  value: 24
+- name: ROCRAND_GFX942_TEST_PIPELINE_ID
+  value: 183
+- name: ROCRAND_PIPELINE_ID
+  value: 95
+- name: ROCRAND_TAGGED_PIPELINE_ID
+  value: 41
+- name: ROCSOLVER_GFX942_TEST_PIPELINE_ID
+  value: 199
+- name: ROCSOLVER_PIPELINE_ID
+  value: 81
+- name: ROCSOLVER_TAGGED_PIPELINE_ID
+  value: 55
+- name: ROCSPARSE_GFX942_TEST_PIPELINE_ID
+  value: 191
+- name: ROCSPARSE_PIPELINE_ID
+  value: 98
+- name: ROCSPARSE_TAGGED_PIPELINE_ID
+  value: 67
+- name: ROCT_THUNK_INTERFACE_PIPELINE_ID
+  value: 3
+- name: ROCT_THUNK_INTERFACE_TAGGED_PIPELINE_ID
+  value: 9
+- name: ROCTHRUST_GFX942_TEST_PIPELINE_ID
+  value: 194
+- name: ROCTHRUST_PIPELINE_ID
+  value: 94
+- name: ROCTHRUST_TAGGED_PIPELINE_ID
+  value: 26
+- name: ROCTRACER_GFX942_TEST_PIPELINE_ID
+  value: 181
+- name: ROCTRACER_PIPELINE_ID
+  value: 141
+- name: ROCTRACER_TAGGED_PIPELINE_ID
+  value: 30
+- name: ROCWMMA_GFX942_TEST_PIPELINE_ID
+  value: 193
+- name: ROCWMMA_PIPELINE_ID
+  value: 109
+- name: ROCWMMA_TAGGED_PIPELINE_ID
+  value: 57
+- name: RPP_GFX942_TEST_PIPELINE_ID
+  value: 182
+- name: RPP_PIPELINE_ID
+  value: 78
+- name: RPP_TAGGED_PIPELINE_ID
+  value: 39

.gitmodules

@@ -1,183 +0,0 @@
-[submodule "libs/ROCK-Kernel-Driver"]
-	path = libs/ROCK-Kernel-Driver
-	url = ../ROCK-Kernel-Driver
-[submodule "libs/ROCT-Thunk-Interface"]
-	path = libs/ROCT-Thunk-Interface
-	url = ../ROCT-Thunk-Interface
-[submodule "libs/ROCR-Runtime"]
-	path = libs/ROCR-Runtime
-	url = ../ROCR-Runtime
-[submodule "libs/amdsmi"]
-	path = libs/amdsmi
-	url = ../amdsmi
-[submodule "libs/rocm_smi_lib"]
-	path = libs/rocm_smi_lib
-	url = ../rocm_smi_lib
-[submodule "libs/rocm-core"]
-	path = libs/rocm-core
-	url = ../rocm-core
-[submodule "libs/rocm-cmake"]
-	path = libs/rocm-cmake
-	url = ../rocm-cmake
-[submodule "libs/rocminfo"]
-	path = libs/rocminfo
-	url = ../rocminfo
-[submodule "libs/rocm_bandwidth_test"]
-	path = libs/rocm_bandwidth_test
-	url = ../rocm_bandwidth_test
-[submodule "libs/rocprofiler"]
-	path = libs/rocprofiler
-	url = ../rocprofiler
-[submodule "libs/roctracer"]
-	path = libs/roctracer
-	url = ../roctracer
-[submodule "libs/rdc"]
-	path = libs/rdc
-	url = ../rdc
-[submodule "libs/HIP"]
-	path = libs/HIP
-	url = ../HIP
-[submodule "libs/clr"]
-	path = libs/clr
-	url = ../clr
-[submodule "libs/hipother"]
-	path = libs/hipother
-	url = ../hipother
-[submodule "libs/HIPIFY"]
-	path = libs/HIPIFY
-	url = ../HIPIFY
-[submodule "libs/HIPCC"]
-	path = libs/HIPCC
-	url = ../HIPCC
-[submodule "libs/llvm-project"]
-	path = libs/llvm-project
-	url = ../llvm-project
-[submodule "libs/ROCm-Device-Libs"]
-	path = libs/ROCm-Device-Libs
-	url = ../ROCm-Device-Libs
-[submodule "libs/ROCm-CompilerSupport"]
-	path = libs/ROCm-CompilerSupport
-	url = ../ROCm-CompilerSupport
-[submodule "libs/half"]
-	path = libs/half
-	url = ../half
-[submodule "libs/ROCgdb"]
-	path = libs/ROCgdb
-	url = ../ROCgdb
-[submodule "libs/ROCdbgapi"]
-	path = libs/ROCdbgapi
-	url = ../ROCdbgapi
-[submodule "libs/rocr_debug_agent"]
-	path = libs/rocr_debug_agent
-	url = ../rocr_debug_agent
-[submodule "libs/rocBLAS"]
-	path = libs/rocBLAS
-	url = ../rocBLAS
-[submodule "libs/Tensile"]
-	path = libs/Tensile
-	url = ../Tensile
-[submodule "libs/hipTensor"]
-	path = libs/hipTensor
-	url = ../hipTensor
-[submodule "libs/hipBLAS"]
-	path = libs/hipBLAS
-	url = ../hipBLAS
-[submodule "libs/hipBLASLt"]
-	path = libs/hipBLASLt
-	url = ../hipBLASLt
-[submodule "libs/rocFFT"]
-	path = libs/rocFFT
-	url = ../rocFFT
-[submodule "libs/hipFFT"]
-	path = libs/hipFFT
-	url = ../hipFFT
-[submodule "libs/rocRAND"]
-	path = libs/rocRAND
-	url = ../rocRAND
-[submodule "libs/hipRAND"]
-	path = libs/hipRAND
-	url = ../hipRAND
-[submodule "libs/rocSPARSE"]
-	path = libs/rocSPARSE
-	url = ../rocSPARSE
-[submodule "libs/hipSPARSELt"]
-	path = libs/hipSPARSELt
-	url = ../hipSPARSELt
-[submodule "libs/rocSOLVER"]
-	path = libs/rocSOLVER
-	url = ../rocSOLVER
-[submodule "libs/hipSOLVER"]
-	path = libs/hipSOLVER
-	url = ../hipSOLVER
-[submodule "libs/hipSPARSE"]
-	path = libs/hipSPARSE
-	url = ../hipSPARSE
-[submodule "libs/rocALUTION"]
-	path = libs/rocALUTION
-	url = ../rocALUTION
-[submodule "libs/rocThrust"]
-	path = libs/rocThrust
-	url = ../rocThrust
-[submodule "libs/hipCUB"]
-	path = libs/hipCUB
-	url = ../hipCUB
-[submodule "libs/rocPRIM"]
-	path = libs/rocPRIM
-	url = ../rocPRIM
-[submodule "libs/rocWMMA"]
-	path = libs/rocWMMA
-	url = ../rocWMMA
-[submodule "libs/rccl"]
-	path = libs/rccl
-	url = ../rccl
-[submodule "libs/MIOpen"]
-	path = libs/MIOpen
-	url = ../MIOpen
-[submodule "libs/composable_kernel"]
-	path = libs/composable_kernel
-	url = ../composable_kernel
-[submodule "libs/MIVisionX"]
-	path = libs/MIVisionX
-	url = ../MIVisionX
-[submodule "libs/rpp"]
-	path = libs/rpp
-	url = ../rpp
-[submodule "libs/hipfort"]
-	path = libs/hipfort
-	url = ../hipfort
-[submodule "libs/AMDMIGraphX"]
-	path = libs/AMDMIGraphX
-	url = ../AMDMIGraphX
-[submodule "libs/ROCmValidationSuite"]
-	path = libs/ROCmValidationSuite
-	url = ../ROCmValidationSuite
-[submodule "libs/openmp-extras/aomp"]
-	path = libs/openmp-extras/aomp
-	url = ../aomp
-[submodule "libs/openmp-extras/aomp-extras"]
-	path = libs/openmp-extras/aomp-extras
-	url = ../aomp-extras
-[submodule "libs/openmp-extras/flang"]
-	path = libs/openmp-extras/flang
-	url = ../flang
-[submodule "libs/rocDecode"]
-	path = libs/rocDecode
-	url = ../rocDecode
-[submodule "libs/omnitrace"]
-	path = libs/omnitrace
-	url = ../omnitrace
-[submodule "libs/omniperf"]
-	path = libs/omniperf
-	url = ../omniperf
-[submodule "libs/rocprofiler-sdk"]
-	path = libs/rocprofiler-sdk
-	url = ../rocprofiler-sdk
-[submodule "libs/rocm-examples"]
-	path = libs/rocm-examples
-	url = ../rocm-examples
-[submodule "libs/rocPyDecode"]
-	path = libs/rocPyDecode
-	url = ../rocPyDecode
-[submodule "libs/rocAL"]
-	path = libs/rocAL
-	url = ../rocAL

.wordlist.txt

@@ -53,6 +53,7 @@ CSC
 CSE
 CSV
 CSn
+CTest
 CTests
 CU
 CUDA
@@ -97,6 +98,7 @@ ENDPGM
 EPYC
 ESXi
 EoS
+FBGEMM
 FFT
 FFTs
 FFmpeg
@@ -110,6 +112,7 @@ Flang
 Fortran
 Fuyu
 GALB
+GCC
 GCD
 GCDs
 GCN
@@ -175,6 +178,7 @@ Interop
 Intersphinx
 Intra
 Ioffe
+Jinja
 JSON
 Jupyter
 KFD
@@ -221,6 +225,7 @@ Megatron
 Mellanox
 Mellanox's
 Meta's
+Miniconda
 MirroredStrategy
 Multicore
 Multithreaded
@@ -383,6 +388,7 @@ UAC
 UC
 UCC
 UCX
+UE
 UIF
 UMC
 USM
@@ -430,6 +436,7 @@ accuracies
 activations
 addr
 alloc
+allocatable
 allocator
 allocators
 amdgpu
@@ -450,6 +457,7 @@ blit
 bootloader
 boson
 bosons
+br
 buildable
 bursty
 bzip
@@ -577,6 +585,7 @@ invariants
 invocating
 ipo
 kdb
+kfd
 latencies
 libfabric
 libjpeg
@@ -619,6 +628,7 @@ performant
 perl
 pragma
 pre
+prebuild
 prebuilt
 precompiled
 preconditioner
@@ -645,6 +655,7 @@ quasirandom
 queueing
 rccl
 rdc
+rdma
 reStructuredText
 redirections
 refactorization
@@ -710,8 +721,10 @@ subexpression
 subfolder
 subfolders
 submodule
+submodules
 supercomputing
 symlink
+symlinks
 td
 tensorfloat
 th

README.md

@@ -21,7 +21,19 @@ source software compilers, debuggers, and libraries. ROCm is fully integrated in
 
 ## Getting the ROCm Source Code
 
-AMD ROCm is built from open source software. It is, therefore, possible to modify the various components of ROCm by downloading the source code and rebuilding the components. The source code for ROCm components can be cloned from each of the GitHub repositories using git.  For easy access to download the correct versions of each of these tools, the ROCm repository contains submodules that point to the correct versions of each of the ROCm components. They can be found in the `/libs` directory of the ROCm repository.
+AMD ROCm is built from open source software. It is, therefore, possible to modify the various components of ROCm by downloading the source code and rebuilding the components. The source code for ROCm components can be cloned from each of the GitHub repositories using git.  For easy access to download the correct versions of each of these tools, the ROCm repository contains a repo manifest file called [default.xml](./default.xml). You can use this manifest file to download the source code for ROCm software.
+
+### Installing the repo tool
+
+The repo tool from Google allows you to manage multiple git repositories simultaneously. Run the following commands to install the repo tool:
+
+```bash
+mkdir -p ~/bin/
+curl https://storage.googleapis.com/git-repo-downloads/repo > ~/bin/repo
+chmod a+x ~/bin/repo
+```
+
+**Note:** The ```~/bin/``` folder is used as an example. You can specify a different folder to install the repo tool into if you desire.
 
 ### Installing git-lfs
 
@@ -33,12 +45,17 @@ sudo apt-get install git-lfs
 
 ### Downloading the ROCm source code
 
-The following example shows how to download the ROCm source from this repository.
+The following example shows how to use the repo tool to download the ROCm source code. If you choose a directory other than ~/bin/ to install the repo tool, you must use that chosen directory in the code as shown below:
 
 ```bash
-git clone https://github.com/ROCm/ROCm -b amd/dgaliffi/submodules-6-2-0 --recurse-submodules
+mkdir -p ~/ROCm/
+cd ~/ROCm/
+~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.2.x
+~/bin/repo sync
 ```
 
+**Note:** Using this sample code will cause the repo tool to download the open source code associated with the specified ROCm release. Ensure that you have ssh-keys configured on your machine for your GitHub ID prior to the download as explained at [Connecting to GitHub with SSH](https://docs.github.com/en/authentication/connecting-to-github-with-ssh).
+
 ## Building the ROCm source code
 
 Each ROCm component repository contains directions for building that component, such as the rocSPARSE documentation [Installation and Building for Linux](https://rocm.docs.amd.com/projects/rocSPARSE/en/latest/install/Linux_Install_Guide.html). Refer to the specific component documentation for instructions on building the repository.
@@ -59,8 +76,9 @@ The Build time will reduce significantly if we limit the GPU Architecture/s agai
 
 mkdir -p ~/WORKSPACE/      # Or any folder name other than WORKSPACE
 cd ~/WORKSPACE/
-export ROCM_VERSION=6.2.0   # or 6.1.1 6.1.2
-git clone https://github.com/ROCm/ROCm -b amd/dgaliffi/submodules-${ROCM_VERSION} --recurse-submodules
+export ROCM_VERSION=6.2.0
+~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.2.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
+~/bin/repo sync
 
 # --------------------------------------
 # Step 2: Prepare build environment
@@ -69,9 +87,11 @@ git clone https://github.com/ROCm/ROCm -b amd/dgaliffi/submodules-${ROCM_VERSION
 # Option 1: Start a docker container
 # Pulling required base docker images:
 # Ubuntu20.04 built from ROCm/tools/rocm-build/docker/ubuntu20/Dockerfile
-docker pull rocm/rocm-build-ubuntu-20.04:6.1
+docker pull rocm/rocm-build-ubuntu-20.04:6.2
 # Ubuntu22.04 built from ROCm/tools/rocm-build/docker/ubuntu22/Dockerfile
-docker pull rocm/rocm-build-ubuntu-22.04:6.1
+docker pull rocm/rocm-build-ubuntu-22.04:6.2
+# Ubuntu24.04 built from ROCm/tools/rocm-build/docker/ubuntu24/Dockerfile
+docker pull rocm/rocm-build-ubuntu-24.04:6.2
 
 # Start docker container and mount the source code folder:
 docker run -ti \
@@ -90,10 +110,16 @@ docker run -ti \
 # Option 2: Install required packages into the host machine
 # For ubuntu20.04 system
 cd ROCm/tools/rocm-build/docker/ubuntu20
+cp * /tmp && cd /tmp
 bash install-prerequisites.sh
 # For ubuntu22.04 system
 cd ROCm/tools/rocm-build/docker/ubuntu22
+cp * /tmp && cd /tmp
 bash install-prerequisities.sh
+# For ubuntu24.04 system
+cd ROCm/tools/rocm-build/docker/ubuntu24
+cp * /tmp && cd /tmp
+bash install-prerequisites.sh
 
 # --------------------------------------
 # Step 3: Run build command line
@@ -120,13 +146,17 @@ make -f ROCm/tools/rocm-build/ROCm.mk T_rocblas
 out/ubuntu-20.04/20.04/deb/
 # Find built packages in ubuntu22.04:
 out/ubuntu-22.04/22.04/deb/
+# Find built packages in ubuntu24.04:
+out/ubuntu-24.04/24.04/deb/
 
 # Find built logs in ubuntu20.04:
 out/ubuntu-20.04/20.04/logs/
 # Find built logs in ubuntu22.04:
 out/ubuntu-22.04/22.04/logs/
+# Find built logs in ubuntu24.04:
+out/ubuntu-24.04/24.04/logs/
 # All logs pertaining to failed components, end with .errrors extension.
-out/ubuntu-22.04/22.04/logs/rocblas.errors          # Example
+out/ubuntu-22.04/22.04/logs/rocblas.errors      # Example
 # All logs pertaining to building components, end with .inprogress extension.
 out/ubuntu-22.04/22.04/logs/rocblas.inprogress  # Example
 # All logs pertaining to passed components, use the component names.
@@ -137,6 +167,12 @@ Note: [Overview for ROCm.mk](tools/rocm-build/README.md)
 
 ## ROCm documentation
 
+This repository contains the [manifest file](https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md)
+for ROCm releases, changelogs, and release information.
+
+The `default.xml` file contains information for all repositories and the associated commit used to build
+the current ROCm release; `default.xml` uses the [Manifest Format repository](https://gerrit.googlesource.com/git-repo/).
+
 Source code for our documentation is located in the `/docs` folder of most ROCm repositories. The
 `develop` branch of our repositories contains content for the next ROCm release.
 

default.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <manifest>
     <remote name="rocm-org" fetch="https://github.com/ROCm/" />
-    <default revision="refs/tags/rocm-6.2.0"
+    <default revision="refs/tags/rocm-6.2.1"
      remote="rocm-org"
      sync-c="true"
      sync-j="4" />
@@ -16,6 +16,7 @@
     <project name="rocm_bandwidth_test" />
     <project name="rocm_smi_lib" />
     <project name="rocm-core" />
+    <project name="rocm-examples" />
     <project name="rocminfo" />
     <project name="rocprofiler" />
     <project name="rocprofiler-register" />
@@ -24,7 +25,6 @@
 <!--HIP Projects-->
     <project name="HIP" />
     <project name="hip-tests" />
-    <project name="HIP-Examples" />
     <project name="HIPIFY" />
     <project name="clr" />
     <project name="hipother" />

docs/compatibility/compatibility-matrix-historical-6.0.csv

@@ -0,0 +1,117 @@
+ROCm Version,6.2.1,6.2.0, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
+      :ref:`Operating Systems & kernels <OS-kernel-versions>`,Ubuntu 24.04,Ubuntu 24.04,,,,,
+      ,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3, 22.04.2","Ubuntu 22.04.4, 22.04.3, 22.04.2"
+      ,,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
+      ,"RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
+      ,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
+      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
+      ,,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
+      ,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,,,
+      ,.. _architecture-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
+      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
+      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
+      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
+      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
+      ,.. _gpu-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
+      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
+      ,gfx942 [#mi300_621-past-60]_,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
+      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
+      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
+      ,,,,,,,
+      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
+      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1","2.14.0, 2.13.1, 2.12.1"
+      :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
+      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
+      ,,,,,,,
+      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix-past-60:,,,,,,
+      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
+      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
+      ,,,,,,,
+      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix-past-60:,,,,,,
+      Thrust,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+      CUB,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+      ,,,,,,,
+      KFD & USER SPACE [#kfd_support-past-60]_,.. _kfd-userspace-support-compatibility-matrix-past-60:,,,,,,
+      Tested user space versions,"6.1.x, 6.0.x","6.1.x, 6.0.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x, 5.6.x","6.2.x, 6.0.x, 5.7.x, 5.6.x"
+      ,,,,,,,
+      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
+      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.10.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
+      :doc:`MIOpen <miopen:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`MIVisionX <mivisionx:index>`,3.0.0,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
+      :doc:`rocAL <rocal:index>`,2.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
+      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,0.1.0,N/A,N/A,N/A,N/A,N/A
+      :doc:`RPP <rpp:index>`,1.8.0,1.8.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
+      ,,,,,,,
+      COMMUNICATION,.. _commlibs-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`RCCL <rccl:index>`,2.20.5,2.20.5,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
+      ,,,,,,,
+      MATH LIBS,.. _mathlibs-support-compatibility-matrix-past-60:,,,,,,
+      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
+      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
+      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.8.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
+      :doc:`hipFFT <hipfft:index>`,1.0.15,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
+      :doc:`hipFORT <hipfort:index>`,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
+      :doc:`hipRAND <hiprand:index>`,2.11.0,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
+      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.2.0,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
+      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.1.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.1,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
+      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.2.0,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
+      :doc:`rocBLAS <rocblas:index>`,4.2.1,4.2.0,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
+      :doc:`rocFFT <rocfft:index>`,1.0.29,1.0.28,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
+      :doc:`rocRAND <rocrand:index>`,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
+      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.26.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
+      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.2.0,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
+      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.5.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
+      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
+      ,,,,,,,
+      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`hipCUB <hipcub:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.3.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
+      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`rocThrust <rocthrust:index>`,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      ,,,,,,,
+      SUPPORT LIBS,,,,,,,
+      `hipother <https://github.com/ROCm/hipother>`_,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.1,6.2.0,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
+      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.4.05,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
+      ,,,,,,,
+      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`AMD SMI <amdsmi:index>`,24.6.3,24.6.2,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
+      :doc:`ROCm Data Center Tool <rdc:index>`,1.0.0,1.0.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
+      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.3.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
+      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.1,rocm-6.2.0,rocm-6.1.2,rocm-6.1.1,rocm-6.1.0,rocm-6.0.2,rocm-6.0.0
+      ,,,,,,,
+      PERFORMANCE TOOLS,,,,,,,
+      :doc:`Omniperf <omniperf:index>`,2.0.1,2.0.1,N/A,N/A,N/A,N/A,N/A
+      :doc:`Omnitrace <omnitrace:index>`,1.11.2,1.11.2,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
+      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60201,2.0.60200,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
+      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,0.4.0,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCTracer <roctracer:index>`,4.1.60201,4.1.60200,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
+      ,,,,,,,
+      DEVELOPMENT TOOLS,,,,,,,
+      :doc:`HIPIFY <hipify:index>`,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.13.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
+      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
+      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.2.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
+      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.3.0,0.3.0,0.3.0,N/A,N/A
+      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
+      ,,,,,,,
+      COMPILERS,.. _compilers-support-compatibility-matrix-past-60:,,,,,,
+      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
+      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`llvm-project <llvm-project:index>`,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24332,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      ,,,,,,,
+      RUNTIMES,.. _runtime-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      :doc:`HIP <hip:index>`,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
+      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.14.0,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0

docs/compatibility/compatibility-matrix.rst

@@ -1,141 +1,189 @@
 .. meta::
     :description: ROCm compatibility matrix
-    :keywords: AMD, GPU, architecture, hardware, compatibility, requirements
+    :keywords: GPU, architecture, hardware, compatibility, system, requirements, components, libraries
 
 **************************************************************************************
 Compatibility matrix
 **************************************************************************************
 
-Use this matrix to view the ROCm compatibility across successive major and minor releases.
+Use this matrix to view the ROCm compatibility and system requirements across successive major and minor releases.
 
 You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-rocm-compatibility-matrix>`.
 
+.. |br| raw:: html
+
+   <br/>
+
 .. container:: format-big-table
 
   .. csv-table:: 
-      :header: "ROCm Version", "6.2.0", "6.1.2", "6.0.0"
+      :header: "ROCm Version", "6.2.1", "6.2.0", "6.1.0"
       :stub-columns: 1
 
-      :doc:`Operating Systems <rocm-install-on-linux:reference/system-requirements>`, "Ubuntu 24.04","",""
-      ,"Ubuntu 22.04.5 [#Ubuntu220405]_, 22.04.4","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3"
-      ,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
-      ,"RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94]_, 9.3, 9.2","RHEL 9.3, 9.2"
-      ,"RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
-      ,"SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
-      ,,CentOS 7.9,CentOS 7.9
-      ,"Oracle Linux 8.9 [#oracle89]_","Oracle Linux 8.9 [#oracle89]_",""
-      ,".. _architecture-support-compatibility-matrix:",,
+      :ref:`Operating Systems & kernels <OS-kernel-versions>`,Ubuntu 24.04,Ubuntu 24.04,
+      ,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.4, 22.04.3"
+      ,,,"Ubuntu 20.04.6, 20.04.5"
+      ,"RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94]_, 9.3, 9.2"
+      ,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8"
+      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4"
+      ,,,CentOS 7.9
+      ,Oracle Linux 8.9 [#oracle89]_,Oracle Linux 8.9 [#oracle89]_,
+      ,.. _architecture-support-compatibility-matrix:,,
       :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3
       ,CDNA2,CDNA2,CDNA2
       ,CDNA,CDNA,CDNA
       ,RDNA3,RDNA3,RDNA3
       ,RDNA2,RDNA2,RDNA2
-      ,".. _gpu-support-compatibility-matrix:",,
+      ,.. _gpu-support-compatibility-matrix:,,
       :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100
       ,gfx1030,gfx1030,gfx1030
-      ,gfx942 [#mi300_620]_, gfx942 [#mi300_612]_, gfx942 [#mi300_600]_
+      ,gfx942 [#mi300_621]_,gfx942 [#mi300_620]_, gfx942 [#mi300_610]_
       ,gfx90a,gfx90a,gfx90a
       ,gfx908,gfx908,gfx908
       ,,,
-      FRAMEWORK SUPPORT,".. _framework-support-compatibility-matrix:",,
-      :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.15, 2.14, 2.13","2.14, 2.13, 2.12"
+      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix:,,
+      :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13"
+      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15, 2.14, 2.13"
       :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26
-      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.14.1
+      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3
       ,,,
-      THIRD PARTY COMMS,".. _thirdpartycomms-support-compatibility-matrix:",,
-      `UCC <https://github.com/ROCm/ucc>`_,>=1.2.0,>=1.2.0,>=1.2.0
-      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.14.1,>=1.14.1
+      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix:,,
+      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0
+      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.14.1
       ,,,
-      THIRD PARTY ALGORITHM,".. _thirdpartyalgorithm-support-compatibility-matrix:",,
-      Thrust,2.2.0,2.1.0,2.0.1
-      CUB,2.2.0,2.1.0,2.0.1
+      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix:,,
+      Thrust,2.2.0,2.2.0,2.1.0
+      CUB,2.2.0,2.2.0,2.1.0
       ,,,
-      ML & COMPUTER VISION,".. _mllibs-support-compatibility-matrix:",,
+      KFD & USER SPACE [#kfd_support]_,.. _kfd-userspace-support-compatibility-matrix:,,
+      Tested user space versions,"6.1.x, 6.0.x","6.1.x, 6.0.x","6.2.x, 6.0.x, 5.7.x"
+      ,,,
+      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix:,,
       :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0
-      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.9.0,2.8.0
-      :doc:`MIOpen <miopen:index>`,3.2.0,3.1.0,3.0.0
-      :doc:`MIVisionX <mivisionx:index>`,3.0.0,2.5.0,2.5.0
-      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,N/A
-      :doc:`RPP <rpp:index>`,1.8.0,1.5.0,1.4.0
-      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,N/A,N/A
+      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.10.0,2.9.0
+      :doc:`MIOpen <miopen:index>`,3.2.0,3.2.0,3.1.0
+      :doc:`MIVisionX <mivisionx:index>`,3.0.0,3.0.0,2.5.0
+      :doc:`rocAL <rocal:index>`,2.0.0,1.0.0,1.0.0
+      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.5.0
+      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,0.1.0,N/A
+      :doc:`RPP <rpp:index>`,1.8.0,1.8.0,1.5.0
       ,,,
-      COMMUNICATION,".. _commlibs-support-compatibility-matrix:",,
-      :doc:`RCCL <rccl:index>`,2.20.5,2.18.6,2.18.3
+      COMMUNICATION,.. _commlibs-support-compatibility-matrix:,,
+      :doc:`RCCL <rccl:index>`,2.20.5,2.20.5,2.18.6
       ,,,
-      MATH LIBS,".. _mathlibs-support-compatibility-matrix:",,
+      MATH LIBS,.. _mathlibs-support-compatibility-matrix:,,
       `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0
-      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.1.0,2.0.0
-      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.7.0,0.6.0
-      :doc:`hipFFT <hipfft:index>`,1.0.14,1.0.14,1.0.13
+      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.2.0,2.1.0
+      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.8.0,0.7.0
+      :doc:`hipFFT <hipfft:index>`,1.0.15,1.0.14,1.0.14
       :doc:`hipFORT <hipfort:index>`,0.4.0,0.4.0,0.4.0
-      :doc:`hipRAND <hiprand:index>`,2.11.0,2.10.16,2.10.16
-      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.1.1,2.0.0
-      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.0.1,3.0.0
-      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.0,0.1.0
-      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.1.1,3.0.3
-      :doc:`rocBLAS <rocblas:index>`,4.2.0,4.1.2,4.0.0
-      :doc:`rocFFT <rocfft:index>`,1.0.28,1.0.27,1.0.23
-      :doc:`rocRAND <rocrand:index>`,3.1.0,3.0.1,2.10.17
-      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.25.0,3.24.0
-      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.1.2,3.0.2
-      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.4.0,1.3.0
-      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.39.0
+      :doc:`hipRAND <hiprand:index>`,2.11.0,2.11.0,2.10.16
+      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.2.0,2.1.0
+      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.1.1,3.0.1
+      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.1,0.1.0
+      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.2.0,3.1.1
+      :doc:`rocBLAS <rocblas:index>`,4.2.1,4.2.0,4.1.0
+      :doc:`rocFFT <rocfft:index>`,1.0.29,1.0.28,1.0.26
+      :doc:`rocRAND <rocrand:index>`,3.1.0,3.1.0,3.0.1
+      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.26.0,3.25.0
+      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.2.0,3.1.2
+      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.5.0,1.4.0
+      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.40.0
       ,,,
-      PRIMITIVES,".. _primitivelibs-support-compatibility-matrix:",,
-      :doc:`hipCUB <hipcub:index>`,3.2.0,3.1.0,3.0.0
-      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.2.0,1.1.0
-      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.1.0,3.0.0
-      :doc:`rocThrust <rocthrust:index>`,3.0.1,3.0.1,3.0.0
+      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix:,,
+      :doc:`hipCUB <hipcub:index>`,3.2.0,3.2.0,3.1.0
+      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.3.0,1.2.0
+      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.2.0,3.1.0
+      :doc:`rocThrust <rocthrust:index>`,3.1.0,3.0.1,3.0.1
       ,,,
       SUPPORT LIBS,,,
-      `hipother <https://github.com/ROCm/hipother>`_,6.2.41133,6.1.40093,6.1.32830
-      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.0,6.1.2,6.0.0
-      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.1.4246,20240125.5.08,20231016.2.245
+      `hipother <https://github.com/ROCm/hipother>`_,6.2.41134,6.2.41133,6.1.40091
+      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.1,6.2.0,6.1.0
+      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.4.05,20240607.1.4246,20240125.3.30
       ,,,
-      SYSTEM MGMT TOOLS,".. _tools-support-compatibility-matrix:",,
-      :doc:`AMD SMI <amdsmi:index>`,24.6.2,24.5.1,23.4.2
-      :doc:`ROCm Data Center Tool <rdc:index>`,1.0.0,0.3.0,0.3.0
+      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix:,,
+      :doc:`AMD SMI <amdsmi:index>`,24.6.3,24.6.2,24.4.1
+      :doc:`ROCm Data Center Tool <rdc:index>`,1.0.0,1.0.0,0.3.0
       :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0
-      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.2.0,6.0.0
-      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.0,rocm-6.1.2,rocm-6.0.0
+      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.3.0,7.0.0
+      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.1,rocm-6.2.0,rocm-6.1.0
       ,,,
       PERFORMANCE TOOLS,,,
-      :doc:`Omniperf <omniperf:index>`,2.0.1,N/A,N/A
-      :doc:`Omnitrace <omnitrace:index>`,1.11.2,N/A,N/A
+      :doc:`Omniperf <omniperf:index>`,2.0.1,2.0.1,N/A
+      :doc:`Omnitrace <omnitrace:index>`,1.11.2,1.11.2,N/A
       :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0
-      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60200,2.0.60102,2.0.60000
-      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,N/A,N/A
-      :doc:`ROCTracer <roctracer:index>`,4.1.60200,4.1.60102,4.1.60000
+      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60201,2.0.60200,2.0.60100
+      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,0.4.0,N/A
+      :doc:`ROCTracer <roctracer:index>`,4.1.60201,4.1.60200,4.1.60100
       ,,,
       DEVELOPMENT TOOLS,,,
-      :doc:`HIPIFY <hipify:index>`,18.0.0.24232,17.0.0.24193,17.0.0.23483
-      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.12.0,0.11.0
-      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.71.0,0.71.0
-      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.1.0,13.2.0
-      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.3.0,N/A
+      :doc:`HIPIFY <hipify:index>`,18.0.0.24355,18.0.0.24232,17.0.0.24103
+      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.13.0,0.12.0
+      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.76.0,0.71.0
+      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.2.0,14.1.0
+      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.3.0
       :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3
       ,,,
-      COMPILERS,".. _compilers-support-compatibility-matrix:",,
-      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,0.5.0,0.5.0
-      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24232,17.0.0.24193,17.0.0.23483
-      `llvm-project <https://github.com/ROCm/llvm-project>`_,18.0.0.24232,17.0.0.24193,17.0.0.23483
-      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24232,17.0.0.24193,17.0.0.23483
+      COMPILERS,.. _compilers-support-compatibility-matrix:,,
+      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,0.5.0
+      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.0.0
+      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24355,18.0.0.24232,17.0.0.24103
+      :doc:`llvm-project <llvm-project:index>`,18.0.0.24355,18.0.0.24232,17.0.0.24103
+      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24332,18.0.0.24232,17.0.0.24103
       ,,,
-      RUNTIMES,".. _runtime-support-compatibility-matrix:",,
-      :doc:`HIP <hip:index>`,6.2.41133,6.1.40093,6.1.32830
+      RUNTIMES,.. _runtime-support-compatibility-matrix:,,
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41134,6.2.41133,6.1.40091
+      :doc:`HIP <hip:index>`,6.2.41134,6.2.41133,6.1.40091
       `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0
-      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.13.0,1.13.0,1.12.0
+      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.14.0,1.13.0,1.13.0
 
 
 .. rubric:: Footnotes
 
-.. [#Ubuntu220405] Preview support of Ubuntu 22.04.5 only
 .. [#red-hat94] RHEL 9.4 is supported only on AMD Instinct MI300A.
 .. [#oracle89] Oracle Linux is supported only on AMD Instinct MI300X.
+.. [#mi300_621] **For ROCm 6.2.1** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
 .. [#mi300_620] **For ROCm 6.2.0** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
-.. [#mi300_612] **For ROCm 6.1.2** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4 and Oracle Linux.
-.. [#mi300_600] **For ROCm 6.0.0** - MI300A (gfx942) is supported on Ubuntu 22.04.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.3.
+.. [#mi300_610] **For ROCm 6.1.0** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4.
+.. [#kfd_support] ROCm provides forward and backward compatibility between the Kernel Fusion Driver (KFD) and its user space software for +/- 2 releases. These are the compatibility combinations that are currently supported.
+
+
+.. _OS-kernel-versions:
+
+Operating systems and kernel versions
+*************************************
+
+Use this look up table to confirm which operating system and kernel versions are supported with ROCm.
+
+.. csv-table:: 
+   :header: "OS", "Version", "Kernel"
+   :widths: 40, 20, 40
+   :stub-columns: 1
+
+   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 24.04, "6.8 GA"
+   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 22.04.05, "5.15 GA, 6.8 HWE"
+   , 22.04.04, "5.15 GA, 6.5 HWE"
+   , 22.04.03, "5.15 GA, 6.2 HWE"
+   , 22.04.02, "5.15 GA, 5.19 HWE"
+   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 20.04.06, "5.15 HWE"
+   , 20.04.05, "5.15 HWE"
+   ,,
+   `Red Hat Enterprise Linux (RHEL) <https://access.redhat.com/articles/3078#RHEL9>`_, 9.4, 5.14.0
+   ,9.3, 5.14.0
+   ,9.2, 5.14.0
+   ,,
+   `Red Hat Enterprise Linux (RHEL) <https://access.redhat.com/articles/3078#RHEL8>`_, 8.10, 4.18.0
+   ,8.9, 4.18.0
+   ,8.8, 4.18.0
+   ,,
+   `CentOS <https://access.redhat.com/articles/3078#RHEL7>`_, 7.9, 3.10
+   ,,
+   `SUSE Linux Enterprise Server (SLES) <https://www.suse.com/support/kb/doc/?id=000019587#SLE15SP4>`_, 15 SP6, 6.4.0
+   ,15 SP5, 5.14.21
+   ,15 SP4, 5.14.21
+   ,,
+   `Oracle Linux <https://blogs.oracle.com/scoter/post/oracle-linux-and-unbreakable-enterprise-kernel-uek-releases>`_, 8.9, 5.15.0
+ 
 
 ..
    Footnotes and ref anchors in below historical tables should be appended with "-past-60", to differentiate from the 
@@ -157,20 +205,20 @@ Expand for full historical view of:
    You can `download the entire .csv <../downloads/compatibility-matrix-historical-6.0.csv>`_ for offline reference.
 
    .. csv-table::
-      :file: ../data/reference/compatibility-matrix-historical-6.0.csv
-      :widths: 20,10,10,10,10,10,10
+      :file: compatibility-matrix-historical-6.0.csv
+      :widths: 20,10,10,10,10,10,10,10
       :header-rows: 1
       :stub-columns: 1
    
    .. rubric:: Footnotes
 
-   .. [#Ubuntu220405-past-60] Preview support of Ubuntu 22.04.5 only
    .. [#red-hat94-past-60] RHEL 9.4 is supported only on AMD Instinct MI300A.
    .. [#oracle89-past-60] Oracle Linux is supported only on AMD Instinct MI300X.
+   .. [#mi300_621-past-60] **For ROCm 6.2.1** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
    .. [#mi300_620-past-60] **For ROCm 6.2.0** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
    .. [#mi300_612-past-60] **For ROCm 6.1.2** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4 and Oracle Linux.
    .. [#mi300_611-past-60] **For ROCm 6.1.1** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4 and Oracle Linux.
    .. [#mi300_610-past-60] **For ROCm 6.1.0** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4.
    .. [#mi300_602-past-60] **For ROCm 6.0.2** - MI300A (gfx942) is supported on Ubuntu 22.04.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.3.
    .. [#mi300_600-past-60] **For ROCm 6.0.0** - MI300A (gfx942) is supported on Ubuntu 22.04.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.3.
-
+   .. [#kfd_support-past-60] ROCm provides forward and backward compatibility between the Kernel Fusion Driver (KFD) and its user space software for +/- 2 releases. These are the compatibility combinations that are currently supported.

docs/conceptual/gpu-arch/mi250.md

@@ -33,8 +33,8 @@ Units (CU). The MI250 GCD has 104 active CUs. Each compute unit is further
 subdivided into four SIMD units that process SIMD instructions of 16 data
 elements per instruction (for the FP64 data type). This enables the CU to
 process 64 work items (a so-called “wavefront”) at a peak clock frequency of 1.7
-GHz. Therefore, the theoretical maximum FP64 peak performance per GCD is 45.3
-TFLOPS for vector instructions. The MI250 compute units also provide specialized
+GHz. Therefore, the theoretical maximum FP64 peak performance per GCD is 22.6
+TFLOPS for vector instructions. This equates to 45.3 TFLOPS for vector instructions for both GCDs together. The MI250 compute units also provide specialized
 execution units (also called matrix cores), which are geared toward executing
 matrix operations like matrix-matrix multiplications. For FP64, the peak
 performance of these units amounts to 90.5 TFLOPS.

docs/conf.py

@@ -10,7 +10,7 @@ import shutil
 shutil.copy2("../RELEASE.md", "./about/release-notes.md")
 
 os.system("mkdir -p ../_readthedocs/html/downloads")
-os.system("cp data/reference/compatibility-matrix-historical-6.0.csv ../_readthedocs/html/downloads/compatibility-matrix-historical-6.0.csv")
+os.system("cp compatibility/compatibility-matrix-historical-6.0.csv ../_readthedocs/html/downloads/compatibility-matrix-historical-6.0.csv")
 
 latex_engine = "xelatex"
 latex_elements = {
@@ -30,16 +30,16 @@ if os.environ.get("READTHEDOCS", "") == "True":
 project = "ROCm Documentation"
 author = "Advanced Micro Devices, Inc."
 copyright = "Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved."
-version = "6.2.0"
-release = "6.2.0"
+version = "6.2.1"
+release = "6.2.1"
 setting_all_article_info = True
 all_article_info_os = ["linux", "windows"]
 all_article_info_author = ""
 
 # pages with specific settings
 article_pages = [
-    {"file": "about/release-notes", "os": ["linux", "windows"], "date": "2024-08-02"},
-    {"file": "about/changelog", "os": ["linux", "windows"], "date": "2024-08-02"},
+    {"file": "about/release-notes", "os": ["linux", "windows"], "date": "2024-09-20"},
+    {"file": "about/changelog", "os": ["linux", "windows"], "date": "2024-09-20"},
     {"file": "how-to/deep-learning-rocm", "os": ["linux"]},
     {"file": "how-to/rocm-for-ai/index", "os": ["linux"]},
     {"file": "how-to/rocm-for-ai/install", "os": ["linux"]},

docs/contribute/feedback.md

@@ -6,26 +6,10 @@
 
 # Providing feedback
 
-There are four standard ways to provide feedback on this repository.
+Your feedback is welcome. You can provide feedback either through GitHub Discussions or GitHub Issues.
 
-## Pull request
+Use [GitHub Discussions](https://github.com/ROCm/ROCm/discussions) to ask questions, view announcements, and communicate with other members of the community.
 
-All contributions to ROCm documentation should arrive via the
-[GitHub Flow](https://docs.github.com/en/get-started/quickstart/github-flow)
-targeting the develop branch of the repository.
+Use [GitHub Issues](https://github.com/ROCm/ROCm/issues) to submit issues you find with ROCm or with the ROCm documentation.
 
-For more in-depth information on creating a pull request (PR), see
-[Contributing](./contributing.md).
-
-## GitHub discussions
-
-To ask questions or view answers to frequently asked questions, refer to
-[GitHub Discussions](https://github.com/ROCm/ROCm/discussions).
-On GitHub Discussions, in addition to asking and answering questions,
-members can share updates, have open-ended conversations,
-and follow along on via public announcements.
-
-## GitHub issue
-
-Issues on existing or absent documentation can be filed in
-[GitHub Issues](https://github.com/ROCm/ROCm/issues).
+For information about contributing to the ROCm repository and creating a pull request (PR), see [Contributing](./contributing.md).

docs/data/reference/compatibility-matrix-historical-6.0.csv

@@ -1,111 +0,0 @@
-ROCm Version,6.2.0, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
-      :doc:`Operating Systems <rocm-install-on-linux:reference/system-requirements>`,Ubuntu 24.04,,,,,
-      ,"Ubuntu 22.04.5 [#Ubuntu220405-past-60]_, 22.04.4","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3"
-      ,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
-      ,"RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
-      ,"RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
-      ,"SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
-      ,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
-      ,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,,,
-      ,".. _architecture-support-compatibility-matrix-past-60:",,,,,
-      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
-      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
-      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
-      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
-      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
-      ,".. _gpu-support-compatibility-matrix-past-60:",,,,,
-      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
-      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
-      ,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
-      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
-      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
-      ,,,,,,
-      FRAMEWORK SUPPORT,".. _framework-support-compatibility-matrix-past-60:",,,,,
-      :doc:`PyTorch <rocm-install-on-linux:how-to/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <rocm-install-on-linux:how-to/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.15, 2.14, 2.13","2.15, 2.14, 2.13","2.15, 2.14, 2.13","2.14, 2.13, 2.12","2.14, 2.13, 2.12"
-      :doc:`JAX <rocm-install-on-linux:how-to/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
-      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
-      ,,,,,,
-      THIRD PARTY COMMS,".. _thirdpartycomms-support-compatibility-matrix-past-60:",,,,,
-      `UCC <https://github.com/ROCm/ucc>`_,>=1.2.0,>=1.2.0,>=1.2.0,>=1.2.0,>=1.2.0,>=1.2.0
-      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
-      ,,,,,,
-      THIRD PARTY ALGORITHM,".. _thirdpartyalgorithm-support-compatibility-matrix-past-60:",,,,,
-      Thrust,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-      CUB,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-      ,,,,,,
-      ML & COMPUTER VISION,".. _mllibs-support-compatibility-matrix-past-60:",,,,,
-      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
-      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
-      :doc:`MIOpen <miopen:index>`,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`MIVisionX <mivisionx:index>`,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
-      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
-      :doc:`RPP <rpp:index>`,1.8.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
-      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,N/A,N/A,N/A,N/A,N/A
-      ,,,,,,
-      COMMUNICATION,".. _commlibs-support-compatibility-matrix-past-60:",,,,,
-      :doc:`RCCL <rccl:index>`,2.20.5,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
-      ,,,,,,
-      MATH LIBS,".. _mathlibs-support-compatibility-matrix-past-60:",,,,,
-      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
-      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
-      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
-      :doc:`hipFFT <hipfft:index>`,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
-      :doc:`hipFORT <hipfort:index>`,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
-      :doc:`hipRAND <hiprand:index>`,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
-      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
-      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
-      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
-      :doc:`rocBLAS <rocblas:index>`,4.2.0,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
-      :doc:`rocFFT <rocfft:index>`,1.0.28,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
-      :doc:`rocRAND <rocrand:index>`,3.1.0,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
-      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
-      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
-      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
-      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
-      ,,,,,,
-      PRIMITIVES,".. _primitivelibs-support-compatibility-matrix-past-60:",,,,,
-      :doc:`hipCUB <hipcub:index>`,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
-      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`rocThrust <rocthrust:index>`,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      ,,,,,,
-      SUPPORT LIBS,,,,,,
-      `hipother <https://github.com/ROCm/hipother>`_,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.0,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
-      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
-      ,,,,,,
-      SYSTEM MGMT TOOLS,".. _tools-support-compatibility-matrix-past-60:",,,,,
-      :doc:`AMD SMI <amdsmi:index>`,24.6.2,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
-      :doc:`ROCm Data Center Tool <rdc:index>`,1.0.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
-      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
-      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.0,rocm-6.1.2,rocm-6.1.1,rocm-6.1.0,rocm-6.0.2,rocm-6.0.0
-      ,,,,,,
-      PERFORMANCE TOOLS,,,,,,
-      :doc:`Omniperf <omniperf:index>`,2.0.1,N/A,N/A,N/A,N/A,N/A
-      :doc:`Omnitrace <omnitrace:index>`,1.11.2,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
-      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60200,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
-      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCTracer <roctracer:index>`,4.1.60200,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
-      ,,,,,,
-      DEVELOPMENT TOOLS,,,,,,
-      :doc:`HIPIFY <hipify:index>`,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
-      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
-      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
-      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.3.0,0.3.0,0.3.0,N/A,N/A
-      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
-      ,,,,,,
-      COMPILERS,".. _compilers-support-compatibility-matrix-past-60:",,,,,
-      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
-      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      `llvm-project <https://github.com/ROCm/llvm-project>`_,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      ,,,,,,
-      RUNTIMES,".. _runtime-support-compatibility-matrix-past-60:",,,,,
-      :doc:`HIP <hip:index>`,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
-      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0

docs/how-to/llm-fine-tuning-optimization/llm-inference-frameworks.rst

@@ -137,6 +137,12 @@ Installing vLLM
 
 Refer to :ref:`mi300x-vllm-optimization` for performance optimization tips.
 
+ROCm provides a prebuilt optimized Docker image for validating the performance of LLM inference with vLLM 
+on the MI300X accelerator. The Docker image includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 .. _fine-tuning-llms-tgi:
 
 Hugging Face TGI

docs/how-to/llm-fine-tuning-optimization/model-acceleration-libraries.rst

@@ -251,3 +251,287 @@ page describes the options.
 
 Learn more about optimizing kernels with TunableOp in
 :ref:`Optimizing Triton kernels <mi300x-tunableop>`.
+
+
+FBGEMM and FBGEMM_GPU
+=====================
+
+FBGEMM (Facebook General Matrix Multiplication) is a low-precision, high-performance CPU kernel library
+for matrix-matrix multiplications and convolutions. It is used for server-side inference
+and as a back end for PyTorch quantized operators. FBGEMM offers optimized on-CPU performance for reduced precision calculations,
+strong performance on native tensor formats, and the ability to generate
+high-performance shape- and size-specific kernels at runtime.
+
+FBGEMM_GPU collects several high-performance PyTorch GPU operator libraries  
+for use in training and inference. It provides efficient table-batched embedding functionality,
+data layout transformation, and quantization support.
+
+For more information about FBGEMM and FBGEMM_GPU, see the `PyTorch FBGEMM GitHub <https://github.com/pytorch/FBGEMM>`_
+and the `PyTorch FBGEMM documentation <https://pytorch.org/FBGEMM/>`_.
+The `Meta blog post about FBGEMM <https://engineering.fb.com/2018/11/07/ml-applications/fbgemm/>`_ provides
+additional background about the library.
+
+Installing FBGEMM_GPU
+----------------------
+
+Installing FBGEMM_GPU consists of the following steps:
+
+*  Set up an isolated Miniconda environment
+*  Install ROCm using Docker or the :doc:`package manager <rocm-install-on-linux:install/native-install/index>`
+*  Install the nightly `PyTorch <https://pytorch.org/>`_ build
+*  Complete the pre-build and build tasks
+  
+.. note::
+
+   FBGEMM_GPU doesn't require the installation of FBGEMM. To optionally install
+   FBGEMM, see the `FBGEMM install instructions <https://pytorch.org/FBGEMM/fbgemm-development/BuildInstructions.html>`_.
+
+Set up the Miniconda environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To install Miniconda, use the following commands.
+
+#. Install a `Miniconda environment <https://docs.anaconda.com/miniconda/>`_ for reproducible builds.
+   All subsequent commands run inside this environment.
+
+   .. code-block:: shell
+
+      export PLATFORM_NAME="$(uname -s)-$(uname -m)"
+
+      # Set the Miniconda prefix directory
+      miniconda_prefix=$HOME/miniconda
+
+      # Download the Miniconda installer
+      wget -q "https://repo.anaconda.com/miniconda/Miniconda3-latest-${PLATFORM_NAME}.sh" -O miniconda.sh
+
+      # Run the installer
+      bash miniconda.sh -b -p "$miniconda_prefix" -u
+
+      # Load the shortcuts
+      . ~/.bashrc
+
+      # Run updates
+      conda update -n base -c defaults -y conda
+
+#. Create a Miniconda environment with Python 3.12:
+
+   .. code-block:: shell
+
+      env_name=<ENV NAME>
+      python_version=3.12
+
+      # Create the environment
+      conda create -y --name ${env_name} python="${python_version}"
+
+      # Upgrade PIP and pyOpenSSL package
+      conda run -n ${env_name} pip install --upgrade pip
+      conda run -n ${env_name} python -m pip install pyOpenSSL>22.1.0
+
+#. Install additional build tools:
+
+   .. code-block:: shell
+
+      conda install -n ${env_name} -y \
+         click \
+         cmake \
+         hypothesis \
+         jinja2 \
+         make \
+         ncurses \
+         ninja \
+         numpy \
+         scikit-build \
+         wheel
+
+Install the ROCm components
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+FBGEMM_GPU can run in a ROCm Docker container or in conjunction with the full ROCm installation.
+The Docker method is recommended because it requires fewer steps and provides a stable environment.
+
+To run FBGEMM_GPU in the Docker container, pull the `Minimal Docker image for ROCm <https://hub.docker.com/r/rocm/rocm-terminal>`_.
+This image includes all preinstalled ROCm packages required to integrate FBGEMM. To pull
+and run the ROCm Docker image, use this command:
+
+.. code-block:: shell
+
+   # Run for ROCm 6.2.0
+   docker run -it --network=host --shm-size 16G --device=/dev/kfd --device=/dev/dri --group-add video \
+   --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --ipc=host rocm/rocm-terminal:6.2 /bin/bash
+
+.. note::
+
+   The `Full Docker image for ROCm <https://hub.docker.com/r/rocm/dev-ubuntu-20.04>`_, which includes all
+   ROCm packages, can also be used. However, it results in a very large container, so the minimal
+   Docker image is recommended.
+
+You can also install ROCm using the package manager. FBGEMM_GPU requires the installation of the full ROCm package.
+For more information, see :doc:`the ROCm installation guide <rocm-install-on-linux:install/detailed-install>`.
+The ROCm package also requires the :doc:`MIOpen <miopen:index>` component as a dependency. 
+To install MIOpen, use the ``apt install`` command.
+
+.. code-block:: shell
+
+   apt install hipify-clang miopen-hip miopen-hip-dev
+
+Install PyTorch
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Install `PyTorch <https://pytorch.org/>`_ using ``pip`` for the most reliable and consistent results.
+
+#. Install the nightly PyTorch build using ``pip``.
+
+   .. code-block:: shell
+
+      # Install the latest nightly, ROCm variant
+      conda run -n ${env_name} pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/rocm6.2/
+
+#. Ensure PyTorch loads correctly. Verify the version and variant of the installation using an ``import`` test.
+
+   .. code-block:: shell
+
+      # Ensure that the package loads properly
+      conda run -n ${env_name} python -c "import torch.distributed"
+
+      # Verify the version and variant of the installation
+      conda run -n ${env_name} python -c "import torch; print(torch.__version__)"
+
+Perform the prebuild and build
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+#. Clone the FBGEMM repository and the relevant submodules. Use ``pip`` to install the 
+   components in ``requirements.txt``. Run the following commands inside the Miniconda environment.
+
+   .. code-block:: shell
+
+      # Select a version tag
+      FBGEMM_VERSION=v0.8.0
+
+      # Clone the repo along with its submodules
+      git clone https://github.com/pytorch/FBGEMM.git --branch=v0.8.0 --recursive fbgemm_${FBGEMM_VERSION}
+
+      # Install additional required packages for building and testing
+      cd fbgemm_${FBGEMM_VERSION}/fbgemm_gpu
+      pip install requirements.txt
+
+#. Clear the build cache to remove stale build information.
+
+   .. code-block:: shell
+
+      # !! Run in fbgemm_gpu/ directory inside the Conda environment !!
+
+      python setup.py clean
+
+#. Set the wheel build variables, including the package name, Python version tag, and Python platform name.
+
+   .. code-block:: shell
+
+      # Set the package name depending on the build variant
+      export package_name=fbgemm_gpu_rocm
+
+      # Set the Python version tag.  It should follow the convention `py<major><minor>`,
+      # for example, Python 3.12 --> py312
+      export python_tag=py312
+
+      # Determine the processor architecture
+      export ARCH=$(uname -m)
+
+      # Set the Python platform name for the Linux case
+      export python_plat_name="manylinux2014_${ARCH}"
+
+#. Build FBGEMM_GPU for the ROCm platform. Set ``ROCM_PATH`` to the path to your ROCm installation. 
+   Run these commands from the ``fbgemm_gpu/`` directory inside the Miniconda environment.
+
+   .. code-block:: shell
+
+      # !! Run in the fbgemm_gpu/ directory inside the Conda environment !!
+
+      export ROCM_PATH=</path/to/rocm>
+
+      # Build for the target architecture of the ROCm device installed on the machine (for example, 'gfx942;gfx90a')
+      # See :doc:`The Linux system requirements <../../reference/system-requirements>` for a list of supported GPUs.
+      export PYTORCH_ROCM_ARCH=$(${ROCM_PATH}/bin/rocminfo | grep -o -m 1 'gfx.*')
+
+      # Build the wheel artifact only
+      python setup.py bdist_wheel \
+         --package_variant=rocm \
+         --python-tag="${python_tag}" \
+         --plat-name="${python_plat_name}" \
+         -DHIP_ROOT_DIR="${ROCM_PATH}" \
+         -DCMAKE_C_FLAGS="-DTORCH_USE_HIP_DSA" \
+         -DCMAKE_CXX_FLAGS="-DTORCH_USE_HIP_DSA"
+
+      # Build and install the library into the Conda environment
+      python setup.py install \
+         --package_variant=rocm \
+         -DHIP_ROOT_DIR="${ROCM_PATH}" \
+         -DCMAKE_C_FLAGS="-DTORCH_USE_HIP_DSA" \
+         -DCMAKE_CXX_FLAGS="-DTORCH_USE_HIP_DSA"  
+
+Post-build validation
+----------------------
+
+After building FBGEMM_GPU, run some verification checks to ensure the build is correct. Continue
+to run all commands inside the ``fbgemm_gpu/`` directory inside the Miniconda environment.
+
+#. The build process generates many build artifacts and C++ templates, so
+   it is important to confirm no undefined symbols remain.
+
+   .. code-block:: shell
+
+      # !! Run in fbgemm_gpu/ directory inside the Conda environment !!
+
+      # Locate the built .SO file
+      fbgemm_gpu_lib_path=$(find . -name fbgemm_gpu_py.so)
+
+      # Check that the undefined symbols don't include fbgemm_gpu-defined functions
+      nm -gDCu "${fbgemm_gpu_lib_path}" | sort
+
+#. Verify the referenced version number of ``GLIBCXX`` and the presence of certain function symbols:
+
+   .. code-block:: shell
+
+      # !! Run in fbgemm_gpu/ directory inside the Conda environment !!
+
+      # Locate the built .SO file
+      fbgemm_gpu_lib_path=$(find . -name fbgemm_gpu_py.so)
+
+      # Note the versions of GLIBCXX referenced by the .SO
+      # The libstdc++.so.6 available on the install target must support these versions
+      objdump -TC "${fbgemm_gpu_lib_path}" | grep GLIBCXX | sed 's/.*GLIBCXX_\([.0-9]*\).*/GLIBCXX_\1/g' | sort -Vu | cat
+
+      # Test for the existence of a given function symbol in the .SO
+      nm -gDC "${fbgemm_gpu_lib_path}" | grep " fbgemm_gpu::merge_pooled_embeddings("
+      nm -gDC "${fbgemm_gpu_lib_path}" | grep " fbgemm_gpu::jagged_2d_to_dense("
+
+Testing FBGEMM
+----------------------
+
+FBGEMM includes tests and benchmarks to validate performance. To run these tests,
+you must use ROCm 5.7 or a more recent version on the host and container. To run FBGEMM tests,
+follow these instructions:
+
+.. code-block:: shell
+
+   # !! Run inside the Conda environment !!
+
+   # From the /fbgemm_gpu/ directory
+   cd test
+
+   export FBGEMM_TEST_WITH_ROCM=1
+   # Enable for debugging failed kernel executions
+   export HIP_LAUNCH_BLOCKING=1
+
+   # Run the test
+   python -m pytest -v -rsx -s -W ignore::pytest.PytestCollectionWarning split_table_batched_embeddings_test.py
+
+To run the FBGEMM_GPU ``uvm`` test, use these commands. These tests only support the AMD MI210 and 
+more recent accelerators. 
+
+.. code-block:: shell
+
+   # Run this inside the Conda environment from the /fbgemm_gpu/ directory
+   export HSA_XNACK=1
+   cd test
+
+   python -m pytest -v -rsx -s -W ignore::pytest.PytestCollectionWarning ./uvm/uvm_test.py

docs/how-to/llm-fine-tuning-optimization/model-quantization.rst

@@ -189,6 +189,9 @@ Installing bitsandbytes
       # Use -DBNB_ROCM_ARCH to specify target GPU arch
       cmake -DBNB_ROCM_ARCH="gfx942" -DCOMPUTE_BACKEND=hip -S .
 
+      # Compile the project
+      make
+
       # Install 
       python setup.py install
 
@@ -219,12 +222,11 @@ To get started with bitsandbytes primitives, use the following code as reference
 Using bitsandbytes with Hugging Face Transformers
 -------------------------------------------------
 
-To load a Transformers model in 4-bit, set ``load_int_4bt=true`` in ``BitsAndBytesConfig``.
+To load a Transformers model in 4-bit, set ``load_in_4bit=true`` in ``BitsAndBytesConfig``.
 
 .. code-block:: python
 
-   from transformers import AutoModelForCausalLM
-   from bitsandbytes import BitsAndBytesConfig
+   from transformers import AutoModelForCausalLM, BitsAndBytesConfig
    
    base_model_name = "NousResearch/Llama-2-7b-hf"
    quantization_config = BitsAndBytesConfig(load_in_4bit=True)
@@ -240,8 +242,7 @@ To load a model in 8-bit for inference, use the ``load_in_8bit`` option.
 
 .. code-block:: python
 
-   from transformers import AutoModelForCausalLM, AutoTokenizer
-   from bitsandbytes import BitsAndBytesConfig
+   from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
    
    base_model_name = "NousResearch/Llama-2-7b-hf"
    

docs/how-to/rocm-for-ai/deploy-your-model.rst

@@ -41,6 +41,15 @@ vLLM walkthrough
 Refer to this developer blog for guidance on serving with vLLM `Inferencing and serving with vLLM on AMD GPUs — ROCm
 Blogs <https://rocm.blogs.amd.com/artificial-intelligence/vllm/README.html>`_
 
+Validating vLLM performance
+---------------------------
+
+ROCm provides a prebuilt optimized Docker image for validating the performance of LLM inference with vLLM 
+on the MI300X accelerator. The Docker image includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 .. _rocm-for-ai-serve-hugging-face-tgi:
 
 Serving using Hugging Face TGI

docs/how-to/rocm-for-ai/install.rst

@@ -22,25 +22,25 @@ If you’re new to ROCm, refer to the :doc:`ROCm quick start install guide for L
 <rocm-install-on-linux:install/quick-start>`.
 
 If you’re using a Radeon GPU for graphics-accelerated applications, refer to the
-:doc:`Radeon installation instructions <radeon:docs/install/install-radeon>`.
+`Radeon installation instructions <https://rocm.docs.amd.com/projects/radeon/en/docs-6.1.3/docs/install/native_linux/install-radeon.html>`_.
 
-ROCm supports two methods for installation. There is no difference in the final ROCm installation between these two
-methods. You can also opt for :ref:`single-version or multi-version installation
-<rocm-install-on-linux:installation-types>`.
+ROCm supports multiple :doc:`installation methods <rocm-install-on-linux:install/install-overview>`:
 
-*  :doc:`Using your Linux distribution's package manager <rocm-install-on-linux:how-to/native-install/index>`
+* :doc:`Using your Linux distribution's package manager <rocm-install-on-linux:install/native-install/index>`
 
-*  :doc:`Using the AMDGPU installer <rocm-install-on-linux:how-to/amdgpu-install>`
+* :doc:`Using the AMDGPU installer <rocm-install-on-linux:install/amdgpu-install>`
+
+* :ref:`Multi-version installation <rocm-install-on-linux:installation-types>`.
 
 .. grid:: 1
 
    .. grid-item-card:: Post-install
 
-      Follow the :doc:`post-installation instructions <rocm-install-on-linux:how-to/native-install/post-install>` to
+      Follow the :doc:`post-installation instructions <rocm-install-on-linux:install/post-install>` to
       configure your system linker, PATH, and verify the installation.
 
       If you encounter any issues during installation, refer to the
-      :doc:`Installation troubleshooting <rocm-install-on-linux:how-to/native-install/install-faq>` guide.
+      :doc:`Installation troubleshooting <rocm-install-on-linux:reference/install-faq>` guide.
 
 Machine learning frameworks
 ===========================

docs/how-to/system-optimization/mi100.md

@@ -365,9 +365,9 @@ installed.
 ## System management
 
 For a complete guide on how to install/manage/uninstall ROCm on Linux, refer to
-{doc}`Quick-start (Linux)<rocm-install-on-linux:tutorial/quick-start>`. To verify that the installation was
+{doc}`Quick-start (Linux)<rocm-install-on-linux:install/quick-start>`. To verify that the installation was
 successful, refer to the
-{doc}`post-install instructions<rocm-install-on-linux:how-to/native-install/post-install>` and
+{doc}`post-install instructions<rocm-install-on-linux:install/post-install>` and
 [system tools](../../reference/rocm-tools.md). Should verification
 fail, consult the [System Debugging Guide](../system-debugging.md).
 

docs/how-to/system-optimization/mi200.md

@@ -350,9 +350,9 @@ installed.
 ## System management
 
 For a complete guide on how to install/manage/uninstall ROCm on Linux, refer to
-{doc}`Quick-start (Linux)<rocm-install-on-linux:tutorial/quick-start>`. For verifying that the
+{doc}`Quick-start (Linux)<rocm-install-on-linux:install/quick-start>`. For verifying that the
 installation was successful, refer to the
-{doc}`post-install instructions<rocm-install-on-linux:how-to/native-install/post-install>` and
+{doc}`post-install instructions<rocm-install-on-linux:install/post-install>` and
 [system tools](../../reference/rocm-tools.md). Should verification
 fail, consult the [System Debugging Guide](../system-debugging.md).
 

docs/how-to/system-optimization/mi300a.rst

@@ -122,6 +122,62 @@ This section describes performance-based settings.
 
        transparent_hugepage=always
 
+* **Increase the amount of allocatable memory**
+
+  By default, when using a device allocator via HIP, it is only possible to allocate 96 GiB out of 
+  a possible 128 GiB of memory on the MI300A. This limitation does not affect host allocations.
+  To increase the available system memory, load the ``amdttm`` module with new values for
+  ``pages_limit`` and ``page_pool_size``. These numbers correspond to the number of 4 KiB pages of memory.
+  To make 128 GiB of memory available across all four devices, for a total amount of 512 GiB,
+  set ``pages_limit`` and ``page_pool_size`` to ``134217728``. For a two-socket system, divide these values
+  by two. After setting these values, reload the AMDGPU driver.
+
+  First, review the current settings using this shell command:
+
+  .. code-block:: shell
+
+     cat /sys/module/amdttm/parameters/pages_limit 
+  
+  To set the amount of allocatable memory to all available memory on all four APU devices, run these commands:
+
+  .. code-block:: shell
+
+     sudo modprobe amdttm pages_limit=134217728 page_pool_size=134217728
+     sudo modprobe amdgpu
+
+  These settings can also be hardcoded in the ``/etc/modprobe.d/amdttm.conf`` file or specified as boot
+  parameters.
+  
+  To use the hardcoded method, 
+  the filesystem must already be set up when the kernel driver is loaded.
+  To hardcode the settings, add the following lines to ``/etc/modprobe.d/amdttm.conf``:
+
+  .. code-block:: shell
+
+     options amdttm pages_limit=134217728
+     options amdttm page_pool_size=134217728
+
+  If the filesystem is not already set up when the kernel driver is loaded, then the options
+  must be specified as boot parameters. To specify the settings
+  as boot parameters when loading the kernel, use this example as a guideline:
+
+  .. code-block:: shell
+
+     vmlinux-[...] amdttm.pages_limit=134217728 amdttm.page_pool_size=134217728 [...]
+
+  To verify the new settings and confirm the change, use this command:
+
+  .. code-block:: shell
+
+     cat /sys/module/amdttm/parameters/pages_limit 
+
+  .. note::
+
+     The system settings for ``pages_limit`` and ``page_pool_size`` are calculated by multiplying the
+     per-APU limit of 4 KiB pages, which is ``33554432``, by the number of APUs on the node. The limit for a system with
+     two APUs ``33554432 x 2`` or ``67108864``.
+     This means the ``modprobe`` command for two APUs is ``sudo modprobe amdttm pages_limit=67108864 page_pool_size=67108864``.
+
 * **Limit the maximum and single memory allocations on the GPU**
   
   Many AI-related applications were originally developed on discrete GPUs. Some of these applications 
@@ -195,10 +251,13 @@ This section describes performance-based settings.
 
 * **Change affinity of ROCm helper threads**
   
-  This change prevents internal ROCm threads from having their CPU core affinity mask 
+  Changing the affinity prevents internal ROCm threads from having their CPU core affinity mask 
   set to all CPU cores available. With this setting, the threads inherit their parent's 
-  CPU core affinity mask. If you have any questions regarding this setting, 
-  contact your MI300A platform vendor. To enable this setting, enter the following command:
+  CPU core affinity mask. Before adjusting this setting, ensure you thoroughly understand 
+  your system topology and how the application, runtime environment, and batch system
+  set the thread-to-core affinity. If you have any questions regarding this setting, 
+  contact your MI300A platform vendor or the AMD support team. 
+  To enable this setting, enter the following command:
 
   .. code-block:: shell
 
@@ -213,9 +272,9 @@ System management
 ========================================
 
 For a complete guide on installing, managing, and uninstalling ROCm on Linux, see
-:doc:`Quick-start (Linux)<rocm-install-on-linux:tutorial/quick-start>`. To verify that the
+:doc:`Quick-start (Linux)<rocm-install-on-linux:install/quick-start>`. To verify that the
 installation was successful, see the
-:doc:`Post-installation instructions<rocm-install-on-linux:install/native-install/post-install>` and 
+:doc:`Post-installation instructions<rocm-install-on-linux:install/post-install>` and 
 :doc:`ROCm tools <../../reference/rocm-tools>` guides. If verification
 fails, consult the :doc:`System debugging guide <../system-debugging>`.
 

docs/how-to/system-optimization/mi300x.rst

@@ -534,7 +534,7 @@ optimizing user applications.
 For a complete guide on how to install, manage, or uninstall ROCm on Linux, refer to
 :doc:`rocm-install-on-linux:install/quick-start`. For verifying that the
 installation was successful, refer to the
-:doc:`rocm-install-on-linux:install/native-install/post-install`.
+:doc:`rocm-install-on-linux:install/post-install`.
 Should verification fail, consult :doc:`/how-to/system-debugging`.
 
 Hardware verification with ROCm

docs/how-to/tuning-guides/mi300x/workload.rst

@@ -150,6 +150,12 @@ the workload to validate improvements and ensure that the changes have had the
 desired effect. Continuous iteration helps refine the performance gains and
 address any new bottlenecks that may emerge.
 
+ROCm provides a prebuilt optimized Docker image that has everything required to implement
+the tips in this section. It includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 .. _mi300x-profiling-tools:
 
 Profiling tools
@@ -372,6 +378,12 @@ Refer to `vLLM documentation <https://docs.vllm.ai/en/latest/models/performance.
 for additional performance tips. :ref:`fine-tuning-llms-vllm` describes vLLM
 usage with ROCm.
 
+ROCm provides a prebuilt optimized Docker image for validating the performance of LLM inference with vLLM 
+on the MI300X accelerator. The Docker image includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 Maximize throughput
 -------------------
 

docs/index.md

@@ -23,57 +23,40 @@ Installation instructions are available from:
 ROCm documentation is organized into the following categories:
 
 ::::{grid} 1 2 2 2
+:gutter: 3
 :class-container: rocm-doc-grid
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ./data/banner-compatibility.jpg
-:img-alt: Compatibility information
-:padding: 2
+:::{grid-item-card} Compatibility
+:class-body: rocm-card-banner rocm-hue-2
 
 * [Compatibility matrix](./compatibility/compatibility-matrix.rst)
 * {doc}`Linux system requirements<rocm-install-on-linux:reference/system-requirements>`
 * {doc}`Windows system requirements<rocm-install-on-windows:reference/system-requirements>`
 * {doc}`Third-party support<rocm-install-on-linux:reference/3rd-party-support-matrix>`
-* {doc}`User/kernel space<rocm-install-on-linux:reference/user-kernel-space-compat-matrix>`
-* {doc}`Docker<rocm-install-on-linux:reference/docker-image-support-matrix>`
-* [OpenMP](./about/compatibility/openmp.md)
-* [Precision support](./compatibility/precision-support.rst)
-* {doc}`ROCm on Radeon GPUs<radeon:index>`
+* {doc}`User and kernel-space support matrix<rocm-install-on-linux:reference/user-kernel-space-compat-matrix>`
+* {doc}`Docker image support matrix<rocm-install-on-linux:reference/docker-image-support-matrix>`
+* {doc}`Use ROCm on Radeon GPUs<radeon:index>`
 :::
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ./data/banner-howto.jpg
-:img-alt: How-to documentation
-:padding: 2
+:::{grid-item-card} How to
+:class-body: rocm-card-banner rocm-hue-12
 
 * [Using ROCm for AI](./how-to/rocm-for-ai/index.rst)
 * [Using ROCm for HPC](./how-to/rocm-for-hpc/index.rst)
 * [Fine-tuning LLMs and inference optimization](./how-to/llm-fine-tuning-optimization/index.rst)
 * [System optimization](./how-to/system-optimization/index.rst)
-  * [AMD Instinct MI300X](./how-to/system-optimization/mi300x.rst)
-  * [AMD Instinct MI300A](./how-to/system-optimization/mi300a.rst)
-  * [AMD Instinct MI200](./how-to/system-optimization/mi200.md)
-  * [AMD Instinct MI100](./how-to/system-optimization/mi100.md)
-  * [AMD Instinct RDNA2](./how-to/system-optimization/w6000-v620.md)
 * [AMD Instinct MI300X tuning guides](./how-to/tuning-guides/mi300x/index.rst)
-  * [System tuning](./how-to/tuning-guides/mi300x/system.rst)
-  * [Workload tuning](./how-to/tuning-guides/mi300x/workload.rst)
 * [System debugging](./how-to/system-debugging.md)
-* [GPU-enabled MPI](./how-to/gpu-enabled-mpi.rst)
+* [Using MPI](./how-to/gpu-enabled-mpi.rst)
 * [Using advanced compiler features](./conceptual/compiler-topics.md)
 * [Setting the number of CUs](./how-to/setting-cus)  
-* [GitHub examples](https://github.com/amd/rocm-examples)
+* [ROCm examples](https://github.com/amd/rocm-examples)
 :::
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ./data/banner-conceptual.jpg
-:img-alt: Conceptual documentation
-:padding: 2
+:::{grid-item-card} Conceptual
+:class-body: rocm-card-banner rocm-hue-8
 
-* [GPU architecture](./conceptual/gpu-arch.md)
+* [GPU architecture overview](./conceptual/gpu-arch.md)
 * [GPU memory](./conceptual/gpu-memory.md)
 * [File structure (Linux FHS)](./conceptual/file-reorg.md)
 * [GPU isolation techniques](./conceptual/gpu-isolation.md)
@@ -84,21 +67,13 @@ ROCm documentation is organized into the following categories:
 :::
 
 <!-- markdownlint-disable MD051 -->
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ./data/banner-reference.jpg
-:img-alt: Reference documentation
-:padding: 2
+:::{grid-item-card} Reference
+:class-body: rocm-card-banner rocm-hue-6
 
-* [Libraries](./reference/api-libraries.md)
-  * [Artificial intelligence](#artificial-intelligence-apis)
-  * [C++ primitives](#cpp-primitives)
-  * [Communication](#communication-libraries)
-  * [Math](#math-apis)
-  * [Random number generators](#random-number-apis)
-  * [HIP runtime](#hip-runtime)
-* [ROCm tools and compilers](./reference/rocm-tools.md)
-* [GPU hardware specifications](./reference/gpu-arch-specs.rst)
+* [ROCm libraries](./reference/api-libraries.md)
+* [ROCm tools, compilers, and runtimes](./reference/rocm-tools.md)
+* [Accelerator and  GPU hardware specifications](./reference/gpu-arch-specs.rst)
+* [Precision support](./reference/precision-support.rst)
 :::
 <!-- markdownlint-enable MD051 -->
 

docs/reference/api-libraries.md

@@ -9,15 +9,13 @@
 # ROCm libraries
 
 ::::{grid} 1 2 2 2
+:gutter: 3
 :class-container: rocm-doc-grid
 
 (artificial-intelligence-apis)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-ai.jpg
-:img-alt: Artificial intelligence APIs
-:padding: 2
+:::{grid-item-card} Machine Learning and Computer Vision
+:class-body: rocm-card-banner rocm-hue-3
 
 * {doc}`Composable Kernel <composable_kernel:index>`
 * {doc}`MIGraphX <amdmigraphx:index>`
@@ -25,16 +23,14 @@
 * {doc}`MIVisionX <mivisionx:index>`
 * {doc}`rocAL <rocal:index>`
 * {doc}`rocDecode <rocdecode:index>`
+* {doc}`rocPyDecode <rocpydecode:index>`
 * {doc}`ROCm Performance Primitives (RPP) <rpp:index>`
 :::
 
 (cpp-primitives)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-cpp-primitives.jpg
-:img-alt: C++ primitives
-:padding: 2
+:::{grid-item-card} Primitives
+:class-body: rocm-card-banner rocm-hue-12
 
 * {doc}`hipCUB <hipcub:index>`
 * {doc}`hipTensor <hiptensor:index>`
@@ -44,40 +40,23 @@
 
 (communication-libraries)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-communication.jpg
-:img-alt: Communication APIs
-:padding: 2
+:::{grid-item-card} Communication
+:class-body: rocm-card-banner rocm-hue-7
 
 * {doc}`RCCL <rccl:index>`
 :::
 
-(hip-runtime)=
-
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-hip.jpg
-:img-alt: HIP APIs
-:padding: 2
-
-* {doc}`HIP runtime <hip:index>`
-* {doc}`HIPIFY <hipify:index>`
-:::
-
 (math-apis)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-math.jpg
-:img-alt: Math APIs
-:padding: 2
+:::{grid-item-card} Math
+:class-body: rocm-card-banner rocm-hue-6
 
 * [half](https://github.com/ROCm/half)
 * {doc}`hipBLAS <hipblas:index>` / {doc}`rocBLAS <rocblas:index>`
 * {doc}`hipBLASLt <hipblaslt:index>`
 * {doc}`hipFFT <hipfft:index>` / {doc}`rocFFT <rocfft:index>`
 * {doc}`hipfort <hipfort:index>`
+* {doc}`hipRAND <hiprand:index>` / {doc}`rocRAND <rocrand:index>`
 * {doc}`hipSOLVER <hipsolver:index>` / {doc}`rocSOLVER <rocsolver:index>`
 * {doc}`hipSPARSE <hipsparse:index>` / {doc}`rocSPARSE <rocsparse:index>`
 * {doc}`hipSPARSELt <hipsparselt:index>`
@@ -86,16 +65,4 @@
 * [Tensile](https://github.com/ROCm/Tensile)
 :::
 
-(random-number-apis)=
-
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-random-number.jpg
-:img-alt: Random number APIs
-:padding: 2
-
-* {doc}`hipRAND <hiprand:index>`
-* {doc}`rocRAND <rocrand:index>`
-:::
-
 ::::