Update what-is-rocm.rst

Tried out test11, even on a more powerful VM. Test was tracking to take more than an hour.
Settled for test03 for smoke tests.

.azuredevops/ci-builds/aomp.yml

@@ -4,11 +4,21 @@ variables:
 
 resources:
   repositories:
-  - repository: release_repo
+  - repository: aomp_repo
     type: github
     endpoint: ROCm
     name: ROCm/aomp
     ref: aomp-dev
+  - repository: aomp-extras_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp-extras
+    ref: aomp-dev
+  - repository: flang_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/flang
+    ref: aomp-dev
   - repository: llvm-project_repo
     type: github
     endpoint: ROCm
@@ -26,4 +36,4 @@ pr: none
 jobs:
   - template: ${{ variables.CI_COMPONENT_PATH }}/aomp.yml
     parameters:
-      checkoutRepo: release_repo
+      checkoutRepo: aomp_repo

.azuredevops/components/AMDMIGraphX.yml

@@ -11,6 +11,7 @@ parameters:
     - git
     - cmake
     - ninja-build
+    - libdrm-dev
     - libnuma-dev
     - python3-pip
     - python3-venv
@@ -34,17 +35,21 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
-    - clr
-    - rocminfo
-    - rocMLIR
-    - MIOpen
     - aomp
     - aomp-extras
-    - rocBLAS
+    - clr
     - composable_kernel
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - MIOpen
+    - rocm-cmake
+    - ROCR-Runtime
+    - rocBLAS
+    - rocminfo
+    - rocMLIR
+    - rocprofiler-register
 
 jobs:
 - job: AMDMIGraphX
@@ -74,7 +79,9 @@ jobs:
     parameters:
       dependencySource: fixed
       fixedComponentName: half
-      fixedPipelineIdentifier: $(half560-pipeline-id)
+      fixedPipelineIdentifier: ${{ variables.HALF560_PIPELINE_ID }}
+      skipLibraryLinking: true
+      skipLlvmSymlink: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
@@ -92,6 +99,7 @@ jobs:
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
         -DCMAKE_BUILD_TYPE=Release
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
         -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
         -DMIGRAPHX_USE_COMPOSABLEKERNEL=OFF

.azuredevops/components/HIP.yml

@@ -21,7 +21,6 @@ parameters:
   type: object
   default:
     - llvm-project
-    - ROCT-Thunk-Interface
     - ROCR-Runtime
     - rocprofiler-register
 - name: rocmDependenciesNvidia
@@ -58,6 +57,9 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: matching_repo
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: hipother_repo
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependenciesAMD }}
@@ -98,6 +100,7 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
       pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
 # checkout triggering repo (either HIP or clr)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:

.azuredevops/components/HIPIFY.yml

@@ -11,42 +11,97 @@ parameters:
     - cmake
     - ninja-build
     - libnuma-dev
-- name: rocmDependencies
-  type: object
-  default:
-    - llvm-project
+    - python3-dev
+    - python3-pip
+    - python-is-python3
+    - mesa-common-dev
+    - ccache
+    - cuda-toolkit
+    - cudnn
 
 jobs:
 - job: HIPIFY
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  - name: UPSTREAM_LLVM_GIT_URL
+    value: https://github.com/llvm/llvm-project.git
+  - name: UPSTREAM_LLVM_TAG
+    value: llvmorg-18.1.2
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
   steps:
+  - task: Bash@3
+    displayName: 'Register CUDA packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-archive-keyring.gpg | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/cuda-archive-keyring.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" | sudo tee /etc/apt/sources.list.d/cuda-ubuntu2204-x86_64.list
+        sudo apt update
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: Bash@3
+    displayName: git clone upstream llvm-project
+    inputs:
+      targetType: inline
+      script: git clone $(UPSTREAM_LLVM_GIT_URL) --depth=1 --branch $(UPSTREAM_LLVM_TAG) --recurse-submodules
+      workingDirectory: $(Pipeline.Workspace)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - script: |
+      mkdir -p $(CCACHE_DIR)
+      echo "##vso[task.prependpath]/usr/lib/ccache:/usr/local/cuda/bin"
+    displayName: Update path for cuda and ccache
+  - task: Cache@2
+    displayName: Ccache caching
+    inputs:
+      key: HIPIFY | $(Agent.OS) | "$(UPSTREAM_LLVM_TAG)"
+      path: $(CCACHE_DIR)
+      restoreKeys: HIPIFY | $(Agent.OS)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
+      componentName: upstream-llvm
+      cmakeBuildDir: $(Pipeline.Workspace)/llvm-project/llvm/build
+      installDir: $(Pipeline.Workspace)/llvm
       extraBuildFlags: >-
         -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DLLVM_ENABLE_PROJECTS=clang
+        -DLLVM_INCLUDE_TESTS=OFF
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
         -GNinja
+  - task: Bash@3
+    displayName: python install lit
+    inputs:
+      targetType: inline
+      script: sudo python3 $(Pipeline.Workspace)/llvm-project/llvm/utils/lit/setup.py install
+  - task: Bash@3
+    displayName: install FileCheck
+    inputs:
+      targetType: inline
+      script: cp $(Pipeline.Workspace)/llvm-project/llvm/build/bin/FileCheck $(Pipeline.Workspace)/llvm/bin
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      componentName: HIPIFY
+      extraBuildFlags: >-
+        -DHIPIFY_CLANG_TESTS=ON
+        -DCMAKE_BUILD_TYPE=Release
+        -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda/targets/x86_64-linux
+        -DCUDA_DNN_ROOT_DIR=/usr/local/cuda/targets/x86_64-linux
+        -DCMAKE_PREFIX_PATH=$(Pipeline.Workspace)/llvm;/usr/local/cuda/targets/x86_64-linux/lib
+        -DLLVM_EXTERNAL_LIT=$(Pipeline.Workspace)/llvm-project/llvm/build/bin/llvm-lit
+      multithreadFlag: -- -j32
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: HIPIFY
+      testDir: $(Build.SourcesDirectory)/build
+      testExecutable: make
+      testParameters: test-hipify
+      testPublishResults: false

.azuredevops/components/MIOpen.yml

@@ -25,6 +25,7 @@ parameters:
     - rocBLAS
     - hipBLAS
     - hipBLASLt
+    - hipBLAS-common
     - half
     - composable_kernel
     - rocm-cmake
@@ -40,7 +41,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LARGE_DISK_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
   strategy:

.azuredevops/components/MIVisionX.yml

@@ -58,6 +58,20 @@ parameters:
     - AMDMIGraphX
     - rpp
     - rocDecode
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - half
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - MIOpen
+    - rocBLAS
+    - rocprofiler-register
+    - ROCR-Runtime
+    - roctracer
+    - rpp
 
 jobs:
 - job: MIVisionX
@@ -103,3 +117,52 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: MIVisionX_testing
+  dependsOn: MIVisionX
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  # anything in /opt may be persistent across runs
+  # so we need to remove the symlink if it already exists
+  - script: |
+      sudo rm -f /opt/rocm
+      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      mkdir mivisionx-tests
+      cd mivisionx-tests
+      cmake /opt/rocm/share/mivisionx/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: MIVisionX
+      testDir: 'mivisionx-tests'

.azuredevops/components/ROCR-Runtime.yml

@@ -16,7 +16,6 @@ parameters:
   type: object
   default:
     - llvm-project
-    - ROCT-Thunk-Interface
     - rocprofiler-register
 
 jobs:
@@ -49,5 +48,5 @@ jobs:
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-      cmakeBuildDir: 'src/build'
+        -DBUILD_SHARED_LIBS=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/ROCmValidationSuite.yml

@@ -19,6 +19,8 @@ parameters:
   type: object
   default:
     - clr
+    - hipBLAS-common
+    - hipBLASLt
     - hipRAND
     - llvm-project
     - rocBLAS
@@ -28,7 +30,19 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocRAND
-    - ROCT-Thunk-Interface
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - hipRAND
+    - llvm-project
+    - rocBLAS
+    - rocm_smi_lib
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
 
 jobs:
 - job: ROCmValidationSuite
@@ -69,12 +83,6 @@ jobs:
       # manual build case: triggered by ROCm/ROCm repo
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
-# Set link to redirect llvm folder
-  - task: Bash@3
-    displayName: create symlink
-    inputs:
-      targetType: inline
-      script: ln -s $(Agent.BuildDirectory)/rocm/llvm $(Agent.BuildDirectory)/rocm/lib/llvm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -86,3 +94,47 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: ROCmValidationSuite_testing
+  dependsOn: ROCmValidationSuite
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: ROCmValidationSuite
+      testExecutable: $(Agent.BuildDirectory)/rocm/bin/rvs -c $(Agent.BuildDirectory)/rocm/share/rocm-validation-suite/conf/MI300X/gst_single.conf
+      testParameters: ''
+      testDir: $(Agent.BuildDirectory)
+      testPublishResults: false

.azuredevops/components/amdsmi.yml

@@ -32,3 +32,30 @@ jobs:
       extraBuildFlags: >-
         -DBUILD_TESTS=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: amdsmi_testing
+  dependsOn: amdsmi
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: amdsmi
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/share/amd_smi/tests/amdsmitst'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/aomp.yml

@@ -5,9 +5,8 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
-- name: offloadEnabled
-  type: boolean
-  default: false
+# reference:
+# https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL_PREREQUISITE.md
 - name: aptPackages
   type: object
   default:
@@ -30,21 +29,49 @@ parameters:
     - libsystemd-dev
     - libssl-dev
     - libstdc++-12-dev
+    - ccache
+    - libgmp-dev
+    - libmpfr-dev
+    - texinfo
+    - libbison-dev
+    - bison
+    - flex
+    - libbabeltrace-dev
+    - libncurses5-dev
+    - liblzma-dev
+    - python3-setuptools
+    - python3-dev
+    - libudev-dev
+  # Referencing comment snippet below but excluding rocprofiler and roctracer.
+  # This is to remove need for separate build per gpu target.
+  # With our selected build flags, compilation and installation work fine without these two.
+  #
+  # snippet from https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L131-L134
+  #
+  # For ROCM build (AOMP_STANDALONE_BUILD=0) the components roct, rocr,
+  # libdevice, project, comgr, rocminfo, hipamd, rocdbgapi, rocgdb,
+  # roctracer, rocprofiler, rocm_smi_lib, and amdsmi should be found
+  # in ROCM in /opt/rocm.  The ROCM build only needs these components:
 - name: rocmDependencies
   type: object
   default:
     - rocm-cmake
     - llvm-project
     - ROCR-Runtime
-    - ROCT-Thunk-Interface
+    - clr
+    - rocminfo
+    - rocprofiler-register
+    - ROCdbgapi
+    - ROCgdb
+    - rocm_smi_lib
+    - amdsmi
 
 jobs:
 - job: aomp
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -52,9 +79,16 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+# checkout the repos tied to openmp-extras, plus llvm-project
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: aomp-extras_repo
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: flang_repo
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: llvm-project_repo
@@ -67,56 +101,309 @@ jobs:
       # manual build case: triggered by ROCm/ROCm repo
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+# Because clang is not being rebuilt and to separate downloaded ROCm
+# dependencies from the new artifacts, we use temporary symbolic links
+# for the compilation and installation to go through.
+  - script: |
+      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      mkdir -p $(Build.BinariesDirectory)/bin
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang $(Build.BinariesDirectory)/bin/clang
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang++ $(Build.BinariesDirectory)/bin/clang++
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/llvm-config $(Build.BinariesDirectory)/bin/llvm-config
+      ln -s $(Agent.BuildDirectory)/rocm/llvm $(Build.BinariesDirectory)/llvm
+    displayName: Extra build environment setup
+# We follow the sequence described in the aomp repo instructions
+# https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL.md
+# But instead of calling build_aomp.sh directly, we have to split up the calls to build each component
+# in order for the omp.h header file to be both be published as an artifact,
+# and be found when building subsequent components.
+# Environment variables specified per bash script were determined from looking through the
+# individual scripts and iteratively testing compilation.
+# Splitting up the build_aomp.sh call is also easier for debugging within Azure, as the former
+# method leads to a giant build log compared to separate logs per script call.
+#
+# Components compiled and the order for non-standalone build found at
+# https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L135-L143
+  - task: Bash@3
+    displayName: Build Prereq
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_prereq.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Build extras
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_extras.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_EXTRAS: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install extras
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_extras.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_EXTRAS: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build openmp
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_openmp.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install openmp
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_openmp.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - script: ln -s $(Build.BinariesDirectory)/include/omp.h $(Build.SourcesDirectory)/llvm-project/llvm/include/omp.h
+    displayName: Link omp header
+  - task: Bash@3
+    displayName: Build offload
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_offload.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install offload
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_offload.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build flang-legacy
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-legacy.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install flang-legacy
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-legacy.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build pgmath
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_pgmath.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install pgmath
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_pgmath.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build flang
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install flang
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build flang_runtime
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang_runtime.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install flang_runtime
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang_runtime.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+# Clean up build environment before publish artifact
+  - script: |
+      rm $(Build.BinariesDirectory)/bin/clang
+      rm $(Build.BinariesDirectory)/bin/clang++
+      rm $(Build.BinariesDirectory)/bin/llvm-config
+      rm $(Build.BinariesDirectory)/llvm
+    displayName: Remove temporary symbolic links
+# Copy the files to artifact staging temporarily to clean up binaries directory
+# and then copy files back to llvm subdirectory in the cleaned up binaries directory
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
-      componentName: llvm-openmp
-      extraBuildFlags: >-
-        -DOPENMP_ENABLE_LIBOMPTARGET=1
-        -DOPENMP_TEST_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-        -DOPENMP_TEST_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-        -DLIBOMPTARGET_AMDGCN_GFXLIST=gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906;gfx908;gfx90a;gfx90c;gfx940;gfx941;gfx942;gfx1030;gfx1031;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103
-        -DDEVICELIBS_ROOT=$(Build.SourcesDirectory)/llvm-project/amd/device-libs
-        -DLIBOMP_COPY_EXPORTS=OFF
-        -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm
-        -DLLVM_MAIN_INCLUDE_DIR=$(Build.SourcesDirectory)/llvm-project/llvm/include
-        -DLIBOMPTARGET_LLVM_INCLUDE_DIRS=$(Build.SourcesDirectory)/llvm-project/llvm/include
-        -DCUDA_TOOLKIT_ROOT_DIR=OFF
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/lib/cmake
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_SKIP_BUILD_RPATH=TRUE
-        -DCMAKE_SKIP_INSTALL_RPATH=TRUE
-        -DCMAKE_SHARED_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-        -DCMAKE_EXE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-        -DCMAKE_MODULE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-        -GNinja
-      cmakeBuildDir: $(Build.SourcesDirectory)/llvm-project/openmp/build
-      installDir: $(Build.BinariesDirectory)/llvm
-# offload does not exist for recent releases, so use CI conditional
-  - ${{ if eq(parameters.offloadEnabled, true) }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      parameters:
-        componentName: llvm-offload
-        extraBuildFlags: >-
-          -DOPENMP_ENABLE_LIBOMPTARGET=1
-          -DOPENMP_TEST_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-          -DOPENMP_TEST_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-          -DLIBOMPTARGET_AMDGCN_GFXLIST=gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906;gfx908;gfx90a;gfx90c;gfx940;gfx941;gfx942;gfx1030;gfx1031;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103
-          -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm
-          -DLLVM_MAIN_INCLUDE_DIR=$(Build.SourcesDirectory)/llvm-project/llvm/include
-          -DLIBOMPTARGET_LLVM_INCLUDE_DIRS=$(Build.SourcesDirectory)/llvm-project/llvm/include
-          -DCUDA_TOOLKIT_ROOT_DIR=OFF
-          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/lib/cmake
-          -DCMAKE_BUILD_TYPE=Release
-          -DCMAKE_SKIP_BUILD_RPATH=TRUE
-          -DCMAKE_SKIP_INSTALL_RPATH=TRUE
-          -DCMAKE_SHARED_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-          -DCMAKE_EXE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-          -DCMAKE_MODULE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-          -GNinja
-        cmakeBuildDir: $(Build.SourcesDirectory)/llvm-project/offload/build
-        installDir: $(Build.BinariesDirectory)/llvm
+      sourceDir: $(Build.BinariesDirectory)
+      targetDir: $(Build.ArtifactStagingDirectory)
+  - task: DeleteFiles@1
+    displayName: 'Cleanup Binaries Directory'
+    inputs:
+      SourceFolder: '$(Build.BinariesDirectory)'
+      Contents: '/**/*'
+      RemoveDotFiles: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.ArtifactStagingDirectory)
+      targetDir: $(Build.BinariesDirectory)/llvm
+  - task: DeleteFiles@1
+    displayName: 'Cleanup Staging Directory'
+    inputs:
+      SourceFolder: $(Build.ArtifactStagingDirectory)
+      Contents: '/**/*'
+      RemoveDotFiles: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/composable_kernel.yml

@@ -13,6 +13,7 @@ parameters:
     - git
     - python3-pip
     - libdrm-dev
+    - ccache
 - name: rocmDependencies
   type: object
   default:
@@ -22,13 +23,24 @@ parameters:
     - clr
     - rocminfo
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - composable_kernel
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: composable_kernel
-  timeoutInMinutes: 180
+  timeoutInMinutes: 240
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
+  - name: DAY_STRING
+    value: $[format('{0:ddMMyyyy}', pipeline.startTime)]
   pool: ${{ variables.ULTRA_BUILD_POOL }}
   workspace:
     clean: all
@@ -56,11 +68,26 @@ jobs:
       # manual build case: triggered by ROCm/ROCm repo
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - script: |
+      mkdir -p $(CCACHE_DIR)
+      echo "##vso[task.prependpath]/usr/lib/ccache"
+    displayName: Update path for ccache
+  - task: Cache@2
+    displayName: Ccache caching
+    inputs:
+      key: composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING) | $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+      path: $(CCACHE_DIR)
+      restoreKeys: |
+        composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING)
+        composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET)
+        composable_kernel | $(Agent.OS)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
         -DCMAKE_HIP_FLAGS="-Wno-missing-include-dirs"
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
@@ -69,3 +96,51 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: composable_kernel_testing
+  dependsOn: composable_kernel
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: TEST_LOG_FILE
+    value: $(Pipeline.Workspace)/ckTestLog.log
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Iterate through test scripts
+    inputs:
+      targetType: inline
+      script: |
+        for file in ./test_*; do
+          ./$file | tee -a $(TEST_LOG_FILE)
+        done
+      workingDirectory: $(Agent.BuildDirectory)/rocm/bin

.azuredevops/components/copyHIP.yml

@@ -24,9 +24,8 @@ jobs:
   - checkout: none
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-download.yml
     parameters:
-      useDefaultBranch: false
       componentName: HIP
-      branchName: develop
+      pipelineId: $(hip-pipeline-id)
   - template:  ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
       sourceDir: $(Agent.BuildDirectory)/rocm

.azuredevops/components/half.yml

@@ -53,3 +53,13 @@ jobs:
         -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  # only run test03 because test11 has too many test cases, taking way too long
+  - task: Bash@3
+    displayName: test03
+    inputs:
+      targetType: inline
+      script: |
+        make clean
+        make test03
+        ./bin/test
+      workingDirectory: $(Build.SourcesDirectory)/test

.azuredevops/components/hip-tests.yml

@@ -0,0 +1,81 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - ninja-build
+    - git
+    - libnuma-dev
+    - mesa-common-dev
+    - ocl-icd-libopencl1
+    - ocl-icd-opencl-dev
+    - opencl-headers
+    - libglu1-mesa-dev
+    - freeglut3-dev
+    - libboost-all-dev
+    - python3-pip
+- name: rocmDependencies
+  type: object
+  default:
+    - llvm-project
+    - ROCR-Runtime
+    - clr
+    - rocminfo
+    - rocprofiler-register
+
+# HIP with AMD backend
+jobs:
+- job: hip_tests
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: HIP_ROCCLR_HOME
+    value: $(Agent.BuildDirectory)/rocm
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  # compile hip-tests
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      componentName: hip-tests
+      cmakeSourceDir: '../catch'
+      customBuildTarget: build_tests
+      extraBuildFlags: >-
+        -DHIP_PLATFORM=amd
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DHIP_PATH=$(Agent.BuildDirectory)/rocm
+        -DOFFLOAD_ARCH_STR="--offload-arch=$(JOB_GPU_TARGET)"
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/hipBLAS.yml

@@ -31,6 +31,21 @@ parameters:
     - rocBLAS
     - rocSPARSE
     - rocSOLVER
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocm-cmake
+    - ROCR-Runtime
+    - rocminfo
+    - rocprofiler-register
+    - rocBLAS
+    - rocSPARSE
+    - rocSOLVER
 
 jobs:
 - job: hipBLAS
@@ -54,18 +69,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - task: Bash@3
-    displayName: 'Download AOCL'
-    inputs:
-      targetType: inline
-      script: wget -nv https://download.amd.com/developer/eula/aocl/aocl-4-1/aocl-linux-aocc-4.1.0_1_amd64.deb
-      workingDirectory: '$(Pipeline.Workspace)'
-  - task: Bash@3
-    displayName: 'Install AOCL'
-    inputs:
-      targetType: inline
-      script: sudo apt install --yes ./aocl-linux-aocc-4.1.0_1_amd64.deb
-      workingDirectory: '$(Pipeline.Workspace)'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
@@ -91,3 +95,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipBLAS_testing
+  dependsOn: hipBLAS
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipBLAS
+      testExecutable: $(Agent.BuildDirectory)/rocm/bin/hipblas-test
+      testParameters: '--yaml hipblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'

.azuredevops/components/hipBLASLt.yml

@@ -8,13 +8,16 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - gfortran
     - git
     - libdrm-dev
     - libmsgpack-dev
+    - libnuma-dev
     - ninja-build
     - python3-pip
     - python3-venv
+    - gfortran
+    - libomp-dev
+    - libblas-dev
 - name: pipModules
   type: object
   default:
@@ -22,10 +25,23 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
+    - aomp
     - clr
-    - hipBLAS
+    - hipBLAS-common
     - llvm-project
     - rocminfo
+    - rocm_smi_lib
+    - rocprofiler-register
+    - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - llvm-project
+    - rocminfo
+    - rocm_smi_lib
     - rocprofiler-register
     - ROCR-Runtime
 
@@ -82,6 +98,7 @@ jobs:
 # $(Pipeline.Workspace)/deps is a temporary folder for the build process
 # $(Pipeline.Workspace)/s/deps is part of the hipBLASLt repo
   - script: mkdir $(Pipeline.Workspace)/deps
+    displayName: Create temp folder for external dependencies
 # hipBLASLt already has a CMake script for external deps, so we can just run that
 # https://github.com/ROCm/hipBLASLt/blob/develop/deps/CMakeLists.txt
   - script: cmake $(Pipeline.Workspace)/s/deps
@@ -93,12 +110,6 @@ jobs:
   - script: sudo make install
     displayName: Install hipBLASLt external dependencies
     workingDirectory: $(Pipeline.Workspace)/deps
-# Set link to redirect llvm folder
-  - task: Bash@3
-    displayName: Symlink to rocm/lib/llvm
-    inputs:
-      targetType: inline
-      script: ln -s $(Agent.BuildDirectory)/rocm/llvm $(Agent.BuildDirectory)/rocm/lib/llvm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -111,7 +122,52 @@ jobs:
         -DTensile_CODE_OBJECT_VERSION=default
         -DTensile_LIBRARY_FORMAT=msgpack
         -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
+        -DBUILD_CLIENTS_TESTS=ON
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+
+- job: hipBLASLt_testing
+  dependsOn: hipBLASLt
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipBLASLt
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipblaslt-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'

.azuredevops/components/hipCUB.yml

@@ -10,7 +10,7 @@ parameters:
   default:
     - cmake
     - ninja-build
-    - googletest
+    - libgtest-dev
     - git
     - python3-pip
 - name: rocmDependencies
@@ -21,6 +21,15 @@ parameters:
     - rocminfo
     - rocPRIM
     - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocPRIM
+    - ROCR-Runtime
+    - rocprofiler-register
 
 jobs:
 - job: hipCUB
@@ -66,3 +75,44 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipCUB_testing
+  dependsOn: hipCUB
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipCUB
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/hipcub'

.azuredevops/components/hipFFT.yml

@@ -27,6 +27,19 @@ parameters:
     - rocminfo
     - rocFFT
     - aomp
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipRAND
+    - llvm-project
+    - rocFFT
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+
 jobs:
 - job: hipFFT
   variables:
@@ -80,3 +93,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipFFT_testing
+  dependsOn: hipFFT
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipFFT
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipfft-test'
+      testParameters: '--test_prob 0.002 --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/hipRAND.yml

@@ -20,6 +20,15 @@ parameters:
     - clr
     - rocminfo
     - rocRAND
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
 
 jobs:
 - job: hipRAND
@@ -70,3 +79,44 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipRAND_testing
+  dependsOn: hipRAND
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipRAND
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/hipRAND'

.azuredevops/components/hipSOLVER.yml

@@ -27,6 +27,21 @@ parameters:
     - ROCR-Runtime
     - rocSPARSE
     - rocSOLVER
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - hipBLAS-common
+    - hipBLASLt
+    - hipSPARSE
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocSOLVER
+    - rocSPARSE
 
 jobs:
 - job: hipSOLVER
@@ -84,3 +99,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipSOLVER_testing
+  dependsOn: hipSOLVER
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipSOLVER
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipsolver-test'
+      testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/hipSPARSE.yml

@@ -25,6 +25,18 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocSPARSE
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - hipBLAS-common
+    - hipBLASLt
+    - rocBLAS
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocSPARSE
 
 jobs:
 - job: hipSPARSE
@@ -81,3 +93,46 @@ jobs:
     parameters:
       artifactName: testMatrices
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipSPARSE_testing
+  dependsOn: hipSPARSE
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipSPARSE
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipsparse-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/hipSPARSELt.yml

@@ -13,21 +13,37 @@ parameters:
     - libmsgpack-dev
     - git
     - python3-pip
+    - gfortran
+    - libgfortran5
+    - libomp-dev
+    - libopenblas-dev
 - name: pipModules
   type: object
   default:
     - joblib
-# rocm dependencies should match dependencies-rocm.yml
 - name: rocmDependencies
   type: object
   default:
-    - llvm-project
-    - ROCR-Runtime
+    - aomp
     - clr
+    - hipSPARSE
+    - llvm-project
+    - rocBLAS
     - rocminfo
     - rocprofiler-register
-    - hipSPARSE
+    - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - llvm-project
+    - hipBLAS-common
+    - hipBLASLt
     - rocBLAS
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: hipSPARSELt
@@ -71,6 +87,22 @@ jobs:
       # manual build case: triggered by ROCm/ROCm repo
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+# Build and install gtest and lapack
+# $(Pipeline.Workspace)/deps is a temporary folder for the build process
+# $(Pipeline.Workspace)/s/deps is part of the hipSPARSELt repo
+  - script: mkdir $(Pipeline.Workspace)/deps
+    displayName: Create temp folder for external dependencies
+# hipSPARSELt already has a CMake script for external deps, so we can just run that
+# https://github.com/ROCm/hipSPARSELt/blob/develop/deps/CMakeLists.txt
+  - script: cmake $(Pipeline.Workspace)/s/deps
+    displayName: Configure hipSPARSELt external dependencies
+    workingDirectory: $(Pipeline.Workspace)/deps
+  - script: make
+    displayName: Build hipSPARSELt external dependencies
+    workingDirectory: $(Pipeline.Workspace)/deps
+  - script: sudo make install
+    displayName: Install hipSPARSELt external dependencies
+    workingDirectory: $(Pipeline.Workspace)/deps
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -84,7 +116,45 @@ jobs:
         -DTensile_LIBRARY_FORMAT=msgpack
         -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DBUILD_CLIENTS_TESTS=ON
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipSPARSELt_testing
+  dependsOn: hipSPARSELt
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipSPARSELt
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipsparselt-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'

.azuredevops/components/hipTensor.yml

@@ -21,6 +21,14 @@ parameters:
     - clr
     - rocminfo
     - composable_kernel
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: hipTensor
@@ -67,3 +75,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipTensor_testing
+  timeoutInMinutes: 90
+  dependsOn: hipTensor
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipTensor
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/hiptensor'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/llvm-project.yml

@@ -41,6 +41,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
+      skipLlvmSymlink: true
       # CI case: download latest default branch build
       ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging

.azuredevops/components/rccl.yml

@@ -16,6 +16,7 @@ parameters:
     - git
     - ninja-build
     - libstdc++-12-dev
+    - libnuma-dev
 - name: rocmDependencies
   type: object
   default:
@@ -51,10 +52,12 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+      submoduleBehaviour: recursive
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}

.azuredevops/components/rocAL.yml

@@ -25,6 +25,7 @@ parameters:
     - libturbojpeg-dev
     - libjpeg-turbo-official=3.0.2-20240124
     - libopencv-dev
+    - --allow-downgrades # for libjpeg-turbo
 - name: pipModules
   type: object
   default:
@@ -44,6 +45,17 @@ parameters:
     - rpp
     - MIVisionX
     - aomp
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - half
+    - llvm-project
+    - MIVisionX
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rpp
 
 jobs:
 - job: rocAL
@@ -142,3 +154,81 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocAL_testing
+  dependsOn: rocAL
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: ROCM_PATH
+    value: $(Agent.BuildDirectory)/rocm
+  - name: CMAKE_INCLUDE_PATH
+    value: $(Agent.BuildDirectory)/rocm/include/rocal
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - task: Bash@3
+    displayName: 'Register libjpeg-turbo packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
+        sudo apt update
+        apt-cache show libjpeg-turbo-official | grep Version
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Link libjpeg-turbo
+    inputs:
+      targetType: inline
+      script: |
+        echo /opt/libjpeg-turbo/lib64 | sudo tee /etc/ld.so.conf.d/libjpeg-turbo.conf
+        sudo ldconfig -v
+  - task: Bash@3
+    displayName: Build rocAL tests
+    inputs:
+      targetType: inline
+      script: |
+        mkdir rocAL-tests
+        cd rocAL-tests
+        cmake $(Agent.BuildDirectory)/rocm/share/rocal/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocAL
+      testDir: rocAL-tests
+      reloadAMDGPU: true
+  - task: Bash@3
+    displayName: Clean up libjpeg-turbo
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm /etc/ld.so.conf.d/libjpeg-turbo.conf
+        sudo ldconfig -v

.azuredevops/components/rocALUTION.yml

@@ -28,6 +28,21 @@ parameters:
     - ROCR-Runtime
     - rocRAND
     - rocSPARSE
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocminfo
+    - rocPRIM
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+    - rocSPARSE
 
 jobs:
 - job: rocALUTION
@@ -79,3 +94,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocALUTION_testing
+  dependsOn: rocALUTION
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocALUTION
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocalution-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocBLAS.yml

@@ -24,6 +24,7 @@ parameters:
   type: object
   default:
     - joblib
+    - pyyaml
 - name: rocmDependencies
   type: object
   default:
@@ -37,6 +38,20 @@ parameters:
     - rocm-core
     - aomp
     - aomp-extras
+    - hipBLAS-common
+    - hipBLASLt
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - rocm_smi_lib
+    - ROCR-Runtime
 
 jobs:
 - job: rocBLAS
@@ -69,28 +84,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - task: Bash@3
-    displayName: 'Download AOCL'
-    inputs:
-      targetType: inline
-      workingDirectory: '$(Pipeline.Workspace)'
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        script: wget -nv https://download.amd.com/developer/eula/aocl/aocl-4-2/aocl-linux-gcc-4.2.0_1_amd64.deb
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        script: wget -nv https://download.amd.com/developer/eula/aocl/aocl-4-1/aocl-linux-aocc-4.1.0_1_amd64.deb
-  - task: Bash@3
-    displayName: 'Install AOCL'
-    inputs:
-      targetType: inline
-      workingDirectory: '$(Pipeline.Workspace)'
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        script: sudo apt install --yes ./aocl-linux-gcc-4.2.0_1_amd64.deb
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        script: sudo apt install --yes ./aocl-linux-aocc-4.1.0_1_amd64.deb
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
@@ -123,3 +117,47 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocBLAS_testing
+  dependsOn: rocBLAS
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocBLAS
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocblas-test'
+      testParameters: '--yaml rocblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocDecode.yml

@@ -30,6 +30,13 @@ parameters:
     - rocminfo
     - rocm-core
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocDecode
@@ -73,9 +80,56 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocDecode_testing
+  dependsOn: rocDecode
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  # anything in /opt may be persistent across runs
+  # so we need to remove the symlink if it already exists
+  - script: |
+      sudo rm -rf /opt/rocm
+      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      mkdir rocDecode-tests
+      cd rocDecode-tests
+      cmake /opt/rocm/share/rocdecode/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocDecode
+      testDir: 'rocDecode-tests'
+  - script: sudo rm /opt/rocm

.azuredevops/components/rocFFT.yml

@@ -19,15 +19,28 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
-    - llvm-project
-    - ROCR-Runtime
+    - aomp
     - clr
+    - hipRAND
+    - llvm-project
+    - rocm-cmake
     - rocminfo
     - rocprofiler-register
-    - hipRAND
+    - ROCR-Runtime
     - rocRAND
-    - rocm-cmake
+- name: rocmTestDependencies
+  type: object
+  default:
     - aomp
+    - clr
+    - hipRAND
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+
 jobs:
 - job: rocFFT
   variables:
@@ -79,3 +92,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocFFT_testing
+  dependsOn: rocFFT
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocFFT
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocfft-test'
+      testParameters: '--test_prob 0.004 --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocMLIR.yml

@@ -12,14 +12,23 @@ parameters:
     - ninja-build
     - git
     - python3-pip
+    - libdrm-dev
+- name: pipModules
+  type: object
+  default:
+    - tomli
 - name: rocmDependencies
   type: object
   default:
     - llvm-project
     - rocm-cmake
+    - clr
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
-- job: rocMLIR
+- job: rocMLIR_library
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -30,6 +39,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
@@ -53,3 +63,63 @@ jobs:
         -DBUILD_FAT_LIBROCKCOMPILER=1
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+# compiling and running test on the test system together
+- job: rocMLIR_testing
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.GFX942_TEST_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      installEnabled: false
+      extraBuildFlags: >-
+        -DCMAKE_BUILD_TYPE=Release
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DROCM_TEST_CHIPSET=$(JOB_GPU_TARGET)
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocMLIR
+      testDir: $(Build.SourcesDirectory)/build
+      testExecutable: ninja
+      testParameters: check-rocmlir

.azuredevops/components/rocPRIM.yml

@@ -21,6 +21,14 @@ parameters:
     - ROCR-Runtime
     - clr
     - rocminfo
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler-register
 
 jobs:
 - job: rocPRIM
@@ -66,3 +74,44 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocPRIM_testing
+  dependsOn: rocPRIM
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocPRIM
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocprim'

.azuredevops/components/rocPyDecode.yml

@@ -0,0 +1,87 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - python3-pip
+    - cmake
+    - ninja-build
+    - pkg-config
+    - ffmpeg
+    - libavcodec-dev
+    - libavformat-dev
+    - libavutil-dev
+    - libva-dev
+    - libdrm-dev
+    - pybind11-dev
+    - python3-pybind11
+    - libdlpack-dev
+- name: pipModules
+  type: object
+  default:
+    - -i
+    - https://test.pypi.org/simple
+    - hip-python
+- name: rocmDependencies
+  type: object
+  default:
+    - rocm-cmake
+    - llvm-project
+    - ROCR-Runtime
+    - clr
+    - rocminfo
+    - rocm-core
+    - rocprofiler-register
+    - rocDecode
+
+jobs:
+- job: rocPyDecode
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_INSTALL_PREFIX_PYTHON=$(Build.BinariesDirectory)
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocRAND.yml

@@ -21,6 +21,14 @@ parameters:
     - ROCR-Runtime
     - clr
     - rocminfo
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocRAND
@@ -68,3 +76,44 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocRAND_testing
+  dependsOn: rocRAND
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocRAND
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocRAND'

.azuredevops/components/rocSOLVER.yml

@@ -20,15 +20,30 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
     - clr
-    - rocminfo
-    - rocBLAS
-    - rocPRIM
-    - rocSPARSE
     - hipSPARSE
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - rocPRIM
+    - ROCR-Runtime
+    - rocSPARSE
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - hipSPARSE
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - rocPRIM
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocSPARSE
 
 jobs:
 - job: rocSOLVER
@@ -94,3 +109,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocSOLVER_testing
+  dependsOn: rocSOLVER
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocSOLVER
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocsolver-test'
+      testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocSPARSE.yml

@@ -29,6 +29,17 @@ parameters:
     - rocminfo
     - rocPRIM
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocSPARSE
@@ -91,3 +102,48 @@ jobs:
     parameters:
       artifactName: testMatrices
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocSPARSE_testing
+  timeoutInMinutes: 90
+  dependsOn: rocSPARSE
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocSPARSE
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocsparse-test'
+      testParameters: '--gtest_filter="*quick*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+

.azuredevops/components/rocThrust.yml

@@ -24,6 +24,16 @@ parameters:
     - rocminfo
     - rocPRIM
     - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocPRIM
+    - ROCR-Runtime
+    - hipRAND
+    - rocprofiler-register
 
 jobs:
 - job: rocThrust
@@ -69,3 +79,44 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocThrust_testing
+  dependsOn: rocThrust
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocThrust
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocthrust'

.azuredevops/components/rocWMMA.yml

@@ -28,6 +28,20 @@ parameters:
     - rocBLAS
     - aomp
     - rocm_smi_lib
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocm_smi_lib
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocWMMA
@@ -75,3 +89,45 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocWMMA_testing
+  timeoutInMinutes: 90
+  dependsOn: rocWMMA
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocWMMA
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocwmma'

.azuredevops/components/rocm-cmake.yml

@@ -44,5 +44,7 @@ jobs:
       git config --global user.email "you@example.com"
       git config --global user.name "Your Name"
     displayName: "ctest setup"
-#  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm-cmake
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/rocm-core.yml

@@ -29,5 +29,5 @@ jobs:
         -DCPACK_GENERATOR=DEB
         -DCPACK_DEBIAN_PACKAGE_RELEASE="local.9999~99.99"
         -DCPACK_RPM_PACKAGE_RELEASE="local.9999"
-        -DROCM_VERSION="$(last-release)"
+        -DROCM_VERSION="$(next-release)"
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/rocm-examples.yml

@@ -8,20 +8,51 @@ parameters:
 - name: aptPackages
   type: object
   default:
+    - cmake
     - libglfw3-dev
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
     - AMDMIGraphX
     - clr
     - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
     - hipCUB
+    - hipFFT
     - HIPIFY
     - hipRAND
     - hipSOLVER
     - hipSPARSE
     - llvm-project
     - rocBLAS
+    - rocFFT
+    - rocPRIM
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+    - rocSOLVER
+    - rocSPARSE
+    - rocThrust
+- name: rocmTestDependencies
+  type: object
+  default:
+    - AMDMIGraphX
+    - clr
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - hipCUB
+    - hipFFT
+    - HIPIFY
+    - hipRAND
+    - hipSOLVER
+    - hipSPARSE
+    - llvm-project
+    - rocBLAS
+    - rocFFT
+    - rocminfo
     - rocPRIM
     - rocprofiler-register
     - ROCR-Runtime
@@ -35,8 +66,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -46,13 +76,13 @@ jobs:
       gfx90a:
         JOB_GPU_TARGET: gfx90a
   steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
@@ -70,8 +100,72 @@ jobs:
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCM_ROOT=$(Agent.BuildDirectory)/rocm
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
         -DCMAKE_EXE_LINKER_FLAGS=-fgpu-rdc
+  - task: Bash@3
+    displayName: Move rocm-examples binaries to rocm/examples
+    inputs:
+      targetType: inline
+      script: |
+        mkdir -p $(Build.BinariesDirectory)/examples
+        mv $(Build.BinariesDirectory)/bin/* $(Build.BinariesDirectory)/examples
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocm_examples_testing
+  dependsOn: rocm_examples
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: TEST_LOG_FILE
+    value: $(Pipeline.Workspace)/rocm-examplesTestLog.log
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Unload and reload AMDGPU
+    inputs:
+      targetType: inline
+      script: |
+        sudo modprobe -r amdgpu
+        sudo modprobe amdgpu
+  - task: Bash@3
+    displayName: Iterate through examples
+    inputs:
+      targetType: inline
+      script: |
+        for file in *; do
+          echo Now running: $file
+          ./$file | tee -a $(TEST_LOG_FILE)
+        done
+      workingDirectory: $(Agent.BuildDirectory)/rocm/examples

.azuredevops/components/rocm_bandwidth_test.yml

@@ -24,7 +24,11 @@ parameters:
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
-    - ROCT-Thunk-Interface
+- name: rocmTestDependencies
+  type: object
+  default:
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocm_bandwidth_test
@@ -65,3 +69,40 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/include;$(Agent.BuildDirectory)/rocm/include/hsa
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocm_bandwidth_test_testing
+  dependsOn: rocm_bandwidth_test
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm_bandwidth_test
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/bin/rocm-bandwidth-test'
+      testParameters: ''
+      testPublishResults: false

.azuredevops/components/rocm_smi_lib.yml

@@ -26,3 +26,27 @@ jobs:
         -DBUILD_TESTS=ON
         -DROCM_DEP_ROCMCORE=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocm_smi_lib_testing
+  dependsOn: rocm_smi_lib
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm_smi_lib
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/share/rocm_smi/rsmitst_tests/rsmitst'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocminfo.yml

@@ -10,6 +10,11 @@ parameters:
   default:
     - ROCR-Runtime
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - ROCR-Runtime
+    - rocprofiler-register
 
 jobs:
 - job: rocminfo
@@ -28,6 +33,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
+      skipLlvmSymlink: true
       # CI case: download latest default branch build
       ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
@@ -40,3 +46,43 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCRTST_BLD_TYPE=release
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocminfo_testing
+  dependsOn: rocminfo
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocminfo
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/bin/rocminfo'
+      testParameters: ''
+      testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm_agent_enumerator
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/bin/rocm_agent_enumerator'
+      testParameters: ''
+      testPublishResults: false

.azuredevops/components/rocprofiler.yml

@@ -39,7 +39,6 @@ parameters:
     - rocminfo
     - ROCR-Runtime
     - rocprofiler-register
-    - ROCT-Thunk-Interface
     - roctracer
 
 jobs:
@@ -69,29 +68,12 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# Manually download aqlprofile, hard-coded 6.1.0 version
-  - task: Bash@3
-    displayName: 'Download aqlprofile'
-    inputs:
-      targetType: inline
-      script: wget -nv https://repo.radeon.com/rocm/misc/aqlprofile/ubuntu-22.04/hsa-amd-aqlprofile_1.0.0.60200.60200-crdnnh.14213~22.04_amd64.deb
-      workingDirectory: '$(Pipeline.Workspace)'
-  - task: Bash@3
-    displayName: 'Extract aqlprofile'
-    inputs:
-      targetType: inline
-      script: |
-        mkdir hsa-amd-aqlprofile
-        dpkg-deb -R hsa-amd-aqlprofile_1.0.0.60200.60200-crdnnh.14213~22.04_amd64.deb hsa-amd-aqlprofile
-      workingDirectory: '$(Pipeline.Workspace)'
-  - task: Bash@3
-    displayName: 'Move aqlprofile'
-    inputs:
-      targetType: inline
-      script: |
-        mkdir -p $(Agent.BuildDirectory)/rocm
-        cp -R hsa-amd-aqlprofile/opt/rocm-6.2.0-14213/* $(Agent.BuildDirectory)/rocm
-      workingDirectory: '$(Pipeline.Workspace)'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}

.azuredevops/components/rocr_debug_agent.yml

@@ -12,6 +12,8 @@ parameters:
     - ninja-build
     - libelf-dev
     - libdw-dev
+    - libstdc++-12-dev
+    - python-is-python3
 - name: rocmDependencies
   type: object
   default:
@@ -22,6 +24,15 @@ parameters:
     - rocminfo
     - ROCR-Runtime
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - ROCdbgapi
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocr_debug_agent
@@ -58,3 +69,52 @@ jobs:
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocr_debug_agent_testing
+  dependsOn: rocr_debug_agent
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      componentName: rocr_debug_agent-tests
+      extraBuildFlags: >-
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake;$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+      cmakeBuildDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'
+      cmakeSourceDir: '.'
+      installEnabled: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocr_debug_agent
+      testDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'

.azuredevops/components/roctracer.yml

@@ -25,6 +25,13 @@ parameters:
     - rocprofiler-register
     - clr
     - rocminfo
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: roctracer
@@ -74,3 +81,47 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: roctracer_testing
+  dependsOn: roctracer
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: roctracer
+      testExecutable: $(Agent.BuildDirectory)/rocm/share/roctracer/run_tests.sh
+      testParameters: ''
+      testDir: $(Agent.BuildDirectory)
+      testPublishResults: false

.azuredevops/nightly/pytorch.yml

@@ -54,6 +54,14 @@ parameters:
     - libdrm-amdgpu1
     - ccache
     - zip
+    - libjpeg-turbo-official
+    - libjpeg-dev
+    - libwebp-dev
+    - libfreetype-dev
+    - gnutls-bin
+    - ffmpeg
+    - libopenblas-dev
+    - liblapack-dev
 - name: pipModules
   type: object
   default:
@@ -76,6 +84,9 @@ parameters:
     - ninja
     - packaging
     - optree>=0.12.0
+  # list for vision
+    - auditwheel
+    - future
 # list from https://github.com/pytorch/builder/blob/main/manywheel/build_rocm.sh
 - name: rocmDependencies
   type: object
@@ -105,6 +116,7 @@ parameters:
     # below are additional dependencies not called out by build script, but throw errors during cmake
     - hipCUB
     - rocThrust
+    - hipBLAS-common
 
 trigger: none
 pr: none
@@ -145,8 +157,6 @@ jobs:
     value: 3.10
   - name: PYTORCH_ROOT
     value: $(Build.SourcesDirectory)/pytorch
-  - name: CMAKE_ARGS
-    value: -GNinja
   - name: DESIRED_DEVTOOLSET
     value: cxx11-abi
   pool: ${{ variables.ULTRA_BUILD_POOL }}
@@ -156,6 +166,16 @@ jobs:
 # copy environment setup from https://github.com/pytorch/builder/blob/main/manywheel/Dockerfile
 # but instead of centos, use ubuntu environment
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
+  - task: Bash@3
+    displayName: 'Register libjpeg-turbo packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
+        sudo apt update
+        apt-cache show libjpeg-turbo-official | grep Version
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
@@ -175,12 +195,10 @@ jobs:
       dependencySource: staging
       gpuTarget: $(JOB_GPU_TARGET)
   - task: Bash@3
-    displayName: ROCm symbolic links
+    displayName: ROCm symbolic link
     inputs:
       targetType: inline
-      script: |
-        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-        sudo ln -s $(Agent.BuildDirectory)/rocm/llvm $(Agent.BuildDirectory)/rocm/lib/llvm
+      script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
   - checkout: self
   - task: Bash@3
     displayName: git clone pytorch builder
@@ -194,12 +212,6 @@ jobs:
       targetType: inline
       script: git clone https://github.com/pytorch/pytorch.git --depth=1 --recurse-submodules
       workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: Patch out forced GPU testing block in pytorch build script
-    inputs:
-      targetType: inline
-      script: git apply $(Build.SourcesDirectory)/.azuredevops/patches/pytorch/0001-ROCm-CI-patches.patch
-      workingDirectory: $(Build.SourcesDirectory)/builder
   - task: Bash@3
     displayName: Install patchelf
     inputs:
@@ -232,8 +244,7 @@ jobs:
     displayName: Install AOTriton Shared Library
     inputs:
       targetType: inline
-      script: |
-        sudo bash ./common/install_aotriton.sh /opt/rocm
+      script: sudo bash ./common/install_aotriton.sh /opt/rocm
       workingDirectory: $(Build.SourcesDirectory)/pytorch/.ci/docker
   - task: Bash@3
     displayName: Run ROCm Build Script
@@ -246,16 +257,65 @@ jobs:
         GPU_TARGET=$(JOB_GPU_TARGET)
         DESIRED_PYTHON=$(DESIRED_PYTHON)
         PYTORCH_ROOT=$(PYTORCH_ROOT)
-        CMAKE_ARGS=$(CMAKE_ARGS)
+        CMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         AOTRITON_INSTALLED_PREFIX=$(AOTRITON_INSTALLED_PREFIX)
         DESIRED_DEVTOOLSET=$(DESIRED_DEVTOOLSET)
         TORCH_PACKAGE_NAME=torch.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
         PYTORCH_BUILD_VERSION=$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)
         PYTORCH_BUILD_NUMBER=$(date -u +%Y%m%d)
+        SKIP_ALL_TESTS=1
         bash ./manywheel/build_rocm.sh
       workingDirectory: $(Build.SourcesDirectory)/builder
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: /remote/wheelhouserocm$(ROCM_VERSION)
+      contentsString: '*.whl'
+# common helper source for pytorch vision and audio
+  - task: Bash@3
+    displayName: git clone pytorch test-infra
+    inputs:
+      targetType: inline
+      script: git clone https://github.com/pytorch/test-infra.git --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: install package helper
+    inputs:
+      targetType: inline
+      script: python3 -m pip install test-infra/tools/pkg-helpers
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: pytorch pkg helpers
+    inputs:
+      targetType: inline
+      script: CU_VERSION=${CU_VERSION} CHANNEL=${CHANNEL} python -m pytorch_pkg_helpers
+# get torch vision source and build
+  - task: Bash@3
+    displayName: git clone pytorch vision
+    inputs:
+      targetType: inline
+      script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Build vision
+    inputs:
+      targetType: inline
+      script: >-
+        BUILD_VERSION=$(ROCM_BRANCH).$(JOB_GPU_TARGET)-$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)$(date -u +%Y%m%d)
+        python3 setup.py bdist_wheel
+      workingDirectory: $(Build.SourcesDirectory)/vision
+  - task: Bash@3
+    displayName: Relocate vision
+    inputs:
+      targetType: inline
+      script: python3 packaging/wheel/relocate.py
+      workingDirectory: $(Build.SourcesDirectory)/vision
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.SourcesDirectory)/vision/dist
+      contentsString: '*.whl'
+      clean: false
   - task: PublishPipelineArtifact@1
-    displayName: 'ROCm pytorch wheel file Publish'
+    displayName: 'wheel file Publish'
     retryCountOnTaskFailure: 3
     inputs:
-      targetPath: /remote/wheelhouserocm$(ROCM_VERSION)
+      targetPath: $(Build.BinariesDirectory)

.azuredevops/nightly/rocm-nightly.yml

@@ -11,6 +11,7 @@ parameters:
     - half
     - HIP
     - hipBLAS
+    - hipBLAS-common
     - hipBLASLt
     - hipCUB
     - hipFFT
@@ -48,7 +49,6 @@ parameters:
     - rocr_debug_agent
     - rocSOLVER
     - rocSPARSE
-    - ROCT-Thunk-Interface
     - rocThrust
     - roctracer
     - rocWMMA
@@ -72,6 +72,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - task: DeleteFiles@1
     displayName: 'Cleanup checkout space'
@@ -84,28 +90,26 @@ jobs:
       SourceFolder: '$(Build.ArtifactStagingDirectory)'
       Contents: '/**/*'
       RemoveDotFiles: true
-  - script: sudo chmod 777 /mnt
-    displayName: 'Set permissions for /mnt'
   - script: df -h
     displayName: System disk space before ROCm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
       dependencySource: staging
-      extractToMnt: true
       skipLibraryLinking: true
+      gpuTarget: $(JOB_GPU_TARGET)
   - script: df -h
     displayName: System disk space after ROCm
-  - script: du -sh /mnt/rocm
+  - script: du -sh $(Agent.BuildDirectory)/rocm
     displayName: Uncompressed ROCm size
   - task: ArchiveFiles@2
     displayName: Compress rocm-nightly
     inputs:
-      rootFolderOrFile: /mnt/rocm
+      rootFolderOrFile: $(Agent.BuildDirectory)/rocm
       includeRootFolder: false
       archiveType: tar
       tarCompression: gz
-      archiveFile: $(Build.ArtifactStagingDirectory)/$(Build.DefinitionName)_$(Build.BuildNumber)_ubuntu2204.tar.gz
+      archiveFile: $(Build.ArtifactStagingDirectory)/$(Build.DefinitionName)_$(Build.BuildNumber)_ubuntu2204_$(JOB_GPU_TARGET).tar.gz
   - script: du -sh $(Build.ArtifactStagingDirectory)
     displayName: Compressed ROCm size
   - task: PublishPipelineArtifact@1

.azuredevops/patches/pytorch/0001-ROCm-CI-patches.patch

@@ -1,40 +0,0 @@
-From b2d3c88f7a8b179e814e72f76f27e25c82659200 Mon Sep 17 00:00:00 2001
-From: Joseph Macaranas <Joseph.Macaranas@amd.com>
-Date: Tue, 30 Jul 2024 05:43:06 -0400
-Subject: [PATCH] ROCm CI patches
-
----
- manywheel/build_common.sh | 9 ---------
- 1 file changed, 9 deletions(-)
-
-diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh
-index 08ca924..52c468f 100644
---- a/manywheel/build_common.sh
-+++ b/manywheel/build_common.sh
-@@ -475,11 +475,9 @@ if [[ -z "$BUILD_PYTHONLESS" ]]; then
-   fi
- 
-   pip uninstall -y "$TORCH_PACKAGE_NAME"
--  
-   if [[ "$USE_SPLIT_BUILD" == "true" ]]; then
-     pip install "$TORCH_NO_PYTHON_PACKAGE_NAME" --no-index -f /$WHEELHOUSE_DIR --no-dependencies -v
-   fi
--  
-   pip install "$TORCH_PACKAGE_NAME" --no-index -f /$WHEELHOUSE_DIR --no-dependencies -v
- 
-   # Print info on the libraries installed in this wheel
-@@ -491,11 +489,4 @@ if [[ -z "$BUILD_PYTHONLESS" ]]; then
-       ldd "$installed_lib" || true
-   done
- 
--  # Run the tests
--  echo "$(date) :: Running tests"
--  pushd "$PYTORCH_ROOT"
--  LD_LIBRARY_PATH=/usr/local/nvidia/lib64 \
--          "${SOURCE_DIR}/../run_tests.sh" manywheel "${py_majmin}" "$DESIRED_CUDA"
--  popd
--  echo "$(date) :: Finished tests"
- fi
--- 
-2.44.0.windows.1
-

.azuredevops/tag-builds/aomp.yml

@@ -9,16 +9,26 @@ parameters:
 
 resources:
   repositories:
-  - repository: release_repo
+  - repository: aomp_repo
     type: github
     endpoint: ROCm
     name: ROCm/aomp
     ref: ${{ parameters.checkoutRef }}
+  - repository: aomp-extras_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp-extras
+    ref: ${{ parameters.checkoutRef }}
+  - repository: flang_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/flang
+    ref: ${{ parameters.checkoutRef }}
   - repository: llvm-project_repo
     type: github
     endpoint: ROCm
     name: ROCm/llvm-project
-    ref: ${{ parameters.checkoutRef }}
+    ref: amd-staging
 
 trigger: none
 pr: none
@@ -26,5 +36,5 @@ pr: none
 jobs:
   - template: ${{ variables.CI_COMPONENT_PATH }}/aomp.yml
     parameters:
-      checkoutRepo: release_repo
+      checkoutRepo: aomp_repo
       checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/hip-tests.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/hip-tests
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/hip-tests.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocPyDecode.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocPyDecode
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocPyDecode.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/templates/steps/artifact-download.yml

@@ -25,10 +25,10 @@ parameters:
     amdsmi: develop
     aomp-extras: aomp-dev
     aomp: aomp-dev
-    clr: develop
+    clr: amd-staging
     composable_kernel: develop
     half: rocm
-    HIP: develop
+    HIP: amd-staging
     hipBLAS: develop
     hipBLASLt: develop
     hipBLAS-common: develop
@@ -64,16 +64,24 @@ parameters:
     rocPRIM: develop
     rocprofiler-register: amd-mainline
     rocprofiler: amd-staging
-    ROCR-Runtime: master
+    ROCR-Runtime: amd-staging
     rocRAND: develop
     rocr_debug_agent: amd-staging
     rocSOLVER: develop
     rocSPARSE: develop
-    ROCT-Thunk-Interface: master
     rocThrust: develop
     roctracer: amd-staging
     rocWMMA: develop
-    rpp: master
+    rpp: develop
+- name: allowPartiallySucceededBuilds
+  type: object
+  default:
+    - rocm-cmake
+    - amdsmi
+    - HIPIFY
+    - rocm_smi_lib
+    - rocFFT
+    - MIVisionX
 # BELOW REQUIRED IF useDefaultBranch false
 - name: branchName
   type: string
@@ -89,13 +97,13 @@ steps:
     specificBuildWithTriggering: true
     itemPattern: '**/*${{ parameters.fileFilter }}*'
     ${{ if eq(parameters.latestFromBranch, true) }}:
-      ${{ if notIn(parameters.componentName, 'aomp', 'clr', 'rocMLIR') }}: # remove this once these pipelines are functional + up-to-date
+      ${{ if notIn(parameters.componentName, 'aomp') }}: # remove this once these pipelines are functional + up-to-date
         buildVersionToDownload: latestFromBranch # default is 'latest'
     ${{ if eq(parameters.useDefaultBranch, true) }}:
       branchName: refs/heads/${{ parameters.defaultBranchList[parameters.componentName] }}
     ${{ else }}:
       branchName: ${{ parameters.branchName }}
-    ${{ if in(parameters.componentName, 'rocm-cmake') }}:
+    ${{ if containsValue(parameters.allowPartiallySucceededBuilds, parameters.componentName) }}:
       allowPartiallySucceededBuilds: true
     targetPath: '$(Pipeline.Workspace)/d'
 - task: ExtractFiles@1

.azuredevops/templates/steps/build-cmake.yml

@@ -14,6 +14,9 @@ parameters:
 - name: cmakeSourceDir
   type: string
   default: '..'
+- name: customBuildTarget
+  type: string
+  default: ''
 - name: cmakeTarget
   type: string
   default: 'install'
@@ -49,7 +52,10 @@ steps:
   displayName: '${{parameters.componentName }} Build'
   inputs:
     workingDirectory: ${{ parameters.cmakeBuildDir }}
-    cmakeArgs: '--build ${{ parameters.cmakeTargetDir }} ${{ parameters.multithreadFlag }}'
+    ${{ if eq(parameters.customBuildTarget, '') }}:
+      cmakeArgs: '--build ${{ parameters.cmakeTargetDir }} ${{ parameters.multithreadFlag }}'
+    ${{ else }}:
+      cmakeArgs: '--build ${{ parameters.cmakeTargetDir }} --target ${{ parameters.customBuildTarget }} ${{ parameters.multithreadFlag }}'
     retryCountOnTaskFailure: 10
 - script: df -h
   displayName: Disk space after build

.azuredevops/templates/steps/ccache.yml

@@ -1,11 +1,14 @@
 # to use ccache, run this template step before build-cmake.yml
 # and add this cmake flag to extraBuildFlags
 # -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
 steps:
 - task: Cache@2
   displayName: Ccache caching
   inputs:
     key: 'ccache | "$(Agent.OS)" | $(Build.SourceVersion)'
-    path: $(CCACHE_DIR)
+    path: ${{ variables.CCACHE_DIR }}
     restoreKeys: |
       ccache | "$(Agent.OS)"

.azuredevops/templates/steps/dependencies-aocl.yml

@@ -0,0 +1,27 @@
+parameters:
+- name: repositoryUrl
+  type: string
+  default: https://download.amd.com/developer/eula/aocl/aocl-4-2
+- name: packageName
+  type: string
+  default: aocl-linux-gcc-4.2.0_1_amd64.deb
+
+steps:
+- task: Bash@3
+  displayName: Download AOCL
+  inputs:
+    targetType: inline
+    workingDirectory: $(Pipeline.Workspace)
+    script: wget -nv ${{ parameters.repositoryUrl }}/${{ parameters.packageName }}
+- task: Bash@3
+  displayName: Install AOCL
+  inputs:
+    targetType: inline
+    workingDirectory: $(Pipeline.Workspace)
+    script: sudo apt install -y ./${{ parameters.packageName }}
+- task: Bash@3
+  displayName: Clean up AOCL
+  inputs:
+    targetType: inline
+    workingDirectory: $(Pipeline.Workspace)
+    script: rm -f ${{ parameters.packageName }}

.azuredevops/templates/steps/dependencies-aqlprofile.yml

@@ -0,0 +1,47 @@
+parameters:
+- name: dependencySource
+  type: string
+  default: staging
+  values:
+    - staging
+    - tag-builds
+- name: repositoryUrl
+  type: object
+  default: 
+    staging: https://repo.radeon.com/rocm/misc/aqlprofile/ubuntu-22.04
+    tag-builds: https://repo.radeon.com/rocm/apt/6.1/pool/main/h/hsa-amd-aqlprofile
+- name: packageName
+  type: object
+  default:
+    staging: hsa-amd-aqlprofile_1.0.0.60200.60200-crdnnh.14213~22.04_amd64.deb
+    tag-builds: hsa-amd-aqlprofile_1.0.0.60100.60100-82~22.04_amd64.deb
+
+steps:
+- task: Bash@3
+  displayName: 'Download aqlprofile'
+  inputs:
+    targetType: inline
+    script: wget -nv ${{ parameters.repositoryUrl[parameters.dependencySource] }}/${{ parameters.packageName[parameters.dependencySource] }}
+    workingDirectory: '$(Pipeline.Workspace)'
+- task: Bash@3
+  displayName: 'Extract aqlprofile'
+  inputs:
+    targetType: inline
+    script: |
+      mkdir hsa-amd-aqlprofile
+      dpkg-deb -R ${{ parameters.packageName[parameters.dependencySource] }} hsa-amd-aqlprofile
+    workingDirectory: '$(Pipeline.Workspace)'
+- task: Bash@3
+  displayName: 'Copy aqlprofile files'
+  inputs:
+    targetType: inline
+    script: |
+      mkdir -p $(Agent.BuildDirectory)/rocm
+      cp -R hsa-amd-aqlprofile/opt/rocm-*/* $(Agent.BuildDirectory)/rocm
+    workingDirectory: '$(Pipeline.Workspace)'
+- task: Bash@3
+  displayName: 'Clean up aqlprofile'
+  inputs:
+    targetType: inline
+    script: rm -rf hsa-amd-aqlprofile ${{ parameters.packageName[parameters.dependencySource] }}
+    workingDirectory: '$(Pipeline.Workspace)'

.azuredevops/templates/steps/dependencies-cmake-latest.yml

@@ -5,6 +5,6 @@ steps:
   inputs:
     targetType: inline
     script: |
-      sudo apt purge cmake
+      sudo apt purge cmake -y
       sudo snap install cmake --classic
       hash -r

.azuredevops/templates/steps/dependencies-other.yml

@@ -12,31 +12,23 @@ steps:
   displayName: 'sudo apt-get update'
   inputs:
     targetType: inline
-    script: sudo apt-get --yes update
-  env:
-    DEBIAN_FRONTEND: noninteractive
+    script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes update
 - task: Bash@3
   displayName: 'sudo apt-get upgrade'
   inputs:
     targetType: inline
-    script: sudo apt-get --yes upgrade
-  env:
-    DEBIAN_FRONTEND: noninteractive
+    script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes upgrade
 - task: Bash@3
   displayName: 'sudo apt-get fix'
   inputs:
     targetType: inline
-    script: sudo apt --yes --fix-broken install
-  env:
-    DEBIAN_FRONTEND: noninteractive
+    script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes --fix-broken install
 - ${{ if gt(length(parameters.aptPackages), 0) }}:
   - task: Bash@3
     displayName: 'sudo apt-get install ...'
     inputs:
       targetType: inline
-      script: sudo apt-get --yes install ${{ join(' ', parameters.aptPackages) }}
-    env:
-      DEBIAN_FRONTEND: noninteractive
+      script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install ${{ join(' ', parameters.aptPackages) }}
 - ${{ if gt(length(parameters.pipModules), 0) }}:
   - task: Bash@3
     displayName: 'pip install  ...'

.azuredevops/templates/steps/dependencies-rocm.yml

@@ -29,120 +29,125 @@ parameters:
 - name: stagingPipelineIdentifiers
   type: object
   default:
-    AMDMIGraphX: $(amdmigraphx-pipeline-id)
-    amdsmi: $(amdsmi-pipeline-id)
-    aomp-extras: $(aomp-extras-pipeline-id)
-    aomp: $(aomp-pipeline-id)
-    clr: $(clr-pipeline-id)
-    composable_kernel: $(composable-kernel-pipeline-id)
-    half: $(half-pipeline-id)
-    HIP: $(hip-pipeline-id)
-    hipBLAS: $(hipblas-pipeline-id)
-    hipBLASLt: $(hipblaslt-pipeline-id)
-    hipBLAS-common: $(hipblas-common-pipeline-id)
-    hipCUB: $(hipcub-pipeline-id)
-    hipFFT: $(hipfft-pipeline-id)
-    hipfort: $(hipfort-pipeline-id)
-    HIPIFY: $(hipify-pipeline-id)
-    hipRAND: $(hiprand-pipeline-id)
-    hipSOLVER: $(hipsolver-pipeline-id)
-    hipSPARSE: $(hipsparse-pipeline-id)
-    hipSPARSELt: $(hipsparselt-pipeline-id)
-    hipTensor: $(hiptensor-pipeline-id)
-    llvm-project: $(llvm-project-pipeline-id)
-    MIOpen: $(miopen-pipeline-id)
-    MIVisionX: $(mivisionx-pipeline-id)
-    rccl: $(rccl-pipeline-id)
-    rdc: $(rdc-pipeline-id)
-    rocAL: $(rocal-pipeline-id)
-    rocALUTION: $(rocalution-pipeline-id)
-    rocBLAS: $(rocblas-pipeline-id)
-    ROCdbgapi : $(rocdbgapi-pipeline-id)
-    rocDecode: $(rocdecode-pipeline-id)
-    rocFFT: $(rocfft-pipeline-id)
-    ROCgdb: $(rocgdb-pipeline-id)
-    rocm-cmake: $(rocm-cmake-pipeline-id)
-    rocm-core: $(rocm-core-pipeline-id)
-    rocm-examples: $(rocm-examples-pipeline-id)
-    rocminfo: $(rocminfo-pipeline-id)
-    rocMLIR: $(rocmlir-pipeline-id)
-    ROCmValidationSuite: $(rocmvalidationsuite-pipeline-id)
-    rocm_bandwidth_test: $(rocm-bandwidth-test-pipeline-id)
-    rocm_smi_lib: $(rocm-smi-lib-pipeline-id)
-    rocPRIM: $(rocprim-pipeline-id)
-    rocprofiler-register: $(rocprofiler-register-pipeline-id)
-    rocprofiler: $(rocprofiler-pipeline-id)
-    ROCR-Runtime: $(rocr-runtime-pipeline-id)
-    rocRAND: $(rocrand-pipeline-id)
-    rocr_debug_agent: $(rocr-debug-agent-pipeline-id)
-    rocSOLVER: $(rocsolver-pipeline-id)
-    rocSPARSE: $(rocsparse-pipeline-id)
-    ROCT-Thunk-Interface: $(roct-thunk-interface-pipeline-id)
-    rocThrust: $(rocthrust-pipeline-id)
-    roctracer: $(roctracer-pipeline-id)
-    rocWMMA: $(rocwmma-pipeline-id)
-    rpp: $(rpp-pipeline-id)
+    AMDMIGraphX: $(AMDMIGRAPHX_PIPELINE_ID)
+    amdsmi: $(AMDSMI_PIPELINE_ID)
+    aomp-extras: $(AOMP_EXTRAS_PIPELINE_ID)
+    aomp: $(AOMP_PIPELINE_ID)
+    clr: $(CLR_PIPELINE_ID)
+    composable_kernel: $(COMPOSABLE_KERNEL_PIPELINE_ID)
+    half: $(HALF_PIPELINE_ID)
+    HIP: $(HIP_PIPELINE_ID)
+    hipBLAS: $(HIPBLAS_PIPELINE_ID)
+    hipBLAS-common: $(HIPBLAS_COMMON_PIPELINE_ID)
+    hipBLASLt: $(HIPBLASLT_PIPELINE_ID)
+    hipCUB: $(HIPCUB_PIPELINE_ID)
+    hipFFT: $(HIPFFT_PIPELINE_ID)
+    hipfort: $(HIPFORT_PIPELINE_ID)
+    HIPIFY: $(HIPIFY_PIPELINE_ID)
+    hipRAND: $(HIPRAND_PIPELINE_ID)
+    hipSOLVER: $(HIPSOLVER_PIPELINE_ID)
+    hipSPARSE: $(HIPSPARSE_PIPELINE_ID)
+    hipSPARSELt: $(HIPSPARSELT_PIPELINE_ID)
+    hipTensor: $(HIPTENSOR_PIPELINE_ID)
+    llvm-project: $(LLVM_PROJECT_PIPELINE_ID)
+    MIOpen: $(MIOpen_PIPELINE_ID)
+    MIVisionX: $(MIVISIONX_PIPELINE_ID)
+    rccl: $(RCCL_PIPELINE_ID)
+    rdc: $(RDC_PIPELINE_ID)
+    rocAL: $(ROCAL_PIPELINE_ID)
+    rocALUTION: $(ROCALUTION_PIPELINE_ID)
+    rocBLAS: $(ROCBLAS_PIPELINE_ID)
+    ROCdbgapi: $(ROCDBGAPI_PIPELINE_ID)
+    rocDecode: $(ROCDECODE_PIPELINE_ID)
+    rocFFT: $(ROCFFT_PIPELINE_ID)
+    ROCgdb: $(ROCGDB_PIPELINE_ID)
+    rocm-cmake: $(ROCM_CMAKE_PIPELINE_ID)
+    rocm-core: $(ROCM_CORE_PIPELINE_ID)
+    rocm-examples: $(ROCM_EXAMPLES_PIPELINE_ID)
+    rocminfo: $(ROCMINFO_PIPELINE_ID)
+    rocMLIR: $(ROCMLIR_PIPELINE_ID)
+    ROCmValidationSuite: $(ROCMVALIDATIONSUITE_PIPELINE_ID)
+    rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_PIPELINE_ID)
+    rocm_smi_lib: $(ROCM_SMI_LIB_PIPELINE_ID)
+    rocPRIM: $(ROCPRIM_PIPELINE_ID)
+    rocprofiler-register: $(ROCPROFILER_REGISTER_PIPELINE_ID)
+    rocprofiler: $(ROCPROFILER_PIPELINE_ID)
+    ROCR-Runtime: $(ROCR_RUNTIME_PIPELINE_ID)
+    rocRAND: $(ROCRAND_PIPELINE_ID)
+    rocr_debug_agent: $(ROCR_DEBUG_AGENT_PIPELINE_ID)
+    rocSOLVER: $(ROCSOLVER_PIPELINE_ID)
+    rocSPARSE: $(ROCSPARSE_PIPELINE_ID)
+    ROCT-Thunk-Interface: $(ROCT_THUNK_INTERFACE_PIPELINE_ID)
+    rocThrust: $(ROCTHRUST_PIPELINE_ID)
+    roctracer: $(ROCTRACER_PIPELINE_ID)
+    rocWMMA: $(ROCWMMA_PIPELINE_ID)
+    rpp: $(RPP_PIPELINE_ID)
 - name: taggedPipelineIdentifiers
   type: object
   default:
-    AMDMIGraphX: $(amdmigraphx-tagged-pipeline-id)
-    amdsmi: $(amdsmi-tagged-pipeline-id)
-    aomp-extras: $(aomp-extras-tagged-pipeline-id)
-    aomp: $(aomp-tagged-pipeline-id)
-    clr: $(clr-tagged-pipeline-id)
-    composable_kernel: $(composable-kernel-tagged-pipeline-id)
-    half: $(half-tagged-pipeline-id)
-    HIP: $(hip-tagged-pipeline-id)
-    hipBLAS: $(hipblas-tagged-pipeline-id)
-    hipBLASLt: $(hipblaslt-tagged-pipeline-id)
-    hipBLAS-common: $(hipblas-common-tagged-pipeline-id)
-    hipCUB: $(hipcub-tagged-pipeline-id)
-    hipFFT: $(hipfft-tagged-pipeline-id)
-    hipfort: $(hipfort-tagged-pipeline-id)
-    HIPIFY: $(hipify-tagged-pipeline-id)
-    hipRAND: $(hiprand-tagged-pipeline-id)
-    hipSOLVER: $(hipsolver-tagged-pipeline-id)
-    hipSPARSE: $(hipsparse-tagged-pipeline-id)
-    hipSPARSELt: $(hipsparselt-tagged-pipeline-id)
-    hipTensor: $(hiptensor-tagged-pipeline-id)
-    llvm-project: $(llvm-project-tagged-pipeline-id)
-    MIOpen: $(miopen-tagged-pipeline-id)
-    MIVisionX: $(mivisionx-tagged-pipeline-id)
-    rccl: $(rccl-tagged-pipeline-id)
-    rdc: $(rdc-tagged-pipeline-id)
-    rocAL: $(rocal-tagged-pipeline-id)
-    rocALUTION: $(rocalution-tagged-pipeline-id)
-    rocBLAS: $(rocblas-tagged-pipeline-id)
-    ROCdbgapi : $(rocdbgapi-tagged-pipeline-id)
-    rocDecode: $(rocdecode-tagged-pipeline-id)
-    rocFFT: $(rocfft-tagged-pipeline-id)
-    ROCgdb: $(rocgdb-tagged-pipeline-id)
-    rocm-cmake: $(rocm-cmake-tagged-pipeline-id)
-    rocm-core: $(rocm-core-tagged-pipeline-id)
-    rocm-examples: $(rocm-examples-tagged-pipeline-id)
-    rocminfo: $(rocminfo-tagged-pipeline-id)
-    rocMLIR: $(rocmlir-tagged-pipeline-id)
-    ROCmValidationSuite: $(rocmvalidationsuite-tagged-pipeline-id)
-    rocm_bandwidth_test: $(rocm-bandwidth-test-tagged-pipeline-id)
-    rocm_smi_lib: $(rocm-smi-lib-tagged-pipeline-id)
-    rocPRIM: $(rocprim-tagged-pipeline-id)
-    rocprofiler-register: $(rocprofiler-register-tagged-pipeline-id)
-    rocprofiler: $(rocprofiler-tagged-pipeline-id)
-    ROCR-Runtime: $(rocr-runtime-tagged-pipeline-id)
-    rocRAND: $(rocrand-tagged-pipeline-id)
-    rocr_debug_agent: $(rocr-debug-agent-tagged-pipeline-id)
-    rocSOLVER: $(rocsolver-tagged-pipeline-id)
-    rocSPARSE: $(rocsparse-tagged-pipeline-id)
-    ROCT-Thunk-Interface: $(roct-thunk-interface-tagged-pipeline-id)
-    rocThrust: $(rocthrust-tagged-pipeline-id)
-    roctracer: $(roctracer-tagged-pipeline-id)
-    rocWMMA: $(rocwmma-tagged-pipeline-id)
-    rpp: $(rpp-tagged-pipeline-id)
+    AMDMIGraphX: $(AMDMIGRAPHX_TAGGED_PIPELINE_ID)
+    amdsmi: $(AMDSMI_TAGGED_PIPELINE_ID)
+    aomp-extras: $(AOMP_EXTRAS_TAGGED_PIPELINE_ID)
+    aomp: $(AOMP_TAGGED_PIPELINE_ID)
+    clr: $(CLR_TAGGED_PIPELINE_ID)
+    composable_kernel: $(COMPOSABLE_KERNEL_TAGGED_PIPELINE_ID)
+    half: $(HALF_TAGGED_PIPELINE_ID)
+    HIP: $(HIP_TAGGED_PIPELINE_ID)
+    hipBLAS: $(HIPBLAS_TAGGED_PIPELINE_ID)
+    hipBLAS-common: $(HIPBLAS_COMMON_TAGGED_PIPELINE_ID)
+    hipBLASLt: $(HIPBLASLT_TAGGED_PIPELINE_ID)
+    hipCUB: $(HIPCUB_TAGGED_PIPELINE_ID)
+    hipFFT: $(HIPFFT_TAGGED_PIPELINE_ID)
+    hipfort: $(HIPFORT_TAGGED_PIPELINE_ID)
+    HIPIFY: $(HIPIFY_TAGGED_PIPELINE_ID)
+    hipRAND: $(HIPRAND_TAGGED_PIPELINE_ID)
+    hipSOLVER: $(HIPSOLVER_TAGGED_PIPELINE_ID)
+    hipSPARSE: $(HIPSPARSE_TAGGED_PIPELINE_ID)
+    hipSPARSELt: $(HIPSPARSELT_TAGGED_PIPELINE_ID)
+    hipTensor: $(HIPTENSOR_TAGGED_PIPELINE_ID)
+    llvm-project: $(LLVM_PROJECT_TAGGED_PIPELINE_ID)
+    MIOpen: $(MIOpen_TAGGED_PIPELINE_ID)
+    MIVisionX: $(MIVISIONX_TAGGED_PIPELINE_ID)
+    rccl: $(RCCL_TAGGED_PIPELINE_ID)
+    rdc: $(RDC_TAGGED_PIPELINE_ID)
+    rocAL: $(ROCAL_TAGGED_PIPELINE_ID)
+    rocALUTION: $(ROCALUTION_TAGGED_PIPELINE_ID)
+    rocBLAS: $(ROCBLAS_TAGGED_PIPELINE_ID)
+    ROCdbgapi: $(ROCDBGAPI_TAGGED_PIPELINE_ID)
+    rocDecode: $(ROCDECODE_TAGGED_PIPELINE_ID)
+    rocFFT: $(ROCFFT_TAGGED_PIPELINE_ID)
+    ROCgdb: $(ROCGDB_TAGGED_PIPELINE_ID)
+    rocm-cmake: $(ROCM_CMAKE_TAGGED_PIPELINE_ID)
+    rocm-core: $(ROCM_CORE_TAGGED_PIPELINE_ID)
+    rocm-examples: $(ROCM_EXAMPLES_TAGGED_PIPELINE_ID)
+    rocminfo: $(ROCMINFO_TAGGED_PIPELINE_ID)
+    rocMLIR: $(ROCMLIR_TAGGED_PIPELINE_ID)
+    ROCmValidationSuite: $(ROCMVALIDATIONSUITE_TAGGED_PIPELINE_ID)
+    rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_TAGGED_PIPELINE_ID)
+    rocm_smi_lib: $(ROCM_SMI_LIB_TAGGED_PIPELINE_ID)
+    rocPRIM: $(ROCPRIM_TAGGED_PIPELINE_ID)
+    rocprofiler-register: $(ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID)
+    rocprofiler: $(ROCPROFILER_TAGGED_PIPELINE_ID)
+    ROCR-Runtime: $(ROCR_RUNTIME_TAGGED_PIPELINE_ID)
+    rocRAND: $(ROCRAND_TAGGED_PIPELINE_ID)
+    rocr_debug_agent: $(ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID)
+    rocSOLVER: $(ROCSOLVER_TAGGED_PIPELINE_ID)
+    rocSPARSE: $(ROCSPARSE_TAGGED_PIPELINE_ID)
+    ROCT-Thunk-Interface: $(ROCT_THUNK_INTERFACE_TAGGED_PIPELINE_ID)
+    rocThrust: $(ROCTHRUST_TAGGED_PIPELINE_ID)
+    roctracer: $(ROCTRACER_TAGGED_PIPELINE_ID)
+    rocWMMA: $(ROCWMMA_TAGGED_PIPELINE_ID)
+    rpp: $(RPP_TAGGED_PIPELINE_ID)
 # set to true if you're calling this template file multiple files in same pipeline
 # only leave last call false to optimize sequence
 - name: skipLibraryLinking
   type: boolean
   default: false
+# set to true if llvm-project is not downloaded in a particular call
+# or if you just don't want the symlink
+- name: skipLlvmSymlink
+  type: boolean
+  default: false
 # some ROCm components can specify GPU target and this will affect downloads
 - name: gpuTarget
   type: string
@@ -229,8 +234,23 @@ steps:
       pipelineId: ${{ parameters.fixedPipelineIdentifier }}
       latestFromBranch: false
       extractToMnt: ${{ parameters.extractToMnt }}
+# Set link to redirect llvm folder
+- ${{ if eq(parameters.skipLlvmSymlink, false) }}:
+  - task: Bash@3
+    displayName: Symlink from rocm/llvm to rocm/lib/llvm
+    inputs:
+      targetType: inline
+      script: sudo ln -s $(Agent.BuildDirectory)/rocm/llvm $(Agent.BuildDirectory)/rocm/lib/llvm
+  - task: Bash@3
+    displayName: Symlink executables from rocm/llvm/bin to rocm/bin
+    inputs:
+      targetType: inline
+      script: |
+        for file in amdclang amdclang++ amdclang-cl amdclang-cpp amdflang amdlld aompcc mygpu mycpu offload-arch; do
+          sudo ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/$file $(Agent.BuildDirectory)/rocm/bin/$file
+        done
 - task: Bash@3
-  displayName: 'list downloaded ROCm files'
+  displayName: 'List downloaded ROCm files'
   inputs:
     targetType: inline
     ${{ if eq(parameters.extractToMnt, true) }}:
@@ -239,21 +259,21 @@ steps:
       script: ls -1R $(Agent.BuildDirectory)/rocm
 - ${{ if eq(parameters.skipLibraryLinking, false) }}:
   - task: Bash@3
-    displayName: 'link ROCm shared libraries'
+    displayName: 'Link ROCm shared libraries'
     inputs:
       targetType: inline
 # OS ignores if the ROCm lib folder shows up more than once
       ${{ if eq(parameters.extractToMnt, true) }}:
         script: |
-          echo /mnt/rocm/lib | sudo tee -a /etc/ld.so.conf
-          echo /mnt/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf
-          sudo cat /etc/ld.so.conf
+          echo /mnt/rocm/lib | sudo tee /etc/ld.so.conf.d/rocm-ci.conf
+          echo /mnt/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
+          sudo cat /etc/ld.so.conf.d/rocm-ci.conf
           sudo ldconfig -v
           ldconfig -p
       ${{ else }}:
         script: |
-          echo $(Agent.BuildDirectory)/rocm/lib | sudo tee -a /etc/ld.so.conf
-          echo $(Agent.BuildDirectory)/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf
-          sudo cat /etc/ld.so.conf
+          echo $(Agent.BuildDirectory)/rocm/lib | sudo tee /etc/ld.so.conf.d/rocm-ci.conf
+          echo $(Agent.BuildDirectory)/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
+          sudo cat /etc/ld.so.conf.d/rocm-ci.conf
           sudo ldconfig -v
           ldconfig -p

.azuredevops/templates/steps/local-artifact-download.yml

@@ -0,0 +1,45 @@
+parameters:
+  - name: gpuTarget
+    type: string
+    default: ''
+  # Set buildType to specific to download artifacts from previous builds, useful for saving time when debugging
+  - name: buildType
+    type: string
+    default: current
+    values:
+      - current
+      - specific
+  # Must be set if buildType == specific
+  # Set definitionId to the pipeline ID and buildId to the specific build ID
+  - name: definitionId
+    type: string
+    default: 0
+  - name: buildId
+    type: string
+    default: 0
+
+steps:
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Build'
+    inputs:
+      ${{ if eq(parameters.buildType, 'specific') }}:
+        buildType: specific
+        buildVersionToDownload: specific
+        project: ROCm-CI
+        definition: ${{ parameters.definitionId }}
+        buildId: ${{ parameters.buildId }}
+      itemPattern: '**/*${{ parameters.gpuTarget }}*'
+      targetPath: $(System.ArtifactsDirectory)
+  - task: ExtractFiles@1
+    displayName: 'Extract Pipeline Build'
+    inputs:
+      archiveFilePatterns: '$(System.ArtifactsDirectory)/**/*.tar.gz'
+      destinationFolder: '$(Agent.BuildDirectory)/rocm'
+      cleanDestinationFolder: false
+      overwriteExistingFiles: true
+  - task: DeleteFiles@1
+    displayName: 'Clean up Compressed Pipeline Build'
+    inputs:
+      SourceFolder: '$(System.ArtifactsDirectory)'
+      Contents: '/**/*.tar.xz'
+      RemoveDotFiles: true

.azuredevops/templates/steps/test.yml

@@ -10,15 +10,36 @@ parameters:
   default: 'ctest'
 - name: testParameters
   type: string
-  default: '--output-on-failure --force-new-ctest-process --output-junit test_output.xml'
+  default: '-VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml'
 - name: testOutputFile
   type: string
   default: test_output.xml
 - name: testOutputFormat
   type: string
-  default: 'junit'
+  default: 'JUnit'
+  values:
+    - JUnit
+    - NUnit
+    - VSTest
+    - XUnit
+    - CTest
+- name: testPublishResults
+  type: boolean
+  default: true
+- name: reloadAMDGPU
+  type: boolean
+  default: false
 
 steps:
+# Avoids occasional AMDGPU driver issues with opening /dev/kfd
+- ${{ if parameters.reloadAMDGPU }}:
+  - task: Bash@3
+    displayName: Unload and reload AMDGPU
+    inputs:
+      targetType: inline
+      script: |
+        sudo modprobe -r amdgpu
+        sudo modprobe amdgpu
 # run test, continue on failure to publish results
 # and to publish build artifacts
 - task: Bash@3
@@ -28,9 +49,11 @@ steps:
     targetType: inline
     script: ${{ parameters.testExecutable }} ${{ parameters.testParameters }}
     workingDirectory: ${{ parameters.testDir }}
-- task: PublishTestResults@2
-  displayName: '${{ parameters.componentName }} Publish Results'
-  inputs:
-    testResultsFormat: ${{ parameters.testOutputFormat }}
-    testResultsFiles: '**/${{ parameters.testOutputFile }}'
-  condition: succeededOrFailed()
+- ${{ if parameters.testPublishResults }}:
+  - task: PublishTestResults@2
+    displayName: '${{ parameters.componentName }} Publish Results'
+    inputs:
+      searchFolder: ${{ parameters.testDir }}
+      testResultsFormat: ${{ parameters.testOutputFormat }}
+      testResultsFiles: '**/${{ parameters.testOutputFile }}'
+    condition: succeededOrFailed()

.azuredevops/variables-global.yml

@@ -23,6 +23,8 @@ variables:
   value: rocm-ci_build_pool
 - name: LARGE_DISK_BUILD_POOL
   value: rocm-ci_larger_base_disk_pool
+- name: GFX942_TEST_POOL
+  value: gfx942_test_pool
 - name: LATEST_RELEASE_TAG
   value: rocm-6.1.0
 - name: DOCKER_IMAGE_NAME
@@ -31,3 +33,275 @@ variables:
   value: 6.1
 - name: KEYRING_VERSION
   value: 6.1
+- name: AMDMIGRAPHX_GFX942_TEST_PIPELINE_ID
+  value: 197
+- name: AMDMIGRAPHX_PIPELINE_ID
+  value: 113
+- name: AMDMIGRAPHX_TAGGED_PIPELINE_ID
+  value: 60
+- name: AMDSMI_PIPELINE_ID
+  value: 99
+- name: AMDSMI_TAGGED_PIPELINE_ID
+  value: 33
+- name: AOMP_EXTRAS_PIPELINE_ID
+  value: 111
+- name: AOMP_EXTRAS_TAGGED_PIPELINE_ID
+  value: 75
+- name: AOMP_PIPELINE_ID
+  value: 115
+- name: AOMP_TAGGED_PIPELINE_ID
+  value: 76
+- name: CCACHE_DIR
+  value: $(Pipeline.Workspace)/ccache
+- name: CLR_PIPELINE_ID
+  value: 145
+- name: CLR_TAGGED_PIPELINE_ID
+  value: 71
+- name: COMPOSABLE_KERNEL_GFX942_TEST_PIPELINE_ID
+  value: 179
+- name: COMPOSABLE_KERNEL_PIPELINE_ID
+  value: 86
+- name: COMPOSABLE_KERNEL_TAGGED_PIPELINE_ID
+  value: 38
+- name: FLANG_LEGACY_PIPELINE_ID
+  value: 77
+- name: FLANG_LEGACY_TAGGED_PIPELINE_ID
+  value: 77
+- name: HALF_PIPELINE_ID
+  value: 101
+- name: HALF_TAGGED_PIPELINE_ID
+  value: 11
+- name: HALF560_PIPELINE_ID
+  value: 66
+- name: HALF560_TAGGED_PIPELINE_ID
+  value: 66
+- name: HIP_PIPELINE_ID
+  value: 93
+- name: HIP_TAGGED_PIPELINE_ID
+  value: 31
+- name: HIPBLAS_COMMON_PIPELINE_ID
+  value: 223
+- name: HIPBLAS_COMMON_TAGGED_PIPELINE_ID
+  value: 224
+- name: HIPBLAS_GFX942_TEST_PIPELINE_ID
+  value: 202
+- name: HIPBLAS_PIPELINE_ID
+  value: 87
+- name: HIPBLAS_TAGGED_PIPELINE_ID
+  value: 44
+- name: HIPBLASLT_GFX942_TEST_PIPELINE_ID
+  value: 187
+- name: HIPBLASLT_PIPELINE_ID
+  value: 112
+- name: HIPBLASLT_TAGGED_PIPELINE_ID
+  value: 45
+- name: HIPCUB_GFX942_TEST_PIPELINE_ID
+  value: 186
+- name: HIPCUB_PIPELINE_ID
+  value: 97
+- name: HIPCUB_TAGGED_PIPELINE_ID
+  value: 46
+- name: HIPFFT_GFX942_TEST_PIPELINE_ID
+  value: 198
+- name: HIPFFT_PIPELINE_ID
+  value: 121
+- name: HIPFFT_TAGGED_PIPELINE_ID
+  value: 12
+- name: HIPFORT_PIPELINE_ID
+  value: 102
+- name: HIPFORT_TAGGED_PIPELINE_ID
+  value: 34
+- name: HIPIFY_PIPELINE_ID
+  value: 92
+- name: HIPIFY_TAGGED_PIPELINE_ID
+  value: 13
+- name: HIPRAND_GFX942_TEST_PIPELINE_ID
+  value: 188
+- name: HIPRAND_PIPELINE_ID
+  value: 90
+- name: HIPRAND_TAGGED_PIPELINE_ID
+  value: 42
+- name: HIPSOLVER_GFX942_TEST_PIPELINE_ID
+  value: 201
+- name: HIPSOLVER_PIPELINE_ID
+  value: 84
+- name: HIPSOLVER_TAGGED_PIPELINE_ID
+  value: 52
+- name: HIPSPARSE_GFX942_TEST_PIPELINE_ID
+  value: 195
+- name: HIPSPARSE_PIPELINE_ID
+  value: 83
+- name: HIPSPARSE_TAGGED_PIPELINE_ID
+  value: 14
+- name: HIPSPARSELT_GFX942_TEST_PIPELINE_ID
+  value: 200
+- name: HIPSPARSELT_PIPELINE_ID
+  value: 104
+- name: HIPSPARSELT_TAGGED_PIPELINE_ID
+  value: 53
+- name: HIPTENSOR_GFX942_TEST_PIPELINE_ID
+  value: 192
+- name: HIPTENSOR_PIPELINE_ID
+  value: 105
+- name: HIPTENSOR_TAGGED_PIPELINE_ID
+  value: 56
+- name: LAST_RELEASE
+  value: 6.1.0
+- name: LLVM_PROJECT_PIPELINE_ID
+  value: 2
+- name: LLVM_PROJECT_TAGGED_PIPELINE_ID
+  value: 8
+- name: MIOPEN_PIPELINE_ID
+  value: 108
+- name: MIOPEN_TAGGED_PIPELINE_ID
+  value: 58
+- name: MIVISIONX_PIPELINE_ID
+  value: 80
+- name: MIVISIONX_TAGGED_PIPELINE_ID
+  value: 18
+- name: RCCL_GFX942_TEST_PIPELINE_ID
+  value: 184
+- name: RCCL_PIPELINE_ID
+  value: 107
+- name: RCCL_TAGGED_PIPELINE_ID
+  value: 15
+- name: RDC_PIPELINE_ID
+  value: 100
+- name: RDC_TAGGED_PIPELINE_ID
+  value: 59
+- name: REIMAGE_ORG
+  value: AGS-ROCm-CI
+- name: REIMAGE_REPO
+  value: cirrascale-reimage-automation
+- name: ROCAL_PIPELINE_ID
+  value: 151
+- name: ROCALUTION_GFX942_TEST_PIPELINE_ID
+  value: 196
+- name: ROCALUTION_PIPELINE_ID
+  value: 89
+- name: ROCALUTION_TAGGED_PIPELINE_ID
+  value: 16
+- name: ROCBLAS_GFX942_TEST_PIPELINE_ID
+  value: 185
+- name: ROCBLAS_PIPELINE_ID
+  value: 85
+- name: ROCBLAS_TAGGED_PIPELINE_ID
+  value: 32
+- name: ROCDBGAPI_PIPELINE_ID
+  value: 135
+- name: ROCDBGAPI_TAGGED_PIPELINE_ID
+  value: 17
+- name: ROCDECODE_PIPELINE_ID
+  value: 79
+- name: ROCDECODE_TAGGED_PIPELINE_ID
+  value: 21
+- name: ROCFFT_GFX942_TEST_PIPELINE_ID
+  value: 189
+- name: ROCFFT_PIPELINE_ID
+  value: 120
+- name: ROCFFT_TAGGED_PIPELINE_ID
+  value: 19
+- name: ROCGDB_PIPELINE_ID
+  value: 134
+- name: ROCGDB_TAGGED_PIPELINE_ID
+  value: 50
+- name: ROCM_BANDWIDTH_TEST_PIPELINE_ID
+  value: 88
+- name: ROCM_BANDWIDTH_TEST_TAGGED_PIPELINE_ID
+  value: 23
+- name: ROCM_CMAKE_PIPELINE_ID
+  value: 6
+- name: ROCM_CMAKE_TAGGED_PIPELINE_ID
+  value: 7
+- name: ROCM_CORE_PIPELINE_ID
+  value: 103
+- name: ROCM_CORE_TAGGED_PIPELINE_ID
+  value: 22
+- name: ROCM_EXAMPLES_GFX942_TEST_PIPELINE_ID
+  value: 204
+- name: ROCM_SMI_LIB_PIPELINE_ID
+  value: 96
+- name: ROCM_SMI_LIB_TAGGED_PIPELINE_ID
+  value: 47
+- name: ROCMINFO_PIPELINE_ID
+  value: 91
+- name: ROCMINFO_TAGGED_PIPELINE_ID
+  value: 27
+- name: ROCMLIR_PIPELINE_ID
+  value: 229
+- name: ROCMLIR_TAGGED_PIPELINE_ID
+  value: 62
+- name: ROCMVALIDATIONSUITE_PIPELINE_ID
+  value: 106
+- name: ROCMVALIDATIONSUITE_TAGGED_PIPELINE_ID
+  value: 43
+- name: ROCPRIM_GFX942_TEST_PIPELINE_ID
+  value: 180
+- name: ROCPRIM_PIPELINE_ID
+  value: 82
+- name: ROCPRIM_TAGGED_PIPELINE_ID
+  value: 20
+- name: ROCPROFILER_GFX942_TEST_PIPELINE_ID
+  value: 190
+- name: ROCPROFILER_PIPELINE_ID
+  value: 143
+- name: ROCPROFILER_REGISTER_PIPELINE_ID
+  value: 1
+- name: ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID
+  value: 25
+- name: ROCPROFILER_TAGGED_PIPELINE_ID
+  value: 28
+- name: ROCR_DEBUG_AGENT_PIPELINE_ID
+  value: 136
+- name: ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID
+  value: 29
+- name: ROCR_RUNTIME_PIPELINE_ID
+  value: 10
+- name: ROCR_RUNTIME_TAGGED_PIPELINE_ID
+  value: 24
+- name: ROCRAND_GFX942_TEST_PIPELINE_ID
+  value: 183
+- name: ROCRAND_PIPELINE_ID
+  value: 95
+- name: ROCRAND_TAGGED_PIPELINE_ID
+  value: 41
+- name: ROCSOLVER_GFX942_TEST_PIPELINE_ID
+  value: 199
+- name: ROCSOLVER_PIPELINE_ID
+  value: 81
+- name: ROCSOLVER_TAGGED_PIPELINE_ID
+  value: 55
+- name: ROCSPARSE_GFX942_TEST_PIPELINE_ID
+  value: 191
+- name: ROCSPARSE_PIPELINE_ID
+  value: 98
+- name: ROCSPARSE_TAGGED_PIPELINE_ID
+  value: 67
+- name: ROCT_THUNK_INTERFACE_PIPELINE_ID
+  value: 3
+- name: ROCT_THUNK_INTERFACE_TAGGED_PIPELINE_ID
+  value: 9
+- name: ROCTHRUST_GFX942_TEST_PIPELINE_ID
+  value: 194
+- name: ROCTHRUST_PIPELINE_ID
+  value: 94
+- name: ROCTHRUST_TAGGED_PIPELINE_ID
+  value: 26
+- name: ROCTRACER_GFX942_TEST_PIPELINE_ID
+  value: 181
+- name: ROCTRACER_PIPELINE_ID
+  value: 141
+- name: ROCTRACER_TAGGED_PIPELINE_ID
+  value: 30
+- name: ROCWMMA_GFX942_TEST_PIPELINE_ID
+  value: 193
+- name: ROCWMMA_PIPELINE_ID
+  value: 109
+- name: ROCWMMA_TAGGED_PIPELINE_ID
+  value: 57
+- name: RPP_GFX942_TEST_PIPELINE_ID
+  value: 182
+- name: RPP_PIPELINE_ID
+  value: 78
+- name: RPP_TAGGED_PIPELINE_ID
+  value: 39

.wordlist.txt

@@ -97,6 +97,7 @@ ENDPGM
 EPYC
 ESXi
 EoS
+FBGEMM
 FFT
 FFTs
 FFmpeg
@@ -110,6 +111,7 @@ Flang
 Fortran
 Fuyu
 GALB
+GCC
 GCD
 GCDs
 GCN
@@ -175,6 +177,7 @@ Interop
 Intersphinx
 Intra
 Ioffe
+Jinja
 JSON
 Jupyter
 KFD
@@ -221,6 +224,7 @@ Megatron
 Mellanox
 Mellanox's
 Meta's
+Miniconda
 MirroredStrategy
 Multicore
 Multithreaded
@@ -430,6 +434,7 @@ accuracies
 activations
 addr
 alloc
+allocatable
 allocator
 allocators
 amdgpu
@@ -619,6 +624,7 @@ performant
 perl
 pragma
 pre
+prebuild
 prebuilt
 precompiled
 preconditioner
@@ -710,8 +716,10 @@ subexpression
 subfolder
 subfolders
 submodule
+submodules
 supercomputing
 symlink
+symlinks
 td
 tensorfloat
 th

README.md

@@ -76,8 +76,8 @@ The Build time will reduce significantly if we limit the GPU Architecture/s agai
 
 mkdir -p ~/WORKSPACE/      # Or any folder name other than WORKSPACE
 cd ~/WORKSPACE/
-export ROCM_VERSION=6.1.0   # or 6.1.1 6.1.2
-~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.1.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
+export ROCM_VERSION=6.2.0
+~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.2.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
 ~/bin/repo sync
 
 # --------------------------------------
@@ -87,9 +87,11 @@ export ROCM_VERSION=6.1.0   # or 6.1.1 6.1.2
 # Option 1: Start a docker container
 # Pulling required base docker images:
 # Ubuntu20.04 built from ROCm/tools/rocm-build/docker/ubuntu20/Dockerfile
-docker pull rocm/rocm-build-ubuntu-20.04:6.1
+docker pull rocm/rocm-build-ubuntu-20.04:6.2
 # Ubuntu22.04 built from ROCm/tools/rocm-build/docker/ubuntu22/Dockerfile
-docker pull rocm/rocm-build-ubuntu-22.04:6.1
+docker pull rocm/rocm-build-ubuntu-22.04:6.2
+# Ubuntu24.04 built from ROCm/tools/rocm-build/docker/ubuntu24/Dockerfile
+docker pull rocm/rocm-build-ubuntu-24.04:6.2
 
 # Start docker container and mount the source code folder:
 docker run -ti \
@@ -108,10 +110,16 @@ docker run -ti \
 # Option 2: Install required packages into the host machine
 # For ubuntu20.04 system
 cd ROCm/tools/rocm-build/docker/ubuntu20
+cp * /tmp && cd /tmp
 bash install-prerequisites.sh
 # For ubuntu22.04 system
 cd ROCm/tools/rocm-build/docker/ubuntu22
+cp * /tmp && cd /tmp
 bash install-prerequisities.sh
+# For ubuntu24.04 system
+cd ROCm/tools/rocm-build/docker/ubuntu24
+cp * /tmp && cd /tmp
+bash install-prerequisites.sh
 
 # --------------------------------------
 # Step 3: Run build command line
@@ -138,13 +146,17 @@ make -f ROCm/tools/rocm-build/ROCm.mk T_rocblas
 out/ubuntu-20.04/20.04/deb/
 # Find built packages in ubuntu22.04:
 out/ubuntu-22.04/22.04/deb/
+# Find built packages in ubuntu24.04:
+out/ubuntu-24.04/24.04/deb/
 
 # Find built logs in ubuntu20.04:
 out/ubuntu-20.04/20.04/logs/
 # Find built logs in ubuntu22.04:
 out/ubuntu-22.04/22.04/logs/
+# Find built logs in ubuntu24.04:
+out/ubuntu-24.04/24.04/logs/
 # All logs pertaining to failed components, end with .errrors extension.
-out/ubuntu-22.04/22.04/logs/rocblas.errors          # Example
+out/ubuntu-22.04/22.04/logs/rocblas.errors      # Example
 # All logs pertaining to building components, end with .inprogress extension.
 out/ubuntu-22.04/22.04/logs/rocblas.inprogress  # Example
 # All logs pertaining to passed components, use the component names.

RELEASE.md

@@ -433,7 +433,7 @@ links in the *Version* column to go to the detailed component changelogs.
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/tensile/en/docs-6.2.0">Tensile</a></td>
+                <td><a href="https://github.com/ROCm/tensile/">Tensile</a></td>
                 <td>4.40.0&nbsp;&Rightarrow;&nbsp;<a href="#tensile-4-41-0">4.41.0</a></td>
                 <td><a href="https://github.com/ROCm/tensile/releases/tag/rocm-6.2.0"><i
                             class="fab fa-github fa-lg"></i></a></td>
@@ -524,7 +524,7 @@ links in the *Version* column to go to the detailed component changelogs.
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/ROCProfiler/en/docs-6.2.0">ROCProfiler</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocprofiler/en/docs-6.2.0/">ROCProfiler</a></td>
                 <td>2.0.0&nbsp;&Rightarrow;&nbsp;<a href="#rocprofiler-2-0-0">2.0.0</a></td>
                 <td><a href="https://github.com/ROCm/ROCProfiler/releases/tag/rocm-6.2.0"><i
                             class="fab fa-github fa-lg"></i></a></td>
@@ -536,7 +536,7 @@ links in the *Version* column to go to the detailed component changelogs.
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr >
-                <td><a href="https://rocm.docs.amd.com/projects/ROCTracer/en/docs-6.2.0">ROCTracer</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/roctracer/en/docs-6.2.0/">ROCTracer</a></td>
                 <td>4.1.0</td>
                 <td><a href="https://github.com/ROCm/ROCTracer/releases/tag/rocm-6.2.0"><i
                             class="fab fa-github fa-lg"></i></a></td>
@@ -546,19 +546,19 @@ links in the *Version* column to go to the detailed component changelogs.
             <tr>
                 <th rowspan="5"></th>
                 <th rowspan="5">Development</th>
-                <td><a href="https://rocm.docs.amd.com/projects/HIPIFY/docs-6.2.0">HIPIFY</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/HIPIFY/en/docs-6.2.0/">HIPIFY</a></td>
                 <td>17.0.0&nbsp;&Rightarrow;&nbsp;<a href="#hipify-18-0-0">18.0.0</a></td>
                 <td><a href="https://github.com/ROCm/HIPIFY/releases/tag/rocm-6.2.0"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
                 <td><a href="https://rocm.docs.amd.com/projects/ROCdbgapi/en/docs-6.2.0">ROCdbgapi</a></td>
-                <td>0.71.0&nbsp;&Rightarrow;&nbsp;<a href="#rocdbgapi-0-75-0">0.75.0</a></td>
+                <td>0.71.0&nbsp;&Rightarrow;&nbsp;<a href="#rocdbgapi-0-76-0">0.76.0</a></td>
                 <td><a href="https://github.com/ROCm/ROCdbgapi/releases/tag/rocm-6.2.0"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocm-cmake/en/docs-6.2.0">ROCm CMake</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/ROCmCMakeBuildTools/en/docs-6.2.0/">ROCm CMake</a></td>
                 <td>0.12.0&nbsp;&Rightarrow;&nbsp;<a href="#rocm-cmake-0-13-0">0.13.0</a></td>
                 <td><a href="https://github.com/ROCm/rocm-cmake/releases/tag/rocm-6.2.0"><i
                             class="fab fa-github fa-lg"></i></a></td>
@@ -989,7 +989,7 @@ on GitHub for more information.
   * `Zihintntl` extension version was upgraded to 1.0 and is no longer experimental.
 
   * Intrinsics were added for `Zk*`, `Zbb`, and `Zbc`. See
-    [Scalar Bit Manipulation Extension Intrinsics](https://github.com/riscv-non-isa/riscv-c-api-doc/blob/master/riscv-c-api.md#scalar-bit-manipulation-extension-intrinsics) in the RISC-V C API specification.
+    [Scalar Bit Manipulation Extension Intrinsics](https://github.com/riscv-non-isa/riscv-c-api-doc/blob/main/src/c-api.adoc#scalar-bit-manipulation-extension-intrinsics) in the RISC-V C API specification.
 
   * Default ABI with F but without D was changed to ilp32f for RV32 and to lp64f for RV64.
 
@@ -1524,7 +1524,7 @@ See [issue #3498](https://github.com/ROCm/ROCm/issues/3498) on GitHub.
 * Deprecated `rocblas_gemm_ex3`, `gemm_batched_ex3` and `gemm_strided_batched_ex3`. They will be removed in the next
   major release of rocBLAS. Refer to [hipBLASLt](https://github.com/ROCm/hipBLASLt) for future 8-bit float usage.
 
-### **ROCdbgapi** (0.75.0)
+### **ROCdbgapi** (0.76.0)
 
 #### Removals
 - Renamed `(AMD_DBGAPI_EXCEPTION_WAVE,AMD_DBGAPI_WAVE_STOP_REASON)_APERTURE_VIOLATION` to `(AMD_DBGAPI_EXCEPTION_WAVE,AMD_DBGAPI_WAVE_STOP_REASON)_ADDRESS_ERROR`.

default.xml

@@ -25,7 +25,6 @@
 <!--HIP Projects-->
     <project name="HIP" />
     <project name="hip-tests" />
-    <project name="HIP-Examples" />
     <project name="HIPIFY" />
     <project name="clr" />
     <project name="hipother" />

docs/compatibility/compatibility-matrix.rst

@@ -39,12 +39,12 @@ You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-
       ,,,
       FRAMEWORK SUPPORT,".. _framework-support-compatibility-matrix:",,
       :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.15, 2.14, 2.13","2.14, 2.13, 2.12"
+      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1"
       :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26
       `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.14.1
       ,,,
       THIRD PARTY COMMS,".. _thirdpartycomms-support-compatibility-matrix:",,
-      `UCC <https://github.com/ROCm/ucc>`_,>=1.2.0,>=1.2.0,>=1.2.0
+      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.2.0
       `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.14.1,>=1.14.1
       ,,,
       THIRD PARTY ALGORITHM,".. _thirdpartyalgorithm-support-compatibility-matrix:",,
@@ -56,8 +56,9 @@ You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-
       :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.9.0,2.8.0
       :doc:`MIOpen <miopen:index>`,3.2.0,3.1.0,3.0.0
       :doc:`MIVisionX <mivisionx:index>`,3.0.0,2.5.0,2.5.0
-      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,N/A
       :doc:`RPP <rpp:index>`,1.8.0,1.5.0,1.4.0
+      :doc:`rocAL <rocal:index>`,1.0.0,1.0.0,1.0.0
+      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,N/A
       :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,N/A,N/A
       ,,,
       COMMUNICATION,".. _commlibs-support-compatibility-matrix:",,
@@ -118,11 +119,13 @@ You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-
       ,,,
       COMPILERS,".. _compilers-support-compatibility-matrix:",,
       `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,0.5.0,0.5.0
+      :doc:`hipCC <hipcc:index>`,1.1.1,1.0.0,1.0.0
       `Flang <https://github.com/ROCm/flang>`_,18.0.0.24232,17.0.0.24193,17.0.0.23483
-      `llvm-project <https://github.com/ROCm/llvm-project>`_,18.0.0.24232,17.0.0.24193,17.0.0.23483
+      :doc:`llvm-project <llvm-project:index>`,18.0.0.24232,17.0.0.24193,17.0.0.23483
       `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24232,17.0.0.24193,17.0.0.23483
       ,,,
       RUNTIMES,".. _runtime-support-compatibility-matrix:",,
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41133,6.1.40093,6.1.32830
       :doc:`HIP <hip:index>`,6.2.41133,6.1.40093,6.1.32830
       `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0
       :doc:`ROCR-Runtime <rocr-runtime:index>`,1.13.0,1.13.0,1.12.0

docs/compatibility/precision-support.rst

@@ -410,7 +410,7 @@ description, refer to the corresponding library data type support page.
         - ❌/❌
         - ❌/❌
       *
-        - rocRAND (:doc:`details <rocrand:data-type-support>`)
+        - rocRAND (:doc:`details <rocrand:api-reference/data-type-support>`)
         - -/✅
         - -/✅
         - -/✅

docs/data/reference/compatibility-matrix-historical-6.0.csv

@@ -22,12 +22,12 @@ ROCm Version,6.2.0, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
       ,,,,,,
       FRAMEWORK SUPPORT,".. _framework-support-compatibility-matrix-past-60:",,,,,
       :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.15, 2.14, 2.13","2.15, 2.14, 2.13","2.15, 2.14, 2.13","2.14, 2.13, 2.12","2.14, 2.13, 2.12"
+      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1","2.14.0, 2.13.1, 2.12.1"
       :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
       `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
       ,,,,,,
       THIRD PARTY COMMS,".. _thirdpartycomms-support-compatibility-matrix-past-60:",,,,,
-      `UCC <https://github.com/ROCm/ucc>`_,>=1.2.0,>=1.2.0,>=1.2.0,>=1.2.0,>=1.2.0,>=1.2.0
+      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
       `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
       ,,,,,,
       THIRD PARTY ALGORITHM,".. _thirdpartyalgorithm-support-compatibility-matrix-past-60:",,,,,
@@ -39,8 +39,9 @@ ROCm Version,6.2.0, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
       :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
       :doc:`MIOpen <miopen:index>`,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
       :doc:`MIVisionX <mivisionx:index>`,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
-      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
       :doc:`RPP <rpp:index>`,1.8.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
+      :doc:`rocAL <rocal:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
       :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,N/A,N/A,N/A,N/A,N/A
       ,,,,,,
       COMMUNICATION,".. _commlibs-support-compatibility-matrix-past-60:",,,,,
@@ -101,11 +102,13 @@ ROCm Version,6.2.0, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
       ,,,,,,
       COMPILERS,".. _compilers-support-compatibility-matrix-past-60:",,,,,
       `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
+      :doc:`hipCC <hipcc:index>`,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
       `Flang <https://github.com/ROCm/flang>`_,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      `llvm-project <https://github.com/ROCm/llvm-project>`_,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`llvm-project <llvm-project:index>`,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
       `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
       ,,,,,,
       RUNTIMES,".. _runtime-support-compatibility-matrix-past-60:",,,,,
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
       :doc:`HIP <hip:index>`,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
       `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
       :doc:`ROCR-Runtime <rocr-runtime:index>`,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0

docs/how-to/llm-fine-tuning-optimization/llm-inference-frameworks.rst

@@ -137,6 +137,12 @@ Installing vLLM
 
 Refer to :ref:`mi300x-vllm-optimization` for performance optimization tips.
 
+ROCm provides a prebuilt optimized Docker image for validating the performance of LLM inference with vLLM 
+on the MI300X accelerator. The Docker image includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 .. _fine-tuning-llms-tgi:
 
 Hugging Face TGI

docs/how-to/llm-fine-tuning-optimization/model-acceleration-libraries.rst

@@ -251,3 +251,287 @@ page describes the options.
 
 Learn more about optimizing kernels with TunableOp in
 :ref:`Optimizing Triton kernels <mi300x-tunableop>`.
+
+
+FBGEMM and FBGEMM_GPU
+=====================
+
+FBGEMM (Facebook General Matrix Multiplication) is a low-precision, high-performance CPU kernel library
+for matrix-matrix multiplications and convolutions. It is used for server-side inference
+and as a back end for PyTorch quantized operators. FBGEMM offers optimized on-CPU performance for reduced precision calculations,
+strong performance on native tensor formats, and the ability to generate
+high-performance shape- and size-specific kernels at runtime.
+
+FBGEMM_GPU collects several high-performance PyTorch GPU operator libraries  
+for use in training and inference. It provides efficient table-batched embedding functionality,
+data layout transformation, and quantization support.
+
+For more information about FBGEMM and FBGEMM_GPU, see the `PyTorch FBGEMM GitHub <https://github.com/pytorch/FBGEMM>`_
+and the `PyTorch FBGEMM documentation <https://pytorch.org/FBGEMM/>`_.
+The `Meta blog post about FBGEMM <https://engineering.fb.com/2018/11/07/ml-applications/fbgemm/>`_ provides
+additional background about the library.
+
+Installing FBGEMM_GPU
+----------------------
+
+Installing FBGEMM_GPU consists of the following steps:
+
+*  Set up an isolated Miniconda environment
+*  Install ROCm using Docker or the :doc:`package manager <rocm-install-on-linux:install/native-install/index>`
+*  Install the nightly `PyTorch <https://pytorch.org/>`_ build
+*  Complete the pre-build and build tasks
+  
+.. note::
+
+   FBGEMM_GPU doesn't require the installation of FBGEMM. To optionally install
+   FBGEMM, see the `FBGEMM install instructions <https://pytorch.org/FBGEMM/fbgemm-development/BuildInstructions.html>`_.
+
+Set up the Miniconda environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To install Miniconda, use the following commands.
+
+#. Install a `Miniconda environment <https://docs.anaconda.com/miniconda/>`_ for reproducible builds.
+   All subsequent commands run inside this environment.
+
+   .. code-block:: shell
+
+      export PLATFORM_NAME="$(uname -s)-$(uname -m)"
+
+      # Set the Miniconda prefix directory
+      miniconda_prefix=$HOME/miniconda
+
+      # Download the Miniconda installer
+      wget -q "https://repo.anaconda.com/miniconda/Miniconda3-latest-${PLATFORM_NAME}.sh" -O miniconda.sh
+
+      # Run the installer
+      bash miniconda.sh -b -p "$miniconda_prefix" -u
+
+      # Load the shortcuts
+      . ~/.bashrc
+
+      # Run updates
+      conda update -n base -c defaults -y conda
+
+#. Create a Miniconda environment with Python 3.12:
+
+   .. code-block:: shell
+
+      env_name=<ENV NAME>
+      python_version=3.12
+
+      # Create the environment
+      conda create -y --name ${env_name} python="${python_version}"
+
+      # Upgrade PIP and pyOpenSSL package
+      conda run -n ${env_name} pip install --upgrade pip
+      conda run -n ${env_name} python -m pip install pyOpenSSL>22.1.0
+
+#. Install additional build tools:
+
+   .. code-block:: shell
+
+      conda install -n ${env_name} -y \
+         click \
+         cmake \
+         hypothesis \
+         jinja2 \
+         make \
+         ncurses \
+         ninja \
+         numpy \
+         scikit-build \
+         wheel
+
+Install the ROCm components
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+FBGEMM_GPU can run in a ROCm Docker container or in conjunction with the full ROCm installation.
+The Docker method is recommended because it requires fewer steps and provides a stable environment.
+
+To run FBGEMM_GPU in the Docker container, pull the `Minimal Docker image for ROCm <https://hub.docker.com/r/rocm/rocm-terminal>`_.
+This image includes all preinstalled ROCm packages required to integrate FBGEMM. To pull
+and run the ROCm Docker image, use this command:
+
+.. code-block:: shell
+
+   # Run for ROCm 6.2.0
+   docker run -it --network=host --shm-size 16G --device=/dev/kfd --device=/dev/dri --group-add video \
+   --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --ipc=host rocm/rocm-terminal:6.2 /bin/bash
+
+.. note::
+
+   The `Full Docker image for ROCm <https://hub.docker.com/r/rocm/dev-ubuntu-20.04>`_, which includes all
+   ROCm packages, can also be used. However, it results in a very large container, so the minimal
+   Docker image is recommended.
+
+You can also install ROCm using the package manager. FBGEMM_GPU requires the installation of the full ROCm package.
+For more information, see :doc:`the ROCm installation guide <rocm-install-on-linux:install/detailed-install>`.
+The ROCm package also requires the :doc:`MIOpen <miopen:index>` component as a dependency. 
+To install MIOpen, use the ``apt install`` command.
+
+.. code-block:: shell
+
+   apt install hipify-clang miopen-hip miopen-hip-dev
+
+Install PyTorch
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Install `PyTorch <https://pytorch.org/>`_ using ``pip`` for the most reliable and consistent results.
+
+#. Install the nightly PyTorch build using ``pip``.
+
+   .. code-block:: shell
+
+      # Install the latest nightly, ROCm variant
+      conda run -n ${env_name} pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/rocm6.2/
+
+#. Ensure PyTorch loads correctly. Verify the version and variant of the installation using an ``import`` test.
+
+   .. code-block:: shell
+
+      # Ensure that the package loads properly
+      conda run -n ${env_name} python -c "import torch.distributed"
+
+      # Verify the version and variant of the installation
+      conda run -n ${env_name} python -c "import torch; print(torch.__version__)"
+
+Perform the prebuild and build
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+#. Clone the FBGEMM repository and the relevant submodules. Use ``pip`` to install the 
+   components in ``requirements.txt``. Run the following commands inside the Miniconda environment.
+
+   .. code-block:: shell
+
+      # Select a version tag
+      FBGEMM_VERSION=v0.8.0
+
+      # Clone the repo along with its submodules
+      git clone https://github.com/pytorch/FBGEMM.git --branch=v0.8.0 --recursive fbgemm_${FBGEMM_VERSION}
+
+      # Install additional required packages for building and testing
+      cd fbgemm_${FBGEMM_VERSION}/fbgemm_gpu
+      pip install requirements.txt
+
+#. Clear the build cache to remove stale build information.
+
+   .. code-block:: shell
+
+      # !! Run in fbgemm_gpu/ directory inside the Conda environment !!
+
+      python setup.py clean
+
+#. Set the wheel build variables, including the package name, Python version tag, and Python platform name.
+
+   .. code-block:: shell
+
+      # Set the package name depending on the build variant
+      export package_name=fbgemm_gpu_rocm
+
+      # Set the Python version tag.  It should follow the convention `py<major><minor>`,
+      # for example, Python 3.12 --> py312
+      export python_tag=py312
+
+      # Determine the processor architecture
+      export ARCH=$(uname -m)
+
+      # Set the Python platform name for the Linux case
+      export python_plat_name="manylinux2014_${ARCH}"
+
+#. Build FBGEMM_GPU for the ROCm platform. Set ``ROCM_PATH`` to the path to your ROCm installation. 
+   Run these commands from the ``fbgemm_gpu/`` directory inside the Miniconda environment.
+
+   .. code-block:: shell
+
+      # !! Run in the fbgemm_gpu/ directory inside the Conda environment !!
+
+      export ROCM_PATH=</path/to/rocm>
+
+      # Build for the target architecture of the ROCm device installed on the machine (for example, 'gfx942;gfx90a')
+      # See :doc:`The Linux system requirements <../../reference/system-requirements>` for a list of supported GPUs.
+      export PYTORCH_ROCM_ARCH=$(${ROCM_PATH}/bin/rocminfo | grep -o -m 1 'gfx.*')
+
+      # Build the wheel artifact only
+      python setup.py bdist_wheel \
+         --package_variant=rocm \
+         --python-tag="${python_tag}" \
+         --plat-name="${python_plat_name}" \
+         -DHIP_ROOT_DIR="${ROCM_PATH}" \
+         -DCMAKE_C_FLAGS="-DTORCH_USE_HIP_DSA" \
+         -DCMAKE_CXX_FLAGS="-DTORCH_USE_HIP_DSA"
+
+      # Build and install the library into the Conda environment
+      python setup.py install \
+         --package_variant=rocm \
+         -DHIP_ROOT_DIR="${ROCM_PATH}" \
+         -DCMAKE_C_FLAGS="-DTORCH_USE_HIP_DSA" \
+         -DCMAKE_CXX_FLAGS="-DTORCH_USE_HIP_DSA"  
+
+Post-build validation
+----------------------
+
+After building FBGEMM_GPU, run some verification checks to ensure the build is correct. Continue
+to run all commands inside the ``fbgemm_gpu/`` directory inside the Miniconda environment.
+
+#. The build process generates many build artifacts and C++ templates, so
+   it is important to confirm no undefined symbols remain.
+
+   .. code-block:: shell
+
+      # !! Run in fbgemm_gpu/ directory inside the Conda environment !!
+
+      # Locate the built .SO file
+      fbgemm_gpu_lib_path=$(find . -name fbgemm_gpu_py.so)
+
+      # Check that the undefined symbols don't include fbgemm_gpu-defined functions
+      nm -gDCu "${fbgemm_gpu_lib_path}" | sort
+
+#. Verify the referenced version number of ``GLIBCXX`` and the presence of certain function symbols:
+
+   .. code-block:: shell
+
+      # !! Run in fbgemm_gpu/ directory inside the Conda environment !!
+
+      # Locate the built .SO file
+      fbgemm_gpu_lib_path=$(find . -name fbgemm_gpu_py.so)
+
+      # Note the versions of GLIBCXX referenced by the .SO
+      # The libstdc++.so.6 available on the install target must support these versions
+      objdump -TC "${fbgemm_gpu_lib_path}" | grep GLIBCXX | sed 's/.*GLIBCXX_\([.0-9]*\).*/GLIBCXX_\1/g' | sort -Vu | cat
+
+      # Test for the existence of a given function symbol in the .SO
+      nm -gDC "${fbgemm_gpu_lib_path}" | grep " fbgemm_gpu::merge_pooled_embeddings("
+      nm -gDC "${fbgemm_gpu_lib_path}" | grep " fbgemm_gpu::jagged_2d_to_dense("
+
+Testing FBGEMM
+----------------------
+
+FBGEMM includes tests and benchmarks to validate performance. To run these tests,
+you must use ROCm 5.7 or a more recent version on the host and container. To run FBGEMM tests,
+follow these instructions:
+
+.. code-block:: shell
+
+   # !! Run inside the Conda environment !!
+
+   # From the /fbgemm_gpu/ directory
+   cd test
+
+   export FBGEMM_TEST_WITH_ROCM=1
+   # Enable for debugging failed kernel executions
+   export HIP_LAUNCH_BLOCKING=1
+
+   # Run the test
+   python -m pytest -v -rsx -s -W ignore::pytest.PytestCollectionWarning split_table_batched_embeddings_test.py
+
+To run the FBGEMM_GPU ``uvm`` test, use these commands. These tests only support the AMD MI210 and 
+more recent accelerators. 
+
+.. code-block:: shell
+
+   # Run this inside the Conda environment from the /fbgemm_gpu/ directory
+   export HSA_XNACK=1
+   cd test
+
+   python -m pytest -v -rsx -s -W ignore::pytest.PytestCollectionWarning ./uvm/uvm_test.py

docs/how-to/llm-fine-tuning-optimization/model-quantization.rst

@@ -189,6 +189,9 @@ Installing bitsandbytes
       # Use -DBNB_ROCM_ARCH to specify target GPU arch
       cmake -DBNB_ROCM_ARCH="gfx942" -DCOMPUTE_BACKEND=hip -S .
 
+      # Compile the project
+      make
+
       # Install 
       python setup.py install
 
@@ -219,12 +222,11 @@ To get started with bitsandbytes primitives, use the following code as reference
 Using bitsandbytes with Hugging Face Transformers
 -------------------------------------------------
 
-To load a Transformers model in 4-bit, set ``load_int_4bt=true`` in ``BitsAndBytesConfig``.
+To load a Transformers model in 4-bit, set ``load_in_4bit=true`` in ``BitsAndBytesConfig``.
 
 .. code-block:: python
 
-   from transformers import AutoModelForCausalLM
-   from bitsandbytes import BitsAndBytesConfig
+   from transformers import AutoModelForCausalLM, BitsAndBytesConfig
    
    base_model_name = "NousResearch/Llama-2-7b-hf"
    quantization_config = BitsAndBytesConfig(load_in_4bit=True)
@@ -240,8 +242,7 @@ To load a model in 8-bit for inference, use the ``load_in_8bit`` option.
 
 .. code-block:: python
 
-   from transformers import AutoModelForCausalLM, AutoTokenizer
-   from bitsandbytes import BitsAndBytesConfig
+   from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
    
    base_model_name = "NousResearch/Llama-2-7b-hf"
    

docs/how-to/rocm-for-ai/deploy-your-model.rst

@@ -41,6 +41,15 @@ vLLM walkthrough
 Refer to this developer blog for guidance on serving with vLLM `Inferencing and serving with vLLM on AMD GPUs — ROCm
 Blogs <https://rocm.blogs.amd.com/artificial-intelligence/vllm/README.html>`_
 
+Validating vLLM performance
+---------------------------
+
+ROCm provides a prebuilt optimized Docker image for validating the performance of LLM inference with vLLM 
+on the MI300X accelerator. The Docker image includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 .. _rocm-for-ai-serve-hugging-face-tgi:
 
 Serving using Hugging Face TGI

docs/how-to/rocm-for-ai/install.rst

@@ -24,23 +24,23 @@ If you’re new to ROCm, refer to the :doc:`ROCm quick start install guide for L
 If you’re using a Radeon GPU for graphics-accelerated applications, refer to the
 :doc:`Radeon installation instructions <radeon:docs/install/install-radeon>`.
 
-ROCm supports two methods for installation. There is no difference in the final ROCm installation between these two
-methods. You can also opt for :ref:`single-version or multi-version installation
-<rocm-install-on-linux:installation-types>`.
+ROCm supports multiple :doc:`installation methods <rocm-install-on-linux:install/install-overview>`:
 
-*  :doc:`Using your Linux distribution's package manager <rocm-install-on-linux:how-to/native-install/index>`
+* :doc:`Using your Linux distribution's package manager <rocm-install-on-linux:install/native-install/index>`
 
-*  :doc:`Using the AMDGPU installer <rocm-install-on-linux:how-to/amdgpu-install>`
+* :doc:`Using the AMDGPU installer <rocm-install-on-linux:install/amdgpu-install>`
+
+* :ref:`Multi-version installation <rocm-install-on-linux:installation-types>`.
 
 .. grid:: 1
 
    .. grid-item-card:: Post-install
 
-      Follow the :doc:`post-installation instructions <rocm-install-on-linux:how-to/native-install/post-install>` to
+      Follow the :doc:`post-installation instructions <rocm-install-on-linux:install/post-install>` to
       configure your system linker, PATH, and verify the installation.
 
       If you encounter any issues during installation, refer to the
-      :doc:`Installation troubleshooting <rocm-install-on-linux:how-to/native-install/install-faq>` guide.
+      :doc:`Installation troubleshooting <rocm-install-on-linux:reference/install-faq>` guide.
 
 Machine learning frameworks
 ===========================

docs/how-to/system-optimization/mi100.md

@@ -365,9 +365,9 @@ installed.
 ## System management
 
 For a complete guide on how to install/manage/uninstall ROCm on Linux, refer to
-{doc}`Quick-start (Linux)<rocm-install-on-linux:tutorial/quick-start>`. To verify that the installation was
+{doc}`Quick-start (Linux)<rocm-install-on-linux:install/quick-start>`. To verify that the installation was
 successful, refer to the
-{doc}`post-install instructions<rocm-install-on-linux:how-to/native-install/post-install>` and
+{doc}`post-install instructions<rocm-install-on-linux:install/post-install>` and
 [system tools](../../reference/rocm-tools.md). Should verification
 fail, consult the [System Debugging Guide](../system-debugging.md).
 

docs/how-to/system-optimization/mi200.md

@@ -350,9 +350,9 @@ installed.
 ## System management
 
 For a complete guide on how to install/manage/uninstall ROCm on Linux, refer to
-{doc}`Quick-start (Linux)<rocm-install-on-linux:tutorial/quick-start>`. For verifying that the
+{doc}`Quick-start (Linux)<rocm-install-on-linux:install/quick-start>`. For verifying that the
 installation was successful, refer to the
-{doc}`post-install instructions<rocm-install-on-linux:how-to/native-install/post-install>` and
+{doc}`post-install instructions<rocm-install-on-linux:install/post-install>` and
 [system tools](../../reference/rocm-tools.md). Should verification
 fail, consult the [System Debugging Guide](../system-debugging.md).
 

docs/how-to/system-optimization/mi300a.rst

@@ -122,6 +122,62 @@ This section describes performance-based settings.
 
        transparent_hugepage=always
 
+* **Increase the amount of allocatable memory**
+
+  By default, when using a device allocator via HIP, it is only possible to allocate 96 GiB out of 
+  a possible 128 GiB of memory on the MI300A. This limitation does not affect host allocations.
+  To increase the available system memory, load the ``amdttm`` module with new values for
+  ``pages_limit`` and ``page_pool_size``. These numbers correspond to the number of 4 KiB pages of memory.
+  To make 128 GiB of memory available across all four devices, for a total amount of 512 GiB,
+  set ``pages_limit`` and ``page_pool_size`` to ``134217728``. For a two-socket system, divide these values
+  by two. After setting these values, reload the AMDGPU driver.
+
+  First, review the current settings using this shell command:
+
+  .. code-block:: shell
+
+     cat /sys/module/amdttm/parameters/pages_limit 
+  
+  To set the amount of allocatable memory to all available memory on all four APU devices, run these commands:
+
+  .. code-block:: shell
+
+     sudo modprobe amdttm pages_limit=134217728 page_pool_size=134217728
+     sudo modprobe amdgpu
+
+  These settings can also be hardcoded in the ``/etc/modprobe.d/amdttm.conf`` file or specified as boot
+  parameters.
+  
+  To use the hardcoded method, 
+  the filesystem must already be set up when the kernel driver is loaded.
+  To hardcode the settings, add the following lines to ``/etc/modprobe.d/amdttm.conf``:
+
+  .. code-block:: shell
+
+     options amdttm pages_limit=134217728
+     options amdttm page_pool_size=134217728
+
+  If the filesystem is not already set up when the kernel driver is loaded, then the options
+  must be specified as boot parameters. To specify the settings
+  as boot parameters when loading the kernel, use this example as a guideline:
+
+  .. code-block:: shell
+
+     vmlinux-[...] amdttm.pages_limit=134217728 amdttm.page_pool_size=134217728 [...]
+
+  To verify the new settings and confirm the change, use this command:
+
+  .. code-block:: shell
+
+     cat /sys/module/amdttm/parameters/pages_limit 
+
+  .. note::
+
+     The system settings for ``pages_limit`` and ``page_pool_size`` are calculated by multiplying the
+     per-APU limit of 4 KiB pages, which is ``33554432``, by the number of APUs on the node. The limit for a system with
+     two APUs ``33554432 x 2`` or ``67108864``.
+     This means the ``modprobe`` command for two APUs is ``sudo modprobe amdttm pages_limit=67108864 page_pool_size=67108864``.
+
 * **Limit the maximum and single memory allocations on the GPU**
   
   Many AI-related applications were originally developed on discrete GPUs. Some of these applications 
@@ -195,10 +251,13 @@ This section describes performance-based settings.
 
 * **Change affinity of ROCm helper threads**
   
-  This change prevents internal ROCm threads from having their CPU core affinity mask 
+  Changing the affinity prevents internal ROCm threads from having their CPU core affinity mask 
   set to all CPU cores available. With this setting, the threads inherit their parent's 
-  CPU core affinity mask. If you have any questions regarding this setting, 
-  contact your MI300A platform vendor. To enable this setting, enter the following command:
+  CPU core affinity mask. Before adjusting this setting, ensure you thoroughly understand 
+  your system topology and how the application, runtime environment, and batch system
+  set the thread-to-core affinity. If you have any questions regarding this setting, 
+  contact your MI300A platform vendor or the AMD support team. 
+  To enable this setting, enter the following command:
 
   .. code-block:: shell
 
@@ -213,9 +272,9 @@ System management
 ========================================
 
 For a complete guide on installing, managing, and uninstalling ROCm on Linux, see
-:doc:`Quick-start (Linux)<rocm-install-on-linux:tutorial/quick-start>`. To verify that the
+:doc:`Quick-start (Linux)<rocm-install-on-linux:install/quick-start>`. To verify that the
 installation was successful, see the
-:doc:`Post-installation instructions<rocm-install-on-linux:install/native-install/post-install>` and 
+:doc:`Post-installation instructions<rocm-install-on-linux:install/post-install>` and 
 :doc:`ROCm tools <../../reference/rocm-tools>` guides. If verification
 fails, consult the :doc:`System debugging guide <../system-debugging>`.
 

docs/how-to/system-optimization/mi300x.rst

@@ -534,7 +534,7 @@ optimizing user applications.
 For a complete guide on how to install, manage, or uninstall ROCm on Linux, refer to
 :doc:`rocm-install-on-linux:install/quick-start`. For verifying that the
 installation was successful, refer to the
-:doc:`rocm-install-on-linux:install/native-install/post-install`.
+:doc:`rocm-install-on-linux:install/post-install`.
 Should verification fail, consult :doc:`/how-to/system-debugging`.
 
 Hardware verification with ROCm

docs/how-to/tuning-guides/mi300x/workload.rst

@@ -150,6 +150,12 @@ the workload to validate improvements and ensure that the changes have had the
 desired effect. Continuous iteration helps refine the performance gains and
 address any new bottlenecks that may emerge.
 
+ROCm provides a prebuilt optimized Docker image that has everything required to implement
+the tips in this section. It includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 .. _mi300x-profiling-tools:
 
 Profiling tools
@@ -372,6 +378,12 @@ Refer to `vLLM documentation <https://docs.vllm.ai/en/latest/models/performance.
 for additional performance tips. :ref:`fine-tuning-llms-vllm` describes vLLM
 usage with ROCm.
 
+ROCm provides a prebuilt optimized Docker image for validating the performance of LLM inference with vLLM 
+on the MI300X accelerator. The Docker image includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 Maximize throughput
 -------------------
 

docs/reference/api-libraries.md

@@ -25,6 +25,7 @@
 * {doc}`MIVisionX <mivisionx:index>`
 * {doc}`rocAL <rocal:index>`
 * {doc}`rocDecode <rocdecode:index>`
+* {doc}`rocPyDecode <rocpydecode:index>`
 * {doc}`ROCm Performance Primitives (RPP) <rpp:index>`
 :::
 
@@ -53,18 +54,6 @@
 * {doc}`RCCL <rccl:index>`
 :::
 
-(hip-runtime)=
-
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-hip.jpg
-:img-alt: HIP APIs
-:padding: 2
-
-* {doc}`HIP runtime <hip:index>`
-* {doc}`HIPIFY <hipify:index>`
-:::
-
 (math-apis)=
 
 :::{grid-item-card}
@@ -78,6 +67,7 @@
 * {doc}`hipBLASLt <hipblaslt:index>`
 * {doc}`hipFFT <hipfft:index>` / {doc}`rocFFT <rocfft:index>`
 * {doc}`hipfort <hipfort:index>`
+* {doc}`hipRAND <hiprand:index>` / {doc}`rocRAND <rocrand:index>`
 * {doc}`hipSOLVER <hipsolver:index>` / {doc}`rocSOLVER <rocsolver:index>`
 * {doc}`hipSPARSE <hipsparse:index>` / {doc}`rocSPARSE <rocsparse:index>`
 * {doc}`hipSPARSELt <hipsparselt:index>`
@@ -86,16 +76,4 @@
 * [Tensile](https://github.com/ROCm/Tensile)
 :::
 
-(random-number-apis)=
-
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-random-number.jpg
-:img-alt: Random number APIs
-:padding: 2
-
-* {doc}`hipRAND <hiprand:index>`
-* {doc}`rocRAND <rocrand:index>`
-:::
-
 ::::

docs/reference/rocm-tools.md

@@ -40,6 +40,7 @@
 * {doc}`ROCProfiler <rocprofiler:index>`
 * {doc}`ROCprofiler-SDK <rocprofiler-sdk:index>`
 * {doc}`ROCTracer <roctracer:index>`
+* {doc}`TransferBench <transferbench:index>`
 :::
 
 (development-tools)=

docs/sphinx/requirements.in

@@ -1,2 +1,2 @@
-rocm-docs-core==1.7.0
+rocm-docs-core==1.7.2
 sphinx-reredirects

docs/sphinx/requirements.txt

@@ -26,7 +26,7 @@ charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via sphinx-external-toc
-cryptography==42.0.8
+cryptography==43.0.1
     # via pyjwt
 deprecated==1.2.14
     # via pygithub
@@ -92,7 +92,7 @@ requests==2.32.3
     # via
     #   pygithub
     #   sphinx
-rocm-docs-core==1.7.0
+rocm-docs-core==1.7.2
     # via -r requirements.in
 smmap==5.0.1
     # via gitdb

docs/what-is-rocm.rst

@@ -119,6 +119,8 @@ Performance
   ":doc:`ROCProfiler <rocprofiler:index>`", "Profiling tool for HIP applications"
   ":doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`", "Toolkit for developing analysis tools for profiling and tracing GPU compute applications. This toolkit is in beta and subject to change"
   ":doc:`ROCTracer <roctracer:index>`", "Intercepts runtime API calls and traces asynchronous activity"
+  ":doc:`TransferBench <transferbench:index>`", "Utility to benchmark simultaneous transfers between user-specified devices (CPUs or GPUs)"
+
 
 Development
 ^^^^^^^^^^^

temp.md

@@ -1,58 +0,0 @@
-## Components
-
-The following table lists ROCm components and their individual versions for ROCm 6.2.0. Find an overview of officially
-supported versions of ROCm components, third-party libraries, and frameworks in the
-[Compatibility matrix](https://rocm.docs.amd.com/en/latest/release/docs/6.2.0/compatibility/compatibility-matrix).
-
-| Category | Group | Name | Version |   |
-|----------|-------|------|---------|:-:|
-| **Libraries** | **Machine learning and computer vision** | [Composable Kernel](https://rocm.docs.amd.com/projects/composable_kernel/en/docs/6.2.0) | 1.1.0 | [{fab}`github fa-lg`](https://github.com/ROCm/composable_kernel/releases/tag/rocm-6.2.0) |
-|  |  | [MIGraphX](https://rocm.docs.amd.com/projects/AMDMIGraphX/en/docs/6.2.0) | 2.9 ⇒ [2.10](migraphx-2-10-0) | [{fab}`github fa-lg`](https://github.com/ROCm/AMDMIGraphX/releases/tag/rocm-6.2.0) |
-|  |  | [MIOpen](https://rocm.docs.amd.com/projects/MIOpen/en/docs/6.2.0) | 3.1.0 ⇒ [3.2.0](miopen-3-2-0) | [{fab}`github fa-lg`](https://github.com/ROCm/MIOpen/releases/tag/rocm-6.2.0) |
-|  |  | [MIVisionX](https://rocm.docs.amd.com/projects/MIVisionX/en/docs/6.2.0) | 2.5.0 ⇒ [3.0.0](mivisionx-3-0-0) | [{fab}`github fa-lg`](https://github.com/ROCm/MIVisionX/releases/tag/rocm-6.2.0) |
-|  |  | [rocAL](https://rocm.docs.amd.com/projects/rocAL/en/docs/6.2.0) | 2.0.0 | [{fab}`github fa-lg`](https://github.com/ROCm/rocAL/releases/tag/rocm-6.2.0) |
-|  |  | [rocDecode](https://rocm.docs.amd.com/projects/rocDecode/en/docs/6.2.0) | 0.6.0 | [{fab}`github fa-lg`](https://github.com/ROCm/rocDecode/releases/tag/rocm-6.2.0) |
-|  |  | [rocPyDecode](https://rocm.docs.amd.com/projects/rocPyDecode/en/docs/6.2.0) | 0.1.0 | [{fab}`github fa-lg`](https://github.com/ROCm/rocPyDecode/releases/tag/rocm-6.2.0) |
-|  |  | [RPP](https://rocm.docs.amd.com/projects/rpp/en/docs/6.2.0) | 1.5.0 ⇒ [1.8.0](rpp-1-8-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rpp/releases/tag/rocm-6.2.0) |
-|  | **Communication** | [rccl](https://rocm.docs.amd.com/projects/rccl/en/docs/6.2.0) | 2.18.6 ⇒ [2.20.5](rccl-2-20-5) | [{fab}`github fa-lg`](https://github.com/ROCm/rccl/releases/tag/rocm-6.2.0) |
-|  | **Math** | [hipBLAS](https://rocm.docs.amd.com/projects/hipBLAS/en/docs/6.2.0) | 2.1.0 ⇒ [2.2.0](hipblas-2-2-0) | [{fab}`github fa-lg`](https://github.com/ROCm/hipBLAS/releases/tag/rocm-6.2.0) |
-|  |  | [hipBLASLt](https://rocm.docs.amd.com/projects/hipBLASLt/en/docs/6.2.0) | 0.7.0 ⇒ [0.8.0](hipblaslt-0-8-0) | [{fab}`github fa-lg`](https://github.com/ROCm/hipBLASLt/releases/tag/rocm-6.2.0) |
-|  |  | [hipFFT](https://rocm.docs.amd.com/projects/hipFFT/en/docs/6.2.0) | [1.0.14](hipfft-1-0-14) | [{fab}`github fa-lg`](https://github.com/ROCm/hipFFT/releases/tag/rocm-6.2.0) |
-|  |  | [hipfort](https://rocm.docs.amd.com/projects/hipfort/en/docs/6.2.0) | 0.4-0 | [{fab}`github fa-lg`](https://github.com/ROCm/hipfort/releases/tag/rocm-6.2.0) |
-|  |  | [hipRAND](https://rocm.docs.amd.com/projects/hipRAND/en/docs/6.2.0) | 2.10.17 ⇒ [2.11.0](hiprand-2-11-0) | [{fab}`github fa-lg`](https://github.com/ROCm/hipRAND/releases/tag/rocm-6.2.0) |
-|  |  | [hipSOLVER](https://rocm.docs.amd.com/projects/hipSOLVER/en/docs/6.2.0) | 2.1.1 ⇒ [2.2.0](hipsolver-2-2-0) | [{fab}`github fa-lg`](https://github.com/ROCm/hipSOLVER/releases/tag/rocm-6.2.0) |
-|  |  | [hipSPARSE](https://rocm.docs.amd.com/projects/hipSPARSE/en/docs/6.2.0) | 3.0.1 ⇒ [3.1.1](hipsparse-3-1-1) | [{fab}`github fa-lg`](https://github.com/ROCm/hipSPARSE/releases/tag/rocm-6.2.0) |
-|  |  | [hipSPARSELt](https://rocm.docs.amd.com/projects/hipSPARSELt/en/docs/6.2.0) | 0.2.0 ⇒ [0.2.1](hipsparselt-0-2-1) | [{fab}`github fa-lg`](https://github.com/ROCm/hipSPARSELt/releases/tag/rocm-6.2.0) |
-|  |  | [rocALUTION](https://rocm.docs.amd.com/projects/rocALUTION/en/docs/6.2.0) | 3.1.1 ⇒ [3.2.0](rocalution-3-2-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rocALUTION/releases/tag/rocm-6.2.0) |
-|  |  | [rocBLAS](https://rocm.docs.amd.com/projects/rocBLAS/en/docs/6.2.0) | 4.1.0 ⇒ [4.2.0](rocblas-4-2-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rocBLAS/releases/tag/rocm-6.2.0) |
-|  |  | [rocFFT](https://rocm.docs.amd.com/projects/rocFFT/en/docs/6.2.0) | 1.0.27 ⇒ [1.0.28](rocfft-1-0-28) | [{fab}`github fa-lg`](https://github.com/ROCm/rocFFT/releases/tag/rocm-6.2.0) |
-|  |  | [rocRAND](https://rocm.docs.amd.com/projects/rocRAND/en/docs/6.2.0) | 3.0.0 ⇒ [3.1.0](rocrand-3-1-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rocRAND/releases/tag/rocm-6.2.0) |
-|  |  | [rocSOLVER](https://rocm.docs.amd.com/projects/rocSOLVER/en/docs/6.2.0) | 3.25.0 ⇒ [3.26.0](rocsolver-3-26-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rocSOLVER/releases/tag/rocm-6.2.0) |
-|  |  | [rocSPARSE](https://rocm.docs.amd.com/projects/rocSPARSE/en/docs/6.2.0) | 3.1.1 ⇒ [3.2.0](rocsparse-3-2-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rocSPARSE/releases/tag/rocm-6.2.0) |
-|  |  | [rocWMMA](https://rocm.docs.amd.com/projects/rocWMMA/en/docs/6.2.0) | 1.4.0 ⇒ [1.5.0](rocwmma-1-5-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rocWMMA/releases/tag/rocm-6.2.0) |
-|  |  | [Tensile](https://rocm.docs.amd.com/projects/tensile/en/docs/6.2.0) | 4.40.0 ⇒ [4.41.0](tensile-4-41-0) | [{fab}`github fa-lg`](https://github.com/ROCm/tensile/releases/tag/rocm-6.2.0) |
-|  | **Primitives** | [hipCUB](https://rocm.docs.amd.com/projects/hipCUB/en/docs/6.2.0) | 3.1.0 ⇒ [3.2.0](hipcub-3-2-0) | [{fab}`github fa-lg`](https://github.com/ROCm/hipCUB/releases/tag/rocm-6.2.0) |
-|  |  | [hipTensor](https://rocm.docs.amd.com/projects/hipTensor/en/docs/6.2.0) | 1.2.0 ⇒ [1.3.0](hiptensor-1-3-0) | [{fab}`github fa-lg`](https://github.com/ROCm/hipTensor/releases/tag/rocm-6.2.0) |
-|  |  | [rocPRIM](https://rocm.docs.amd.com/projects/rocPRIM/en/docs/6.2.0) | 3.1.0 ⇒ [3.2.0](rocprim-3-2-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rocPRIM/releases/tag/rocm-6.2.0) |
-|  |  | [rocThrust](https://rocm.docs.amd.com/projects/rocThrust/en/docs/6.2.0) | 3.0.0 ⇒ [3.1.0](rocthrust-3-1-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rocThrust/releases/tag/rocm-6.2.0) |
-| **Tools** | **Development** | [HIPIFY](https://rocm.docs.amd.com/projects/HIPIFY/docs/6.2.0) | 17.0.0 ⇒ [18.0.0](hipify-18-0-0) | [{fab}`github fa-lg`](https://github.com/ROCm/HIPIFY/releases/tag/rocm-6.2.0) |
-|  |  | [ROCdbgapi](https://rocm.docs.amd.com/projects/ROCdbgapi/en/docs/6.2.0) | 0.71.0 ⇒ [0.76.0](rocdbgapi-0-76-0) | [{fab}`github fa-lg`](https://github.com/ROCm/ROCdbgapi/releases/tag/rocm-6.2.0) |
-|  |  | [ROCm CMake](https://rocm.docs.amd.com/projects/rocm-cmake/en/docs/6.2.0) | 0.12.0 ⇒ [0.13.0](rocm-cmake-0-13-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rocm-cmake/releases/tag/rocm-6.2.0) |
-|  |  | [ROCm Debugger (ROCgdb)](https://rocm.docs.amd.com/projects/rocm-cmake/en/docs/6.2.0) | 13 ⇒ [15](rocgdb-15) | [{fab}`github fa-lg`](https://github.com/ROCm/ROCgdb/releases/tag/rocm-6.2.0) |
-|  |  | [ROCr Debug Agent](https://rocm.docs.amd.com/projects/rocr_debug_agent/en/docs/6.2.0) | 2.0.3 | [{fab}`github fa-lg`](https://github.com/ROCm/rocr_debug_agent/releases/tag/rocm-6.2.0) |
-|  | **Performance** | [Omniperf](https://rocm.docs.amd.com/projects/omniperf/en/docs/6.2.0) | 2.0.1 | [{fab}`github fa-lg`](https://github.com/ROCm/omniperf/releases/tag/rocm-6.2.0) |
-|  |  | [Omnitrace](https://rocm.docs.amd.com/projects/omnitrace/en/docs/6.2.0) | 1.11.2 | [{fab}`github fa-lg`](https://github.com/ROCm/omnitrace/releases/tag/rocm-6.2.0) |
-|  |  | [ROCm Bandwidth Test](https://rocm.docs.amd.com/projects/rocm_bandwidth_test/en/docs/6.2.0) | 1.4.0 | [{fab}`github fa-lg`](https://github.com/ROCm/rocm_bandwidth_test/releases/tag/rocm-6.2.0) |
-|  |  | [ROCProfiler](https://rocm.docs.amd.com/projects/ROCProfiler/en/docs/6.2.0) | 2.0.0 ⇒ [2.0.0](rocprofiler-2-0-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rocm_bandwidth_test/releases/tag/rocm-6.2.0) |
-|  |  | [ROCProfiler-SDK](https://rocm.docs.amd.com/projects/rocprofiler-sdk/en/docs/6.2.0) | 0.4.0 | [{fab}`github fa-lg`](https://github.com/ROCm/rocm_bandwidth_test/releases/tag/rocm-6.2.0) |
-|  |  | [ROCTracer](https://rocm.docs.amd.com/projects/ROCTracer/en/docs/6.2.0) | 4.1.0 | [{fab}`github fa-lg`](https://github.com/ROCm/rocm_bandwidth_test/releases/tag/rocm-6.2.0) |
-|  | **System** | [AMD SMI](https://rocm.docs.amd.com/projects/amdsmi/en/docs/6.2.0) | 24.5.2 ⇒ [24.6.1](amd-smi-24-6-1) | [{fab}`github fa-lg`](https://github.com/ROCm/rdc/releases/tag/rocm-6.2.0) |
-|  |  | [rocminfo](https://rocm.docs.amd.com/projects/rdc/en/docs/6.2.0) | 1.0.0 | [{fab}`github fa-lg`](https://github.com/ROCm/rdc/releases/tag/rocm-6.2.0) |
-|  |  | [ROCm Data Center Tool](https://rocm.docs.amd.com/projects/rdc/en/docs/6.2.0) | 0.3.0 ⇒ [1.0.0](rocm-data-center-tool-1-0-0) | [{fab}`github fa-lg`](https://github.com/ROCm/rdc/releases/tag/rocm-6.2.0) |
-|  |  | [ROCm SMI](https://rocm.docs.amd.com/projects/rdc/en/docs/6.2.0) | 7.2.0 | [{fab}`github fa-lg`](https://github.com/ROCm/rdc/releases/tag/rocm-6.2.0) |
-|  |  | [ROCm Validation Suite](https://rocm.docs.amd.com/projects/rdc/en/docs/6.2.0) | 1.0 | [{fab}`github fa-lg`](https://github.com/ROCm/rdc/releases/tag/rocm-6.2.0) |
-|  |  | [TransferBench](https://rocm.docs.amd.com/projects/rdc/en/docs/6.2.0) | 1.5.0 | [{fab}`github fa-lg`](https://github.com/ROCm/rdc/releases/tag/rocm-6.2.0) |
-|  | **Compilers** | [hipCC](https://rocm.docs.amd.com/projects/hipCC/en/docs/6.2.0) | 1.0.0 ⇒ [1.1.1](hipcc-1-1-1) | [{fab}`github fa-lg`](https://github.com/ROCm/llvm-project/releases/tag/rocm-6.2.0) |
-|  |  | [llvm-project](https://rocm.docs.amd.com/projects/llvm-project/en/docs/6.2.0) | 17.0.0 ⇒ [18.0.0](llvm-project-18-0-0) | [{fab}`github fa-lg`](https://github.com/ROCm/llvm-project/releases/tag/rocm-6.2.0) |
-| **Runtimes** |  | [HIP](https://rocm.docs.amd.com/projects/HIP/en/docs/6.2.0) | 6.1 ⇒ [6.2](hip-6-2-0) | [{fab}`github fa-lg`](https://github.com/ROCm/HIP/releases/tag/rocm-6.2.0) |
-|  |  | [ROCr Runtime](https://rocm.docs.amd.com/projects/ROCr-Runtime/en/docs/6.2.0) | 6.1 ⇒ [6.2](hip-6-2-0) | [{fab}`github fa-lg`](https://github.com/ROCm/ROCR-Runtime/releases/tag/rocm-6.2.0) |