Merge pull request #3773 from samjwu/updaterel62

Update 6.2 release notes

.azuredevops/ci-builds/aomp.yml

@@ -4,11 +4,21 @@ variables:
 
 resources:
   repositories:
-  - repository: release_repo
+  - repository: aomp_repo
     type: github
     endpoint: ROCm
     name: ROCm/aomp
-    ref: aomp-dev
+    ref: amd-staging
+  - repository: aomp-extras_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp-extras
+    ref: amd-staging
+  - repository: flang_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/flang
+    ref: amd-staging
   - repository: llvm-project_repo
     type: github
     endpoint: ROCm
@@ -26,4 +36,4 @@ pr: none
 jobs:
   - template: ${{ variables.CI_COMPONENT_PATH }}/aomp.yml
     parameters:
-      checkoutRepo: release_repo
+      checkoutRepo: aomp_repo

.azuredevops/components/AMDMIGraphX.yml

@@ -11,6 +11,7 @@ parameters:
     - git
     - cmake
     - ninja-build
+    - libdrm-dev
     - libnuma-dev
     - python3-pip
     - python3-venv
@@ -34,17 +35,21 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
-    - clr
-    - rocminfo
-    - rocMLIR
-    - MIOpen
     - aomp
     - aomp-extras
-    - rocBLAS
+    - clr
     - composable_kernel
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - MIOpen
+    - rocm-cmake
+    - ROCR-Runtime
+    - rocBLAS
+    - rocminfo
+    - rocMLIR
+    - rocprofiler-register
 
 jobs:
 - job: AMDMIGraphX
@@ -74,7 +79,7 @@ jobs:
     parameters:
       dependencySource: fixed
       fixedComponentName: half
-      fixedPipelineIdentifier: $(half560-pipeline-id)
+      fixedPipelineIdentifier: ${{ variables.HALF560_PIPELINE_ID }}
       skipLibraryLinking: true
       skipLlvmSymlink: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
@@ -94,6 +99,7 @@ jobs:
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
         -DCMAKE_BUILD_TYPE=Release
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
         -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
         -DMIGRAPHX_USE_COMPOSABLEKERNEL=OFF

.azuredevops/components/HIP.yml

@@ -21,7 +21,6 @@ parameters:
   type: object
   default:
     - llvm-project
-    - ROCT-Thunk-Interface
     - ROCR-Runtime
     - rocprofiler-register
 - name: rocmDependenciesNvidia
@@ -101,6 +100,7 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
       pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
 # checkout triggering repo (either HIP or clr)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:

.azuredevops/components/HIPIFY.yml

@@ -11,42 +11,97 @@ parameters:
     - cmake
     - ninja-build
     - libnuma-dev
-- name: rocmDependencies
-  type: object
-  default:
-    - llvm-project
+    - python3-dev
+    - python3-pip
+    - python-is-python3
+    - mesa-common-dev
+    - ccache
+    - cuda-toolkit
+    - cudnn
 
 jobs:
 - job: HIPIFY
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  - name: UPSTREAM_LLVM_GIT_URL
+    value: https://github.com/llvm/llvm-project.git
+  - name: UPSTREAM_LLVM_TAG
+    value: llvmorg-18.1.2
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
   steps:
+  - task: Bash@3
+    displayName: 'Register CUDA packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-archive-keyring.gpg | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/cuda-archive-keyring.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" | sudo tee /etc/apt/sources.list.d/cuda-ubuntu2204-x86_64.list
+        sudo apt update
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: Bash@3
+    displayName: git clone upstream llvm-project
+    inputs:
+      targetType: inline
+      script: git clone $(UPSTREAM_LLVM_GIT_URL) --depth=1 --branch $(UPSTREAM_LLVM_TAG) --recurse-submodules
+      workingDirectory: $(Pipeline.Workspace)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
-    parameters:
-      dependencyList: ${{ parameters.rocmDependencies }}
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        dependencySource: staging
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        dependencySource: tag-builds
+  - script: |
+      mkdir -p $(CCACHE_DIR)
+      echo "##vso[task.prependpath]/usr/lib/ccache:/usr/local/cuda/bin"
+    displayName: Update path for cuda and ccache
+  - task: Cache@2
+    displayName: Ccache caching
+    inputs:
+      key: HIPIFY | $(Agent.OS) | "$(UPSTREAM_LLVM_TAG)"
+      path: $(CCACHE_DIR)
+      restoreKeys: HIPIFY | $(Agent.OS)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
+      componentName: upstream-llvm
+      cmakeBuildDir: $(Pipeline.Workspace)/llvm-project/llvm/build
+      installDir: $(Pipeline.Workspace)/llvm
       extraBuildFlags: >-
         -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm
-        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DLLVM_ENABLE_PROJECTS=clang
+        -DLLVM_INCLUDE_TESTS=OFF
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
         -GNinja
+  - task: Bash@3
+    displayName: python install lit
+    inputs:
+      targetType: inline
+      script: sudo python3 $(Pipeline.Workspace)/llvm-project/llvm/utils/lit/setup.py install
+  - task: Bash@3
+    displayName: install FileCheck
+    inputs:
+      targetType: inline
+      script: cp $(Pipeline.Workspace)/llvm-project/llvm/build/bin/FileCheck $(Pipeline.Workspace)/llvm/bin
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      componentName: HIPIFY
+      extraBuildFlags: >-
+        -DHIPIFY_CLANG_TESTS=ON
+        -DCMAKE_BUILD_TYPE=Release
+        -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda/targets/x86_64-linux
+        -DCUDA_DNN_ROOT_DIR=/usr/local/cuda/targets/x86_64-linux
+        -DCMAKE_PREFIX_PATH=$(Pipeline.Workspace)/llvm;/usr/local/cuda/targets/x86_64-linux/lib
+        -DLLVM_EXTERNAL_LIT=$(Pipeline.Workspace)/llvm-project/llvm/build/bin/llvm-lit
+      multithreadFlag: -- -j32
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: HIPIFY
+      testDir: $(Build.SourcesDirectory)/build
+      testExecutable: make
+      testParameters: test-hipify
+      testPublishResults: false

.azuredevops/components/MIOpen.yml

@@ -41,7 +41,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.LARGE_DISK_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
   strategy:

.azuredevops/components/MIVisionX.yml

@@ -58,6 +58,20 @@ parameters:
     - AMDMIGraphX
     - rpp
     - rocDecode
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - half
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - MIOpen
+    - rocBLAS
+    - rocprofiler-register
+    - ROCR-Runtime
+    - roctracer
+    - rpp
 
 jobs:
 - job: MIVisionX
@@ -103,3 +117,52 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: MIVisionX_testing
+  dependsOn: MIVisionX
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  # anything in /opt may be persistent across runs
+  # so we need to remove the symlink if it already exists
+  - script: |
+      sudo rm -f /opt/rocm
+      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      mkdir mivisionx-tests
+      cd mivisionx-tests
+      cmake /opt/rocm/share/mivisionx/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: MIVisionX
+      testDir: 'mivisionx-tests'

.azuredevops/components/ROCR-Runtime.yml

@@ -16,7 +16,6 @@ parameters:
   type: object
   default:
     - llvm-project
-    - ROCT-Thunk-Interface
     - rocprofiler-register
 
 jobs:
@@ -49,5 +48,5 @@ jobs:
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-      cmakeBuildDir: 'src/build'
+        -DBUILD_SHARED_LIBS=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/ROCmValidationSuite.yml

@@ -19,6 +19,8 @@ parameters:
   type: object
   default:
     - clr
+    - hipBLAS-common
+    - hipBLASLt
     - hipRAND
     - llvm-project
     - rocBLAS
@@ -28,7 +30,19 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocRAND
-    - ROCT-Thunk-Interface
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - hipRAND
+    - llvm-project
+    - rocBLAS
+    - rocm_smi_lib
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
 
 jobs:
 - job: ROCmValidationSuite
@@ -80,3 +94,47 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: ROCmValidationSuite_testing
+  dependsOn: ROCmValidationSuite
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: ROCmValidationSuite
+      testExecutable: $(Agent.BuildDirectory)/rocm/bin/rvs -c $(Agent.BuildDirectory)/rocm/share/rocm-validation-suite/conf/MI300X/gst_single.conf
+      testParameters: ''
+      testDir: $(Agent.BuildDirectory)
+      testPublishResults: false

.azuredevops/components/amdsmi.yml

@@ -32,3 +32,30 @@ jobs:
       extraBuildFlags: >-
         -DBUILD_TESTS=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: amdsmi_testing
+  dependsOn: amdsmi
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: amdsmi
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/share/amd_smi/tests/amdsmitst'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/aomp.yml

@@ -5,9 +5,8 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
-- name: offloadEnabled
-  type: boolean
-  default: false
+# reference:
+# https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL_PREREQUISITE.md
 - name: aptPackages
   type: object
   default:
@@ -30,21 +29,52 @@ parameters:
     - libsystemd-dev
     - libssl-dev
     - libstdc++-12-dev
+    - ccache
+    - libgmp-dev
+    - libmpfr-dev
+    - texinfo
+    - libbison-dev
+    - bison
+    - flex
+    - libbabeltrace-dev
+    - libncurses5-dev
+    - liblzma-dev
+    - python3-setuptools
+    - python3-dev
+    - libudev-dev
+    - parallel
+  # Referencing comment snippet below but excluding rocprofiler and roctracer.
+  # This is to remove need for separate build per gpu target.
+  # With our selected build flags, compilation and installation work fine without these two.
+  #
+  # snippet from https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L131-L134
+  #
+  # For ROCM build (AOMP_STANDALONE_BUILD=0) the components roct, rocr,
+  # libdevice, project, comgr, rocminfo, hipamd, rocdbgapi, rocgdb,
+  # roctracer, rocprofiler, rocm_smi_lib, and amdsmi should be found
+  # in ROCM in /opt/rocm.  The ROCM build only needs these components:
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
+    - amdsmi
+    - clr
     - llvm-project
+    - ROCdbgapi
+    - ROCgdb
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - rocm_smi_lib
+    - rocprofiler-register
     - ROCR-Runtime
-    - ROCT-Thunk-Interface
+    - roctracer
 
 jobs:
 - job: aomp
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
   steps:
@@ -52,9 +82,16 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+# checkout the repos tied to openmp-extras, plus llvm-project
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: aomp-extras_repo
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: flang_repo
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: llvm-project_repo
@@ -67,56 +104,385 @@ jobs:
       # manual build case: triggered by ROCm/ROCm repo
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+# Because clang is not being rebuilt and to separate downloaded ROCm
+# dependencies from the new artifacts, we use temporary symbolic links
+# for the compilation and installation to go through.
+  - script: |
+      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      mkdir -p $(Build.BinariesDirectory)/bin
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang $(Build.BinariesDirectory)/bin/clang
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang++ $(Build.BinariesDirectory)/bin/clang++
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/llvm-config $(Build.BinariesDirectory)/bin/llvm-config
+      ln -s $(Agent.BuildDirectory)/rocm/llvm $(Build.BinariesDirectory)/llvm
+    displayName: Extra build environment setup
+# We follow the sequence described in the aomp repo instructions
+# https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL.md
+# But instead of calling build_aomp.sh directly, we have to split up the calls to build each component
+# in order for the omp.h header file to be both be published as an artifact,
+# and be found when building subsequent components.
+# Environment variables specified per bash script were determined from looking through the
+# individual scripts and iteratively testing compilation.
+# Splitting up the build_aomp.sh call is also easier for debugging within Azure, as the former
+# method leads to a giant build log compared to separate logs per script call.
+#
+# Components compiled and the order for non-standalone build found at
+# https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L135-L143
+  - task: Bash@3
+    displayName: Build Prereq
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_prereq.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Build extras
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_extras.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_EXTRAS: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install extras
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_extras.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_EXTRAS: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build openmp
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_openmp.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install openmp
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_openmp.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - script: ln -s $(Build.BinariesDirectory)/include/omp.h $(Build.SourcesDirectory)/llvm-project/llvm/include/omp.h
+    displayName: Link omp header
+  - task: Bash@3
+    displayName: Build offload
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_offload.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install offload
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_offload.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_LEGACY_OPENMP: 1
+      AOMP_USE_NINJA: 1
+      ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_OPENMP: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build flang-legacy
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-legacy.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install flang-legacy
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-legacy.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build pgmath
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_pgmath.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install pgmath
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_pgmath.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build flang
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install flang
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build flang_runtime
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang_runtime.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install flang_runtime
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang_runtime.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+      INSTALL_FLANG: $(Build.BinariesDirectory)
+# Clean up build environment before publish artifact
+  - script: |
+      rm $(Build.BinariesDirectory)/bin/clang
+      rm $(Build.BinariesDirectory)/bin/clang++
+      rm $(Build.BinariesDirectory)/bin/llvm-config
+      rm $(Build.BinariesDirectory)/llvm
+    displayName: Remove temporary symbolic links
+# Copy the files to artifact staging temporarily to clean up binaries directory
+# and then copy files back to llvm subdirectory in the cleaned up binaries directory
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
-      componentName: llvm-openmp
-      extraBuildFlags: >-
-        -DOPENMP_ENABLE_LIBOMPTARGET=1
-        -DOPENMP_TEST_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-        -DOPENMP_TEST_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-        -DLIBOMPTARGET_AMDGCN_GFXLIST=gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906;gfx908;gfx90a;gfx90c;gfx940;gfx941;gfx942;gfx1030;gfx1031;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103
-        -DDEVICELIBS_ROOT=$(Build.SourcesDirectory)/llvm-project/amd/device-libs
-        -DLIBOMP_COPY_EXPORTS=OFF
-        -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm
-        -DLLVM_MAIN_INCLUDE_DIR=$(Build.SourcesDirectory)/llvm-project/llvm/include
-        -DLIBOMPTARGET_LLVM_INCLUDE_DIRS=$(Build.SourcesDirectory)/llvm-project/llvm/include
-        -DCUDA_TOOLKIT_ROOT_DIR=OFF
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/lib/cmake
-        -DCMAKE_BUILD_TYPE=Release
-        -DCMAKE_SKIP_BUILD_RPATH=TRUE
-        -DCMAKE_SKIP_INSTALL_RPATH=TRUE
-        -DCMAKE_SHARED_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-        -DCMAKE_EXE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-        -DCMAKE_MODULE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-        -GNinja
-      cmakeBuildDir: $(Build.SourcesDirectory)/llvm-project/openmp/build
-      installDir: $(Build.BinariesDirectory)/llvm
-# offload does not exist for recent releases, so use CI conditional
-  - ${{ if eq(parameters.offloadEnabled, true) }}:
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-      parameters:
-        componentName: llvm-offload
-        extraBuildFlags: >-
-          -DOPENMP_ENABLE_LIBOMPTARGET=1
-          -DOPENMP_TEST_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-          -DOPENMP_TEST_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
-          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-          -DLIBOMPTARGET_AMDGCN_GFXLIST=gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906;gfx908;gfx90a;gfx90c;gfx940;gfx941;gfx942;gfx1030;gfx1031;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103
-          -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm
-          -DLLVM_MAIN_INCLUDE_DIR=$(Build.SourcesDirectory)/llvm-project/llvm/include
-          -DLIBOMPTARGET_LLVM_INCLUDE_DIRS=$(Build.SourcesDirectory)/llvm-project/llvm/include
-          -DCUDA_TOOLKIT_ROOT_DIR=OFF
-          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/lib/cmake
-          -DCMAKE_BUILD_TYPE=Release
-          -DCMAKE_SKIP_BUILD_RPATH=TRUE
-          -DCMAKE_SKIP_INSTALL_RPATH=TRUE
-          -DCMAKE_SHARED_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-          -DCMAKE_EXE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-          -DCMAKE_MODULE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib
-          -GNinja
-        cmakeBuildDir: $(Build.SourcesDirectory)/llvm-project/offload/build
-        installDir: $(Build.BinariesDirectory)/llvm
+      sourceDir: $(Build.BinariesDirectory)
+      targetDir: $(Build.ArtifactStagingDirectory)
+  - task: DeleteFiles@1
+    displayName: 'Cleanup Binaries Directory'
+    inputs:
+      SourceFolder: '$(Build.BinariesDirectory)'
+      Contents: '/**/*'
+      RemoveDotFiles: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.ArtifactStagingDirectory)
+      targetDir: $(Build.BinariesDirectory)/llvm
+  - task: DeleteFiles@1
+    displayName: 'Cleanup Staging Directory'
+    inputs:
+      SourceFolder: $(Build.ArtifactStagingDirectory)
+      Contents: '/**/*'
+      RemoveDotFiles: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: aomp_testing
+  dependsOn: aomp
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: aomp-extras_repo
+# these copy steps are from the aomp prototype script for test prep
+  - task: CopyFiles@2
+    displayName: 'Copy AOMP contents'
+    inputs:
+      CleanTargetFolder: false
+      SourceFolder: $(Build.SourcesDirectory)/aomp
+      Contents: |
+        **
+        !**/.git/**
+        !**/.github/**
+        !**/.gitignore
+      TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests
+      retryCount: 3
+  - task: CopyFiles@2
+    displayName: 'Copy FileCheck'
+    inputs:
+      CleanTargetFolder: false
+      SourceFolder: $(Agent.BuildDirectory)/rocm/llvm/bin
+      Contents: FileCheck
+      TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests/bin
+      retryCount: 3
+  - task: Bash@3
+    displayName: Test AOMP
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: ./run_rocm_test.sh
+      workingDirectory: $(Build.SourcesDirectory)/aomp/bin
+    env:
+      AOMP: $(Agent.BuildDirectory)/rocm/llvm
+      AOMP_REPOS_TEST: $(Build.SourcesDirectory)/aomp-test
+      AOMP_TEST_DIR: $(Build.SourcesDirectory)/aomp-test
+      SKIP_TEST_PACKAGE: 1
+      MAINLINE_BUILD: 1
+      SUITE_LIST: smoke

.azuredevops/components/composable_kernel.yml

@@ -13,6 +13,7 @@ parameters:
     - git
     - python3-pip
     - libdrm-dev
+    - ccache
 - name: rocmDependencies
   type: object
   default:
@@ -22,13 +23,24 @@ parameters:
     - clr
     - rocminfo
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - composable_kernel
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: composable_kernel
-  timeoutInMinutes: 180
+  timeoutInMinutes: 240
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
+  - name: DAY_STRING
+    value: $[format('{0:ddMMyyyy}', pipeline.startTime)]
   pool: ${{ variables.ULTRA_BUILD_POOL }}
   workspace:
     clean: all
@@ -56,11 +68,26 @@ jobs:
       # manual build case: triggered by ROCm/ROCm repo
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - script: |
+      mkdir -p $(CCACHE_DIR)
+      echo "##vso[task.prependpath]/usr/lib/ccache"
+    displayName: Update path for ccache
+  - task: Cache@2
+    displayName: Ccache caching
+    inputs:
+      key: composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING) | $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+      path: $(CCACHE_DIR)
+      restoreKeys: |
+        composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING)
+        composable_kernel | $(Agent.OS) | $(JOB_GPU_TARGET)
+        composable_kernel | $(Agent.OS)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
         -DCMAKE_HIP_FLAGS="-Wno-missing-include-dirs"
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
@@ -69,3 +96,51 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: composable_kernel_testing
+  dependsOn: composable_kernel
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: TEST_LOG_FILE
+    value: $(Pipeline.Workspace)/ckTestLog.log
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Iterate through test scripts
+    inputs:
+      targetType: inline
+      script: |
+        for file in ./test_*; do
+          ./$file | tee -a $(TEST_LOG_FILE)
+        done
+      workingDirectory: $(Agent.BuildDirectory)/rocm/bin

.azuredevops/components/half.yml

@@ -53,3 +53,13 @@ jobs:
         -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  # only run test03 because test11 has too many test cases, taking way too long
+  - task: Bash@3
+    displayName: test03
+    inputs:
+      targetType: inline
+      script: |
+        make clean
+        make test03
+        ./bin/test
+      workingDirectory: $(Build.SourcesDirectory)/test

.azuredevops/components/hip-tests.yml

@@ -0,0 +1,144 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - ninja-build
+    - git
+    - libnuma-dev
+    - mesa-common-dev
+    - ocl-icd-libopencl1
+    - ocl-icd-opencl-dev
+    - opencl-headers
+    - libglu1-mesa-dev
+    - freeglut3-dev
+    - libboost-all-dev
+    - python3-pip
+- name: rocmDependencies
+  type: object
+  default:
+    - llvm-project
+    - ROCR-Runtime
+    - clr
+    - rocminfo
+    - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+
+# HIP with AMD backend
+jobs:
+- job: hip_tests
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: HIP_ROCCLR_HOME
+    value: $(Agent.BuildDirectory)/rocm
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  # compile hip-tests
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      componentName: hip-tests
+      cmakeSourceDir: '../catch'
+      customBuildTarget: build_tests
+      extraBuildFlags: >-
+        -DHIP_PLATFORM=amd
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DHIP_PATH=$(Agent.BuildDirectory)/rocm
+        -DOFFLOAD_ARCH_STR="--offload-arch=$(JOB_GPU_TARGET)"
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hip_tests_testing
+  timeoutInMinutes: 180
+  dependsOn: hip_tests
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Symlink rocm_agent_enumerator
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir -p /opt/rocm/bin
+        sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rocm_agent_enumerator /opt/rocm/bin/rocm_agent_enumerator
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hip_tests
+      testDir: $(Agent.BuildDirectory)/rocm/share/hip
+      reloadAMDGPU: true
+  - task: Bash@3
+    displayName: Clean up symlink
+    inputs:
+      targetType: inline
+      script: sudo rm -rf /opt/rocm

.azuredevops/components/hipBLAS.yml

@@ -31,6 +31,21 @@ parameters:
     - rocBLAS
     - rocSPARSE
     - rocSOLVER
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocm-cmake
+    - ROCR-Runtime
+    - rocminfo
+    - rocprofiler-register
+    - rocBLAS
+    - rocSPARSE
+    - rocSOLVER
 
 jobs:
 - job: hipBLAS
@@ -54,18 +69,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - task: Bash@3
-    displayName: 'Download AOCL'
-    inputs:
-      targetType: inline
-      script: wget -nv https://download.amd.com/developer/eula/aocl/aocl-4-1/aocl-linux-aocc-4.1.0_1_amd64.deb
-      workingDirectory: '$(Pipeline.Workspace)'
-  - task: Bash@3
-    displayName: 'Install AOCL'
-    inputs:
-      targetType: inline
-      script: sudo apt install --yes ./aocl-linux-aocc-4.1.0_1_amd64.deb
-      workingDirectory: '$(Pipeline.Workspace)'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
@@ -91,3 +95,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipBLAS_testing
+  dependsOn: hipBLAS
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipBLAS
+      testExecutable: $(Agent.BuildDirectory)/rocm/bin/hipblas-test
+      testParameters: '--yaml hipblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'

.azuredevops/components/hipBLASLt.yml

@@ -8,13 +8,17 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - gfortran
     - git
     - libdrm-dev
     - libmsgpack-dev
+    - libnuma-dev
     - ninja-build
     - python3-pip
     - python3-venv
+    - gfortran
+    - libomp-dev
+    - libblas-dev
+    - ccache
 - name: pipModules
   type: object
   default:
@@ -22,12 +26,27 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
+    - aomp
     - clr
     - hipBLAS-common
     - llvm-project
     - rocminfo
+    - rocm_smi_lib
     - rocprofiler-register
     - ROCR-Runtime
+    - roctracer
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - llvm-project
+    - rocminfo
+    - rocm_smi_lib
+    - rocprofiler-register
+    - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: hipBLASLt
@@ -47,7 +66,7 @@ jobs:
     value: $(Agent.BuildDirectory)/rocm/bin/hipcc
   - name: PATH
     value: $(Agent.BuildDirectory)/rocm/llvm/bin:$(Agent.BuildDirectory)/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
-  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  pool: ${{ variables.ULTRA_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -82,6 +101,7 @@ jobs:
 # $(Pipeline.Workspace)/deps is a temporary folder for the build process
 # $(Pipeline.Workspace)/s/deps is part of the hipBLASLt repo
   - script: mkdir $(Pipeline.Workspace)/deps
+    displayName: Create temp folder for external dependencies
 # hipBLASLt already has a CMake script for external deps, so we can just run that
 # https://github.com/ROCm/hipBLASLt/blob/develop/deps/CMakeLists.txt
   - script: cmake $(Pipeline.Workspace)/s/deps
@@ -93,19 +113,79 @@ jobs:
   - script: sudo make install
     displayName: Install hipBLASLt external dependencies
     workingDirectory: $(Pipeline.Workspace)/deps
+  - script: |
+      mkdir -p $(CCACHE_DIR)
+      echo "##vso[task.prependpath]/usr/lib/ccache"
+    displayName: Update path for ccache
+  - task: Cache@2
+    displayName: Ccache caching
+    inputs:
+      key: hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING) | $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+      path: $(CCACHE_DIR)
+      restoreKeys: |
+        hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET) | $(DAY_STRING)
+        hipBLASLt | $(Agent.OS) | $(JOB_GPU_TARGET)
+        hipBLASLt | $(Agent.OS)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_BUILD_TYPE=Release
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        -DCMAKE_C_COMPILER_LAUNCHER=ccache
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DTensile_LOGIC=
         -DTensile_CPU_THREADS=
         -DTensile_CODE_OBJECT_VERSION=default
         -DTensile_LIBRARY_FORMAT=msgpack
         -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
+        -DBUILD_CLIENTS_TESTS=ON
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+
+- job: hipBLASLt_testing
+  dependsOn: hipBLASLt
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipBLASLt
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipblaslt-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'

.azuredevops/components/hipCUB.yml

@@ -10,7 +10,7 @@ parameters:
   default:
     - cmake
     - ninja-build
-    - googletest
+    - libgtest-dev
     - git
     - python3-pip
 - name: rocmDependencies
@@ -21,6 +21,15 @@ parameters:
     - rocminfo
     - rocPRIM
     - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocPRIM
+    - ROCR-Runtime
+    - rocprofiler-register
 
 jobs:
 - job: hipCUB
@@ -66,3 +75,44 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipCUB_testing
+  dependsOn: hipCUB
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipCUB
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/hipcub'

.azuredevops/components/hipFFT.yml

@@ -27,6 +27,19 @@ parameters:
     - rocminfo
     - rocFFT
     - aomp
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipRAND
+    - llvm-project
+    - rocFFT
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+
 jobs:
 - job: hipFFT
   variables:
@@ -80,3 +93,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipFFT_testing
+  dependsOn: hipFFT
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipFFT
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipfft-test'
+      testParameters: '--test_prob 0.002 --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/hipRAND.yml

@@ -20,6 +20,15 @@ parameters:
     - clr
     - rocminfo
     - rocRAND
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
 
 jobs:
 - job: hipRAND
@@ -70,3 +79,44 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipRAND_testing
+  dependsOn: hipRAND
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipRAND
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/hipRAND'

.azuredevops/components/hipSOLVER.yml

@@ -27,6 +27,21 @@ parameters:
     - ROCR-Runtime
     - rocSPARSE
     - rocSOLVER
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - hipBLAS-common
+    - hipBLASLt
+    - hipSPARSE
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocSOLVER
+    - rocSPARSE
 
 jobs:
 - job: hipSOLVER
@@ -84,3 +99,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipSOLVER_testing
+  dependsOn: hipSOLVER
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipSOLVER
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipsolver-test'
+      testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/hipSPARSE.yml

@@ -25,6 +25,18 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocSPARSE
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - hipBLAS-common
+    - hipBLASLt
+    - rocBLAS
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocSPARSE
 
 jobs:
 - job: hipSPARSE
@@ -81,3 +93,46 @@ jobs:
     parameters:
       artifactName: testMatrices
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipSPARSE_testing
+  dependsOn: hipSPARSE
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipSPARSE
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipsparse-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/hipSPARSELt.yml

@@ -13,21 +13,37 @@ parameters:
     - libmsgpack-dev
     - git
     - python3-pip
+    - gfortran
+    - libgfortran5
+    - libomp-dev
+    - libopenblas-dev
 - name: pipModules
   type: object
   default:
     - joblib
-# rocm dependencies should match dependencies-rocm.yml
 - name: rocmDependencies
   type: object
   default:
-    - llvm-project
-    - ROCR-Runtime
+    - aomp
     - clr
+    - hipSPARSE
+    - llvm-project
+    - rocBLAS
     - rocminfo
     - rocprofiler-register
-    - hipSPARSE
+    - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - llvm-project
+    - hipBLAS-common
+    - hipBLASLt
     - rocBLAS
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: hipSPARSELt
@@ -71,6 +87,22 @@ jobs:
       # manual build case: triggered by ROCm/ROCm repo
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+# Build and install gtest and lapack
+# $(Pipeline.Workspace)/deps is a temporary folder for the build process
+# $(Pipeline.Workspace)/s/deps is part of the hipSPARSELt repo
+  - script: mkdir $(Pipeline.Workspace)/deps
+    displayName: Create temp folder for external dependencies
+# hipSPARSELt already has a CMake script for external deps, so we can just run that
+# https://github.com/ROCm/hipSPARSELt/blob/develop/deps/CMakeLists.txt
+  - script: cmake $(Pipeline.Workspace)/s/deps
+    displayName: Configure hipSPARSELt external dependencies
+    workingDirectory: $(Pipeline.Workspace)/deps
+  - script: make
+    displayName: Build hipSPARSELt external dependencies
+    workingDirectory: $(Pipeline.Workspace)/deps
+  - script: sudo make install
+    displayName: Install hipSPARSELt external dependencies
+    workingDirectory: $(Pipeline.Workspace)/deps
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -84,7 +116,45 @@ jobs:
         -DTensile_LIBRARY_FORMAT=msgpack
         -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DBUILD_CLIENTS_TESTS=ON
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipSPARSELt_testing
+  dependsOn: hipSPARSELt
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipSPARSELt
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './hipsparselt-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*'

.azuredevops/components/hipTensor.yml

@@ -21,6 +21,14 @@ parameters:
     - clr
     - rocminfo
     - composable_kernel
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: hipTensor
@@ -67,3 +75,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipTensor_testing
+  timeoutInMinutes: 90
+  dependsOn: hipTensor
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: hipTensor
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/hiptensor'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/hipfort.yml

@@ -11,26 +11,137 @@ parameters:
     - gfortran
     - git
     - cmake
+    - python3-pip
+    - python3-dev
+    - ninja-build
+- name: rocmDependencies
+  type: object
+  default:
+    - aomp
+    - llvm-project
+    - clr
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - hipFFT
+    - hipRAND
+    - hipSOLVER
+    - hipSPARSE
+    - rocBLAS
+    - rocFFT
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler
+    - rocprofiler-register
+    - rocRAND
+    - ROCR-Runtime
+    - rocSOLVER
+    - rocSPARSE
+    - roctracer
 
 jobs:
 - job: hipfort
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/llvm
+        -DHIPFORT_INSTALL_DIR=$(Build.BinariesDirectory)
+        -DHIPFORT_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/flang
+        -DCMAKE_Fortran_FLAGS="-Mfree -fPIC"
+        -DCMAKE_Fortran_FLAGS_DEBUG=""
+        -DHIPFORT_COMPILER_FLAGS="-cpp"
+        -DHIPFORT_AR=$(Agent.BuildDirectory)/rocm/llvm/bin/llvm-ar
+        -DHIPFORT_RANLIB=$(Agent.BuildDirectory)/rocm/llvm/bin/llvm-ranlib
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DBUILD_TESTING=ON
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: hipfort_testing
+  dependsOn: hipfort
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      extraBuildFlags: >-
-        -DHIPFORT_INSTALL_DIR=/tmp/hipfort
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - task: Bash@3
+    displayName: 'ROCm symbolic link'
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: 'Test hipfort'
+    inputs:
+      targetType: inline
+      script: PATH=$(Agent.BuildDirectory)/rocm/bin:$PATH make run_all
+      workingDirectory: $(Build.SourcesDirectory)/test

.azuredevops/components/llvm-project.yml

@@ -41,6 +41,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
+      skipLlvmSymlink: true
       # CI case: download latest default branch build
       ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
@@ -103,6 +104,14 @@ jobs:
       testExecutable: './bin/llvm-lit'
       testParameters: '-q --xunit-xml-output=lld_test_output.xml ./tools/lld/test'
       testOutputFile: lld_test_output.xml
+  - task: CopyFiles@2
+    displayName: Copy FileCheck for Publishing
+    inputs:
+      CleanTargetFolder: false
+      SourceFolder: llvm/build/bin
+      Contents: FileCheck
+      TargetFolder: $(Build.BinariesDirectory)/llvm/bin
+      retryCount: 3
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: device-libs

.azuredevops/components/rccl.yml

@@ -16,6 +16,7 @@ parameters:
     - git
     - ninja-build
     - libstdc++-12-dev
+    - libnuma-dev
 - name: rocmDependencies
   type: object
   default:
@@ -30,6 +31,20 @@ parameters:
     - HIPIFY
     - aomp
     - aomp-extras
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - aomp-extras
+    - clr
+    - HIPIFY
+    - llvm-project
+    - rocm-cmake
+    - rocm-core
+    - rocm_smi_lib
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rccl
@@ -51,10 +66,12 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+      submoduleBehaviour: recursive
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
@@ -81,3 +98,47 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rccl_testing
+  timeoutInMinutes: 120
+  dependsOn: rccl
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rccl
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rccl-UnitTests'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rdc.yml

@@ -18,16 +18,35 @@ parameters:
     - autoconf
     - libtool
     - pkg-config
+    - libdrm-dev
+    - libyaml-cpp-dev
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
+    - amdsmi
     - clr
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
     - rocminfo
     - rocm_smi_lib
+    - ROCmValidationSuite
+    - rocprofiler
+    - rocprofiler-register
+    - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
     - amdsmi
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocm_smi_lib
+    - ROCmValidationSuite
+    - rocprofiler
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rdc
@@ -37,6 +56,12 @@ jobs:
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -48,6 +73,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
       # CI case: download latest default branch build
       ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
@@ -71,11 +97,71 @@ jobs:
         -DBUILD_SHARED_LIBS=ON
         -DCMAKE_INSTALL_LIBDIR=lib
         -DCMAKE_BUILD_TYPE=Release
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DGRPC_ROOT="$(Build.SourcesDirectory)/bin"
+        -DBUILD_RVS=ON
+        -DBUILD_PROFILER=ON
         -DBUILD_TESTS=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rdc_testing
+  dependsOn: rdc
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo rm -rf /usr/sbin/rdcd
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rdcd /usr/sbin/rdcd
+        echo $(Agent.BuildDirectory)/rocm/lib/rdc/grpc/lib | sudo tee /etc/ld.so.conf.d/grpc.conf
+        sudo ldconfig -v
+  - task: Bash@3
+    displayName: Test rdc
+    inputs:
+      targetType: inline
+      script: >-
+        $(Agent.BuildDirectory)/rocm/share/rdc/rdctst_tests/rdctst
+        --batch_mode
+        --start_rdcd
+        --unauth_comm

.azuredevops/components/rocAL.yml

@@ -25,6 +25,7 @@ parameters:
     - libturbojpeg-dev
     - libjpeg-turbo-official=3.0.2-20240124
     - libopencv-dev
+    - --allow-downgrades # for libjpeg-turbo
 - name: pipModules
   type: object
   default:
@@ -44,6 +45,17 @@ parameters:
     - rpp
     - MIVisionX
     - aomp
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - half
+    - llvm-project
+    - MIVisionX
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rpp
 
 jobs:
 - job: rocAL
@@ -142,3 +154,81 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocAL_testing
+  dependsOn: rocAL
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: ROCM_PATH
+    value: $(Agent.BuildDirectory)/rocm
+  - name: CMAKE_INCLUDE_PATH
+    value: $(Agent.BuildDirectory)/rocm/include/rocal
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - task: Bash@3
+    displayName: 'Register libjpeg-turbo packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
+        sudo apt update
+        apt-cache show libjpeg-turbo-official | grep Version
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Link libjpeg-turbo
+    inputs:
+      targetType: inline
+      script: |
+        echo /opt/libjpeg-turbo/lib64 | sudo tee /etc/ld.so.conf.d/libjpeg-turbo.conf
+        sudo ldconfig -v
+  - task: Bash@3
+    displayName: Build rocAL tests
+    inputs:
+      targetType: inline
+      script: |
+        mkdir rocAL-tests
+        cd rocAL-tests
+        cmake $(Agent.BuildDirectory)/rocm/share/rocal/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocAL
+      testDir: rocAL-tests
+      reloadAMDGPU: true
+  - task: Bash@3
+    displayName: Clean up libjpeg-turbo
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm /etc/ld.so.conf.d/libjpeg-turbo.conf
+        sudo ldconfig -v

.azuredevops/components/rocALUTION.yml

@@ -28,6 +28,21 @@ parameters:
     - ROCR-Runtime
     - rocRAND
     - rocSPARSE
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocminfo
+    - rocPRIM
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+    - rocSPARSE
 
 jobs:
 - job: rocALUTION
@@ -79,3 +94,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocALUTION_testing
+  dependsOn: rocALUTION
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocALUTION
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocalution-test'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocBLAS.yml

@@ -24,6 +24,7 @@ parameters:
   type: object
   default:
     - joblib
+    - pyyaml
 - name: rocmDependencies
   type: object
   default:
@@ -37,6 +38,20 @@ parameters:
     - rocm-core
     - aomp
     - aomp-extras
+    - hipBLAS-common
+    - hipBLASLt
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - rocm_smi_lib
+    - ROCR-Runtime
 
 jobs:
 - job: rocBLAS
@@ -69,28 +84,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - task: Bash@3
-    displayName: 'Download AOCL'
-    inputs:
-      targetType: inline
-      workingDirectory: '$(Pipeline.Workspace)'
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        script: wget -nv https://download.amd.com/developer/eula/aocl/aocl-4-2/aocl-linux-gcc-4.2.0_1_amd64.deb
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        script: wget -nv https://download.amd.com/developer/eula/aocl/aocl-4-1/aocl-linux-aocc-4.1.0_1_amd64.deb
-  - task: Bash@3
-    displayName: 'Install AOCL'
-    inputs:
-      targetType: inline
-      workingDirectory: '$(Pipeline.Workspace)'
-      # CI case: download latest default branch build
-      ${{ if eq(parameters.checkoutRef, '') }}:
-        script: sudo apt install --yes ./aocl-linux-gcc-4.2.0_1_amd64.deb
-      # manual build case: triggered by ROCm/ROCm repo
-      ${{ elseif ne(parameters.checkoutRef, '') }}:
-        script: sudo apt install --yes ./aocl-linux-aocc-4.1.0_1_amd64.deb
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
@@ -123,3 +117,47 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocBLAS_testing
+  dependsOn: rocBLAS
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocBLAS
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocblas-test'
+      testParameters: '--yaml rocblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocDecode.yml

@@ -30,6 +30,13 @@ parameters:
     - rocminfo
     - rocm-core
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocDecode
@@ -78,3 +85,51 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocDecode_testing
+  dependsOn: rocDecode
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  # anything in /opt may be persistent across runs
+  # so we need to remove the symlink if it already exists
+  - script: |
+      sudo rm -rf /opt/rocm
+      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      mkdir rocDecode-tests
+      cd rocDecode-tests
+      cmake /opt/rocm/share/rocdecode/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocDecode
+      testDir: 'rocDecode-tests'
+  - script: sudo rm /opt/rocm

.azuredevops/components/rocFFT.yml

@@ -19,15 +19,28 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
-    - llvm-project
-    - ROCR-Runtime
+    - aomp
     - clr
+    - hipRAND
+    - llvm-project
+    - rocm-cmake
     - rocminfo
     - rocprofiler-register
-    - hipRAND
+    - ROCR-Runtime
     - rocRAND
-    - rocm-cmake
+- name: rocmTestDependencies
+  type: object
+  default:
     - aomp
+    - clr
+    - hipRAND
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+
 jobs:
 - job: rocFFT
   variables:
@@ -79,3 +92,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocFFT_testing
+  dependsOn: rocFFT
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocFFT
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocfft-test'
+      testParameters: '--test_prob 0.004 --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocMLIR.yml

@@ -12,14 +12,23 @@ parameters:
     - ninja-build
     - git
     - python3-pip
+    - libdrm-dev
+- name: pipModules
+  type: object
+  default:
+    - tomli
 - name: rocmDependencies
   type: object
   default:
     - llvm-project
     - rocm-cmake
+    - clr
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
-- job: rocMLIR
+- job: rocMLIR_library
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -30,6 +39,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
@@ -53,3 +63,63 @@ jobs:
         -DBUILD_FAT_LIBROCKCOMPILER=1
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+# compiling and running test on the test system together
+- job: rocMLIR_testing
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.GFX942_TEST_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      installEnabled: false
+      extraBuildFlags: >-
+        -DCMAKE_BUILD_TYPE=Release
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DROCM_TEST_CHIPSET=$(JOB_GPU_TARGET)
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocMLIR
+      testDir: $(Build.SourcesDirectory)/build
+      testExecutable: ninja
+      testParameters: check-rocmlir

.azuredevops/components/rocPRIM.yml

@@ -21,6 +21,14 @@ parameters:
     - ROCR-Runtime
     - clr
     - rocminfo
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler-register
 
 jobs:
 - job: rocPRIM
@@ -66,3 +74,44 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocPRIM_testing
+  dependsOn: rocPRIM
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocPRIM
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocprim'

.azuredevops/components/rocPyDecode.yml

@@ -0,0 +1,87 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - python3-pip
+    - cmake
+    - ninja-build
+    - pkg-config
+    - ffmpeg
+    - libavcodec-dev
+    - libavformat-dev
+    - libavutil-dev
+    - libva-dev
+    - libdrm-dev
+    - pybind11-dev
+    - python3-pybind11
+    - libdlpack-dev
+- name: pipModules
+  type: object
+  default:
+    - -i
+    - https://test.pypi.org/simple
+    - hip-python
+- name: rocmDependencies
+  type: object
+  default:
+    - rocm-cmake
+    - llvm-project
+    - ROCR-Runtime
+    - clr
+    - rocminfo
+    - rocm-core
+    - rocprofiler-register
+    - rocDecode
+
+jobs:
+- job: rocPyDecode
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+      gfx90a:
+        JOB_GPU_TARGET: gfx90a
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_INSTALL_PREFIX_PYTHON=$(Build.BinariesDirectory)
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocRAND.yml

@@ -21,6 +21,14 @@ parameters:
     - ROCR-Runtime
     - clr
     - rocminfo
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocRAND
@@ -68,3 +76,44 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocRAND_testing
+  dependsOn: rocRAND
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocRAND
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocRAND'

.azuredevops/components/rocSOLVER.yml

@@ -20,15 +20,30 @@ parameters:
 - name: rocmDependencies
   type: object
   default:
-    - rocm-cmake
-    - llvm-project
-    - ROCR-Runtime
     - clr
-    - rocminfo
-    - rocBLAS
-    - rocPRIM
-    - rocSPARSE
     - hipSPARSE
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - rocPRIM
+    - ROCR-Runtime
+    - rocSPARSE
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - hipSPARSE
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - rocPRIM
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocSPARSE
 
 jobs:
 - job: rocSOLVER
@@ -94,3 +109,46 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocSOLVER_testing
+  dependsOn: rocSOLVER
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocSOLVER
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocsolver-test'
+      testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocSPARSE.yml

@@ -29,6 +29,17 @@ parameters:
     - rocminfo
     - rocPRIM
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocSPARSE
@@ -91,3 +102,48 @@ jobs:
     parameters:
       artifactName: testMatrices
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocSPARSE_testing
+  timeoutInMinutes: 90
+  dependsOn: rocSPARSE
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocSPARSE
+      testDir: '$(Agent.BuildDirectory)/rocm/bin'
+      testExecutable: './rocsparse-test'
+      testParameters: '--gtest_filter="*quick*" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+

.azuredevops/components/rocThrust.yml

@@ -24,6 +24,16 @@ parameters:
     - rocminfo
     - rocPRIM
     - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocPRIM
+    - ROCR-Runtime
+    - hipRAND
+    - rocprofiler-register
 
 jobs:
 - job: rocThrust
@@ -69,3 +79,44 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocThrust_testing
+  dependsOn: rocThrust
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocThrust
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocthrust'

.azuredevops/components/rocWMMA.yml

@@ -28,6 +28,20 @@ parameters:
     - rocBLAS
     - aomp
     - rocm_smi_lib
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocm_smi_lib
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocWMMA
@@ -75,3 +89,45 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocWMMA_testing
+  timeoutInMinutes: 90
+  dependsOn: rocWMMA
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocWMMA
+      testDir: '$(Agent.BuildDirectory)/rocm/bin/rocwmma'

.azuredevops/components/rocm-cmake.yml

@@ -44,5 +44,7 @@ jobs:
       git config --global user.email "you@example.com"
       git config --global user.name "Your Name"
     displayName: "ctest setup"
-#  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm-cmake
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/rocm-core.yml

@@ -29,5 +29,5 @@ jobs:
         -DCPACK_GENERATOR=DEB
         -DCPACK_DEBIAN_PACKAGE_RELEASE="local.9999~99.99"
         -DCPACK_RPM_PACKAGE_RELEASE="local.9999"
-        -DROCM_VERSION="$(last-release)"
+        -DROCM_VERSION="$(next-release)"
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/rocm-examples.yml

@@ -8,20 +8,51 @@ parameters:
 - name: aptPackages
   type: object
   default:
+    - cmake
     - libglfw3-dev
+    - python3-pip
 - name: rocmDependencies
   type: object
   default:
     - AMDMIGraphX
     - clr
     - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
     - hipCUB
+    - hipFFT
     - HIPIFY
     - hipRAND
     - hipSOLVER
     - hipSPARSE
     - llvm-project
     - rocBLAS
+    - rocFFT
+    - rocPRIM
+    - rocprofiler-register
+    - ROCR-Runtime
+    - rocRAND
+    - rocSOLVER
+    - rocSPARSE
+    - rocThrust
+- name: rocmTestDependencies
+  type: object
+  default:
+    - AMDMIGraphX
+    - clr
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - hipCUB
+    - hipFFT
+    - HIPIFY
+    - hipRAND
+    - hipSOLVER
+    - hipSPARSE
+    - llvm-project
+    - rocBLAS
+    - rocFFT
+    - rocminfo
     - rocPRIM
     - rocprofiler-register
     - ROCR-Runtime
@@ -35,8 +66,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
   strategy:
@@ -46,13 +76,13 @@ jobs:
       gfx90a:
         JOB_GPU_TARGET: gfx90a
   steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
-    parameters:
-      aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
@@ -70,8 +100,72 @@ jobs:
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCM_ROOT=$(Agent.BuildDirectory)/rocm
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
         -DCMAKE_EXE_LINKER_FLAGS=-fgpu-rdc
+  - task: Bash@3
+    displayName: Move rocm-examples binaries to rocm/examples
+    inputs:
+      targetType: inline
+      script: |
+        mkdir -p $(Build.BinariesDirectory)/examples
+        mv $(Build.BinariesDirectory)/bin/* $(Build.BinariesDirectory)/examples
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocm_examples_testing
+  dependsOn: rocm_examples
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: TEST_LOG_FILE
+    value: $(Pipeline.Workspace)/rocm-examplesTestLog.log
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Unload and reload AMDGPU
+    inputs:
+      targetType: inline
+      script: |
+        sudo modprobe -r amdgpu
+        sudo modprobe amdgpu
+  - task: Bash@3
+    displayName: Iterate through examples
+    inputs:
+      targetType: inline
+      script: |
+        for file in *; do
+          echo Now running: $file
+          ./$file | tee -a $(TEST_LOG_FILE)
+        done
+      workingDirectory: $(Agent.BuildDirectory)/rocm/examples

.azuredevops/components/rocm_bandwidth_test.yml

@@ -24,7 +24,11 @@ parameters:
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
-    - ROCT-Thunk-Interface
+- name: rocmTestDependencies
+  type: object
+  default:
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocm_bandwidth_test
@@ -65,3 +69,40 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/include;$(Agent.BuildDirectory)/rocm/include/hsa
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocm_bandwidth_test_testing
+  dependsOn: rocm_bandwidth_test
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm_bandwidth_test
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/bin/rocm-bandwidth-test'
+      testParameters: ''
+      testPublishResults: false

.azuredevops/components/rocm_smi_lib.yml

@@ -26,3 +26,27 @@ jobs:
         -DBUILD_TESTS=ON
         -DROCM_DEP_ROCMCORE=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocm_smi_lib_testing
+  dependsOn: rocm_smi_lib
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm_smi_lib
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/share/rocm_smi/rsmitst_tests/rsmitst'
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'

.azuredevops/components/rocminfo.yml

@@ -10,6 +10,11 @@ parameters:
   default:
     - ROCR-Runtime
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - ROCR-Runtime
+    - rocprofiler-register
 
 jobs:
 - job: rocminfo
@@ -28,6 +33,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
+      skipLlvmSymlink: true
       # CI case: download latest default branch build
       ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
@@ -40,3 +46,43 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCRTST_BLD_TYPE=release
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocminfo_testing
+  dependsOn: rocminfo
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocminfo
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/bin/rocminfo'
+      testParameters: ''
+      testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm_agent_enumerator
+      testDir: '$(Agent.BuildDirectory)'
+      testExecutable: './rocm/bin/rocm_agent_enumerator'
+      testParameters: ''
+      testPublishResults: false

.azuredevops/components/rocprofiler.yml

@@ -39,7 +39,6 @@ parameters:
     - rocminfo
     - ROCR-Runtime
     - rocprofiler-register
-    - ROCT-Thunk-Interface
     - roctracer
 
 jobs:
@@ -69,29 +68,12 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
-# Manually download aqlprofile, hard-coded 6.1.0 version
-  - task: Bash@3
-    displayName: 'Download aqlprofile'
-    inputs:
-      targetType: inline
-      script: wget -nv https://repo.radeon.com/rocm/misc/aqlprofile/ubuntu-22.04/hsa-amd-aqlprofile_1.0.0.60200.60200-crdnnh.14213~22.04_amd64.deb
-      workingDirectory: '$(Pipeline.Workspace)'
-  - task: Bash@3
-    displayName: 'Extract aqlprofile'
-    inputs:
-      targetType: inline
-      script: |
-        mkdir hsa-amd-aqlprofile
-        dpkg-deb -R hsa-amd-aqlprofile_1.0.0.60200.60200-crdnnh.14213~22.04_amd64.deb hsa-amd-aqlprofile
-      workingDirectory: '$(Pipeline.Workspace)'
-  - task: Bash@3
-    displayName: 'Move aqlprofile'
-    inputs:
-      targetType: inline
-      script: |
-        mkdir -p $(Agent.BuildDirectory)/rocm
-        cp -R hsa-amd-aqlprofile/opt/rocm-6.2.0-14213/* $(Agent.BuildDirectory)/rocm
-      workingDirectory: '$(Pipeline.Workspace)'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}

.azuredevops/components/rocr_debug_agent.yml

@@ -12,6 +12,8 @@ parameters:
     - ninja-build
     - libelf-dev
     - libdw-dev
+    - libstdc++-12-dev
+    - python-is-python3
 - name: rocmDependencies
   type: object
   default:
@@ -22,6 +24,15 @@ parameters:
     - rocminfo
     - ROCR-Runtime
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - ROCdbgapi
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rocr_debug_agent
@@ -58,3 +69,52 @@ jobs:
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocr_debug_agent_testing
+  dependsOn: rocr_debug_agent
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      componentName: rocr_debug_agent-tests
+      extraBuildFlags: >-
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake;$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+      cmakeBuildDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'
+      cmakeSourceDir: '.'
+      installEnabled: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocr_debug_agent
+      testDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'

.azuredevops/components/roctracer.yml

@@ -25,6 +25,13 @@ parameters:
     - rocprofiler-register
     - clr
     - rocminfo
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: roctracer
@@ -74,3 +81,47 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: roctracer_testing
+  dependsOn: roctracer
+  condition: succeeded()
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: roctracer
+      testExecutable: $(Agent.BuildDirectory)/rocm/share/roctracer/run_tests.sh
+      testParameters: ''
+      testDir: $(Agent.BuildDirectory)
+      testPublishResults: false

.azuredevops/nightly/pytorch.yml

@@ -54,6 +54,14 @@ parameters:
     - libdrm-amdgpu1
     - ccache
     - zip
+    - libjpeg-turbo-official
+    - libjpeg-dev
+    - libwebp-dev
+    - libfreetype-dev
+    - gnutls-bin
+    - ffmpeg
+    - libopenblas-dev
+    - liblapack-dev
 - name: pipModules
   type: object
   default:
@@ -76,6 +84,9 @@ parameters:
     - ninja
     - packaging
     - optree>=0.12.0
+  # list for vision
+    - auditwheel
+    - future
 # list from https://github.com/pytorch/builder/blob/main/manywheel/build_rocm.sh
 - name: rocmDependencies
   type: object
@@ -146,8 +157,6 @@ jobs:
     value: 3.10
   - name: PYTORCH_ROOT
     value: $(Build.SourcesDirectory)/pytorch
-  - name: CMAKE_ARGS
-    value: -GNinja
   - name: DESIRED_DEVTOOLSET
     value: cxx11-abi
   pool: ${{ variables.ULTRA_BUILD_POOL }}
@@ -157,6 +166,16 @@ jobs:
 # copy environment setup from https://github.com/pytorch/builder/blob/main/manywheel/Dockerfile
 # but instead of centos, use ubuntu environment
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
+  - task: Bash@3
+    displayName: 'Register libjpeg-turbo packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
+        sudo apt update
+        apt-cache show libjpeg-turbo-official | grep Version
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
@@ -193,12 +212,6 @@ jobs:
       targetType: inline
       script: git clone https://github.com/pytorch/pytorch.git --depth=1 --recurse-submodules
       workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: Patch out forced GPU testing block in pytorch build script
-    inputs:
-      targetType: inline
-      script: git apply $(Build.SourcesDirectory)/.azuredevops/patches/pytorch/0001-ROCm-CI-patches.patch
-      workingDirectory: $(Build.SourcesDirectory)/builder
   - task: Bash@3
     displayName: Install patchelf
     inputs:
@@ -231,8 +244,7 @@ jobs:
     displayName: Install AOTriton Shared Library
     inputs:
       targetType: inline
-      script: |
-        sudo bash ./common/install_aotriton.sh /opt/rocm
+      script: sudo bash ./common/install_aotriton.sh /opt/rocm
       workingDirectory: $(Build.SourcesDirectory)/pytorch/.ci/docker
   - task: Bash@3
     displayName: Run ROCm Build Script
@@ -245,16 +257,65 @@ jobs:
         GPU_TARGET=$(JOB_GPU_TARGET)
         DESIRED_PYTHON=$(DESIRED_PYTHON)
         PYTORCH_ROOT=$(PYTORCH_ROOT)
-        CMAKE_ARGS=$(CMAKE_ARGS)
+        CMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         AOTRITON_INSTALLED_PREFIX=$(AOTRITON_INSTALLED_PREFIX)
         DESIRED_DEVTOOLSET=$(DESIRED_DEVTOOLSET)
         TORCH_PACKAGE_NAME=torch.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
         PYTORCH_BUILD_VERSION=$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)
         PYTORCH_BUILD_NUMBER=$(date -u +%Y%m%d)
+        SKIP_ALL_TESTS=1
         bash ./manywheel/build_rocm.sh
       workingDirectory: $(Build.SourcesDirectory)/builder
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: /remote/wheelhouserocm$(ROCM_VERSION)
+      contentsString: '*.whl'
+# common helper source for pytorch vision and audio
+  - task: Bash@3
+    displayName: git clone pytorch test-infra
+    inputs:
+      targetType: inline
+      script: git clone https://github.com/pytorch/test-infra.git --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: install package helper
+    inputs:
+      targetType: inline
+      script: python3 -m pip install test-infra/tools/pkg-helpers
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: pytorch pkg helpers
+    inputs:
+      targetType: inline
+      script: CU_VERSION=${CU_VERSION} CHANNEL=${CHANNEL} python -m pytorch_pkg_helpers
+# get torch vision source and build
+  - task: Bash@3
+    displayName: git clone pytorch vision
+    inputs:
+      targetType: inline
+      script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Build vision
+    inputs:
+      targetType: inline
+      script: >-
+        BUILD_VERSION=$(ROCM_BRANCH).$(JOB_GPU_TARGET)-$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)$(date -u +%Y%m%d)
+        python3 setup.py bdist_wheel
+      workingDirectory: $(Build.SourcesDirectory)/vision
+  - task: Bash@3
+    displayName: Relocate vision
+    inputs:
+      targetType: inline
+      script: python3 packaging/wheel/relocate.py
+      workingDirectory: $(Build.SourcesDirectory)/vision
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.SourcesDirectory)/vision/dist
+      contentsString: '*.whl'
+      clean: false
   - task: PublishPipelineArtifact@1
-    displayName: 'ROCm pytorch wheel file Publish'
+    displayName: 'wheel file Publish'
     retryCountOnTaskFailure: 3
     inputs:
-      targetPath: /remote/wheelhouserocm$(ROCM_VERSION)
+      targetPath: $(Build.BinariesDirectory)

.azuredevops/nightly/rocm-nightly.yml

@@ -49,7 +49,6 @@ parameters:
     - rocr_debug_agent
     - rocSOLVER
     - rocSPARSE
-    - ROCT-Thunk-Interface
     - rocThrust
     - roctracer
     - rocWMMA

.azuredevops/patches/pytorch/0001-ROCm-CI-patches.patch

@@ -1,40 +0,0 @@
-From b2d3c88f7a8b179e814e72f76f27e25c82659200 Mon Sep 17 00:00:00 2001
-From: Joseph Macaranas <Joseph.Macaranas@amd.com>
-Date: Tue, 30 Jul 2024 05:43:06 -0400
-Subject: [PATCH] ROCm CI patches
-
----
- manywheel/build_common.sh | 9 ---------
- 1 file changed, 9 deletions(-)
-
-diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh
-index 08ca924..52c468f 100644
---- a/manywheel/build_common.sh
-+++ b/manywheel/build_common.sh
-@@ -475,11 +475,9 @@ if [[ -z "$BUILD_PYTHONLESS" ]]; then
-   fi
- 
-   pip uninstall -y "$TORCH_PACKAGE_NAME"
--  
-   if [[ "$USE_SPLIT_BUILD" == "true" ]]; then
-     pip install "$TORCH_NO_PYTHON_PACKAGE_NAME" --no-index -f /$WHEELHOUSE_DIR --no-dependencies -v
-   fi
--  
-   pip install "$TORCH_PACKAGE_NAME" --no-index -f /$WHEELHOUSE_DIR --no-dependencies -v
- 
-   # Print info on the libraries installed in this wheel
-@@ -491,11 +489,4 @@ if [[ -z "$BUILD_PYTHONLESS" ]]; then
-       ldd "$installed_lib" || true
-   done
- 
--  # Run the tests
--  echo "$(date) :: Running tests"
--  pushd "$PYTORCH_ROOT"
--  LD_LIBRARY_PATH=/usr/local/nvidia/lib64 \
--          "${SOURCE_DIR}/../run_tests.sh" manywheel "${py_majmin}" "$DESIRED_CUDA"
--  popd
--  echo "$(date) :: Finished tests"
- fi
--- 
-2.44.0.windows.1
-

.azuredevops/tag-builds/aomp.yml

@@ -9,16 +9,26 @@ parameters:
 
 resources:
   repositories:
-  - repository: release_repo
+  - repository: aomp_repo
     type: github
     endpoint: ROCm
     name: ROCm/aomp
     ref: ${{ parameters.checkoutRef }}
+  - repository: aomp-extras_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp-extras
+    ref: ${{ parameters.checkoutRef }}
+  - repository: flang_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/flang
+    ref: ${{ parameters.checkoutRef }}
   - repository: llvm-project_repo
     type: github
     endpoint: ROCm
     name: ROCm/llvm-project
-    ref: ${{ parameters.checkoutRef }}
+    ref: amd-staging
 
 trigger: none
 pr: none
@@ -26,5 +36,5 @@ pr: none
 jobs:
   - template: ${{ variables.CI_COMPONENT_PATH }}/aomp.yml
     parameters:
-      checkoutRepo: release_repo
+      checkoutRepo: aomp_repo
       checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/hip-tests.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/hip-tests
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/hip-tests.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocPyDecode.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocPyDecode
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocPyDecode.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/templates/steps/artifact-download.yml

@@ -64,16 +64,27 @@ parameters:
     rocPRIM: develop
     rocprofiler-register: amd-mainline
     rocprofiler: amd-staging
-    ROCR-Runtime: master
+    ROCR-Runtime: amd-staging
     rocRAND: develop
     rocr_debug_agent: amd-staging
     rocSOLVER: develop
     rocSPARSE: develop
-    ROCT-Thunk-Interface: master
     rocThrust: develop
     roctracer: amd-staging
     rocWMMA: develop
-    rpp: master
+    rpp: develop
+- name: allowPartiallySucceededBuilds
+  type: object
+  default:
+    - amdsmi
+    - aomp
+    - HIPIFY
+    - rccl
+    - rdc
+    - rocm-cmake
+    - rocm_smi_lib
+    - rocFFT
+    - MIVisionX
 # BELOW REQUIRED IF useDefaultBranch false
 - name: branchName
   type: string
@@ -95,7 +106,7 @@ steps:
       branchName: refs/heads/${{ parameters.defaultBranchList[parameters.componentName] }}
     ${{ else }}:
       branchName: ${{ parameters.branchName }}
-    ${{ if in(parameters.componentName, 'rocm-cmake') }}:
+    ${{ if containsValue(parameters.allowPartiallySucceededBuilds, parameters.componentName) }}:
       allowPartiallySucceededBuilds: true
     targetPath: '$(Pipeline.Workspace)/d'
 - task: ExtractFiles@1

.azuredevops/templates/steps/build-cmake.yml

@@ -14,6 +14,9 @@ parameters:
 - name: cmakeSourceDir
   type: string
   default: '..'
+- name: customBuildTarget
+  type: string
+  default: ''
 - name: cmakeTarget
   type: string
   default: 'install'
@@ -49,7 +52,10 @@ steps:
   displayName: '${{parameters.componentName }} Build'
   inputs:
     workingDirectory: ${{ parameters.cmakeBuildDir }}
-    cmakeArgs: '--build ${{ parameters.cmakeTargetDir }} ${{ parameters.multithreadFlag }}'
+    ${{ if eq(parameters.customBuildTarget, '') }}:
+      cmakeArgs: '--build ${{ parameters.cmakeTargetDir }} ${{ parameters.multithreadFlag }}'
+    ${{ else }}:
+      cmakeArgs: '--build ${{ parameters.cmakeTargetDir }} --target ${{ parameters.customBuildTarget }} ${{ parameters.multithreadFlag }}'
     retryCountOnTaskFailure: 10
 - script: df -h
   displayName: Disk space after build

.azuredevops/templates/steps/ccache.yml

@@ -1,11 +1,14 @@
 # to use ccache, run this template step before build-cmake.yml
 # and add this cmake flag to extraBuildFlags
 # -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
 steps:
 - task: Cache@2
   displayName: Ccache caching
   inputs:
     key: 'ccache | "$(Agent.OS)" | $(Build.SourceVersion)'
-    path: $(CCACHE_DIR)
+    path: ${{ variables.CCACHE_DIR }}
     restoreKeys: |
       ccache | "$(Agent.OS)"

.azuredevops/templates/steps/dependencies-aocl.yml

@@ -0,0 +1,27 @@
+parameters:
+- name: repositoryUrl
+  type: string
+  default: https://download.amd.com/developer/eula/aocl/aocl-4-2
+- name: packageName
+  type: string
+  default: aocl-linux-gcc-4.2.0_1_amd64.deb
+
+steps:
+- task: Bash@3
+  displayName: Download AOCL
+  inputs:
+    targetType: inline
+    workingDirectory: $(Pipeline.Workspace)
+    script: wget -nv ${{ parameters.repositoryUrl }}/${{ parameters.packageName }}
+- task: Bash@3
+  displayName: Install AOCL
+  inputs:
+    targetType: inline
+    workingDirectory: $(Pipeline.Workspace)
+    script: sudo apt install -y ./${{ parameters.packageName }}
+- task: Bash@3
+  displayName: Clean up AOCL
+  inputs:
+    targetType: inline
+    workingDirectory: $(Pipeline.Workspace)
+    script: rm -f ${{ parameters.packageName }}

.azuredevops/templates/steps/dependencies-aqlprofile.yml

@@ -0,0 +1,47 @@
+parameters:
+- name: dependencySource
+  type: string
+  default: staging
+  values:
+    - staging
+    - tag-builds
+- name: repositoryUrl
+  type: object
+  default: 
+    staging: https://repo.radeon.com/rocm/apt/6.2/pool/main/h/hsa-amd-aqlprofile
+    tag-builds: https://repo.radeon.com/rocm/apt/6.2/pool/main/h/hsa-amd-aqlprofile
+- name: packageName
+  type: object
+  default:
+    staging: hsa-amd-aqlprofile_1.0.0.60200.60200-66~22.04_amd64.deb
+    tag-builds: hsa-amd-aqlprofile_1.0.0.60200.60200-66~22.04_amd64.deb
+
+steps:
+- task: Bash@3
+  displayName: 'Download aqlprofile'
+  inputs:
+    targetType: inline
+    script: wget -nv ${{ parameters.repositoryUrl[parameters.dependencySource] }}/${{ parameters.packageName[parameters.dependencySource] }}
+    workingDirectory: '$(Pipeline.Workspace)'
+- task: Bash@3
+  displayName: 'Extract aqlprofile'
+  inputs:
+    targetType: inline
+    script: |
+      mkdir hsa-amd-aqlprofile
+      dpkg-deb -R ${{ parameters.packageName[parameters.dependencySource] }} hsa-amd-aqlprofile
+    workingDirectory: '$(Pipeline.Workspace)'
+- task: Bash@3
+  displayName: 'Copy aqlprofile files'
+  inputs:
+    targetType: inline
+    script: |
+      mkdir -p $(Agent.BuildDirectory)/rocm
+      cp -R hsa-amd-aqlprofile/opt/rocm-*/* $(Agent.BuildDirectory)/rocm
+    workingDirectory: '$(Pipeline.Workspace)'
+- task: Bash@3
+  displayName: 'Clean up aqlprofile'
+  inputs:
+    targetType: inline
+    script: rm -rf hsa-amd-aqlprofile ${{ parameters.packageName[parameters.dependencySource] }}
+    workingDirectory: '$(Pipeline.Workspace)'

.azuredevops/templates/steps/dependencies-cmake-latest.yml

@@ -5,6 +5,6 @@ steps:
   inputs:
     targetType: inline
     script: |
-      sudo apt purge cmake
+      sudo apt purge cmake -y
       sudo snap install cmake --classic
       hash -r

.azuredevops/templates/steps/dependencies-other.yml

@@ -12,31 +12,23 @@ steps:
   displayName: 'sudo apt-get update'
   inputs:
     targetType: inline
-    script: sudo apt-get --yes update
-  env:
-    DEBIAN_FRONTEND: noninteractive
+    script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes update
 - task: Bash@3
   displayName: 'sudo apt-get upgrade'
   inputs:
     targetType: inline
-    script: sudo apt-get --yes upgrade
-  env:
-    DEBIAN_FRONTEND: noninteractive
+    script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes upgrade
 - task: Bash@3
   displayName: 'sudo apt-get fix'
   inputs:
     targetType: inline
-    script: sudo apt --yes --fix-broken install
-  env:
-    DEBIAN_FRONTEND: noninteractive
+    script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes --fix-broken install
 - ${{ if gt(length(parameters.aptPackages), 0) }}:
   - task: Bash@3
     displayName: 'sudo apt-get install ...'
     inputs:
       targetType: inline
-      script: sudo apt-get --yes install ${{ join(' ', parameters.aptPackages) }}
-    env:
-      DEBIAN_FRONTEND: noninteractive
+      script: sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install ${{ join(' ', parameters.aptPackages) }}
 - ${{ if gt(length(parameters.pipModules), 0) }}:
   - task: Bash@3
     displayName: 'pip install  ...'

.azuredevops/templates/steps/dependencies-rocm.yml

@@ -29,115 +29,115 @@ parameters:
 - name: stagingPipelineIdentifiers
   type: object
   default:
-    AMDMIGraphX: $(amdmigraphx-pipeline-id)
-    amdsmi: $(amdsmi-pipeline-id)
-    aomp-extras: $(aomp-extras-pipeline-id)
-    aomp: $(aomp-pipeline-id)
-    clr: $(clr-pipeline-id)
-    composable_kernel: $(composable-kernel-pipeline-id)
-    half: $(half-pipeline-id)
-    HIP: $(hip-pipeline-id)
-    hipBLAS: $(hipblas-pipeline-id)
-    hipBLASLt: $(hipblaslt-pipeline-id)
-    hipBLAS-common: $(hipblas-common-pipeline-id)
-    hipCUB: $(hipcub-pipeline-id)
-    hipFFT: $(hipfft-pipeline-id)
-    hipfort: $(hipfort-pipeline-id)
-    HIPIFY: $(hipify-pipeline-id)
-    hipRAND: $(hiprand-pipeline-id)
-    hipSOLVER: $(hipsolver-pipeline-id)
-    hipSPARSE: $(hipsparse-pipeline-id)
-    hipSPARSELt: $(hipsparselt-pipeline-id)
-    hipTensor: $(hiptensor-pipeline-id)
-    llvm-project: $(llvm-project-pipeline-id)
-    MIOpen: $(miopen-pipeline-id)
-    MIVisionX: $(mivisionx-pipeline-id)
-    rccl: $(rccl-pipeline-id)
-    rdc: $(rdc-pipeline-id)
-    rocAL: $(rocal-pipeline-id)
-    rocALUTION: $(rocalution-pipeline-id)
-    rocBLAS: $(rocblas-pipeline-id)
-    ROCdbgapi : $(rocdbgapi-pipeline-id)
-    rocDecode: $(rocdecode-pipeline-id)
-    rocFFT: $(rocfft-pipeline-id)
-    ROCgdb: $(rocgdb-pipeline-id)
-    rocm-cmake: $(rocm-cmake-pipeline-id)
-    rocm-core: $(rocm-core-pipeline-id)
-    rocm-examples: $(rocm-examples-pipeline-id)
-    rocminfo: $(rocminfo-pipeline-id)
-    rocMLIR: $(rocmlir-pipeline-id)
-    ROCmValidationSuite: $(rocmvalidationsuite-pipeline-id)
-    rocm_bandwidth_test: $(rocm-bandwidth-test-pipeline-id)
-    rocm_smi_lib: $(rocm-smi-lib-pipeline-id)
-    rocPRIM: $(rocprim-pipeline-id)
-    rocprofiler-register: $(rocprofiler-register-pipeline-id)
-    rocprofiler: $(rocprofiler-pipeline-id)
-    ROCR-Runtime: $(rocr-runtime-pipeline-id)
-    rocRAND: $(rocrand-pipeline-id)
-    rocr_debug_agent: $(rocr-debug-agent-pipeline-id)
-    rocSOLVER: $(rocsolver-pipeline-id)
-    rocSPARSE: $(rocsparse-pipeline-id)
-    ROCT-Thunk-Interface: $(roct-thunk-interface-pipeline-id)
-    rocThrust: $(rocthrust-pipeline-id)
-    roctracer: $(roctracer-pipeline-id)
-    rocWMMA: $(rocwmma-pipeline-id)
-    rpp: $(rpp-pipeline-id)
+    AMDMIGraphX: $(AMDMIGRAPHX_PIPELINE_ID)
+    amdsmi: $(AMDSMI_PIPELINE_ID)
+    aomp-extras: $(AOMP_EXTRAS_PIPELINE_ID)
+    aomp: $(AOMP_PIPELINE_ID)
+    clr: $(CLR_PIPELINE_ID)
+    composable_kernel: $(COMPOSABLE_KERNEL_PIPELINE_ID)
+    half: $(HALF_PIPELINE_ID)
+    HIP: $(HIP_PIPELINE_ID)
+    hipBLAS: $(HIPBLAS_PIPELINE_ID)
+    hipBLAS-common: $(HIPBLAS_COMMON_PIPELINE_ID)
+    hipBLASLt: $(HIPBLASLT_PIPELINE_ID)
+    hipCUB: $(HIPCUB_PIPELINE_ID)
+    hipFFT: $(HIPFFT_PIPELINE_ID)
+    hipfort: $(HIPFORT_PIPELINE_ID)
+    HIPIFY: $(HIPIFY_PIPELINE_ID)
+    hipRAND: $(HIPRAND_PIPELINE_ID)
+    hipSOLVER: $(HIPSOLVER_PIPELINE_ID)
+    hipSPARSE: $(HIPSPARSE_PIPELINE_ID)
+    hipSPARSELt: $(HIPSPARSELT_PIPELINE_ID)
+    hipTensor: $(HIPTENSOR_PIPELINE_ID)
+    llvm-project: $(LLVM_PROJECT_PIPELINE_ID)
+    MIOpen: $(MIOpen_PIPELINE_ID)
+    MIVisionX: $(MIVISIONX_PIPELINE_ID)
+    rccl: $(RCCL_PIPELINE_ID)
+    rdc: $(RDC_PIPELINE_ID)
+    rocAL: $(ROCAL_PIPELINE_ID)
+    rocALUTION: $(ROCALUTION_PIPELINE_ID)
+    rocBLAS: $(ROCBLAS_PIPELINE_ID)
+    ROCdbgapi: $(ROCDBGAPI_PIPELINE_ID)
+    rocDecode: $(ROCDECODE_PIPELINE_ID)
+    rocFFT: $(ROCFFT_PIPELINE_ID)
+    ROCgdb: $(ROCGDB_PIPELINE_ID)
+    rocm-cmake: $(ROCM_CMAKE_PIPELINE_ID)
+    rocm-core: $(ROCM_CORE_PIPELINE_ID)
+    rocm-examples: $(ROCM_EXAMPLES_PIPELINE_ID)
+    rocminfo: $(ROCMINFO_PIPELINE_ID)
+    rocMLIR: $(ROCMLIR_PIPELINE_ID)
+    ROCmValidationSuite: $(ROCMVALIDATIONSUITE_PIPELINE_ID)
+    rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_PIPELINE_ID)
+    rocm_smi_lib: $(ROCM_SMI_LIB_PIPELINE_ID)
+    rocPRIM: $(ROCPRIM_PIPELINE_ID)
+    rocprofiler-register: $(ROCPROFILER_REGISTER_PIPELINE_ID)
+    rocprofiler: $(ROCPROFILER_PIPELINE_ID)
+    ROCR-Runtime: $(ROCR_RUNTIME_PIPELINE_ID)
+    rocRAND: $(ROCRAND_PIPELINE_ID)
+    rocr_debug_agent: $(ROCR_DEBUG_AGENT_PIPELINE_ID)
+    rocSOLVER: $(ROCSOLVER_PIPELINE_ID)
+    rocSPARSE: $(ROCSPARSE_PIPELINE_ID)
+    ROCT-Thunk-Interface: $(ROCT_THUNK_INTERFACE_PIPELINE_ID)
+    rocThrust: $(ROCTHRUST_PIPELINE_ID)
+    roctracer: $(ROCTRACER_PIPELINE_ID)
+    rocWMMA: $(ROCWMMA_PIPELINE_ID)
+    rpp: $(RPP_PIPELINE_ID)
 - name: taggedPipelineIdentifiers
   type: object
   default:
-    AMDMIGraphX: $(amdmigraphx-tagged-pipeline-id)
-    amdsmi: $(amdsmi-tagged-pipeline-id)
-    aomp-extras: $(aomp-extras-tagged-pipeline-id)
-    aomp: $(aomp-tagged-pipeline-id)
-    clr: $(clr-tagged-pipeline-id)
-    composable_kernel: $(composable-kernel-tagged-pipeline-id)
-    half: $(half-tagged-pipeline-id)
-    HIP: $(hip-tagged-pipeline-id)
-    hipBLAS: $(hipblas-tagged-pipeline-id)
-    hipBLASLt: $(hipblaslt-tagged-pipeline-id)
-    hipBLAS-common: $(hipblas-common-tagged-pipeline-id)
-    hipCUB: $(hipcub-tagged-pipeline-id)
-    hipFFT: $(hipfft-tagged-pipeline-id)
-    hipfort: $(hipfort-tagged-pipeline-id)
-    HIPIFY: $(hipify-tagged-pipeline-id)
-    hipRAND: $(hiprand-tagged-pipeline-id)
-    hipSOLVER: $(hipsolver-tagged-pipeline-id)
-    hipSPARSE: $(hipsparse-tagged-pipeline-id)
-    hipSPARSELt: $(hipsparselt-tagged-pipeline-id)
-    hipTensor: $(hiptensor-tagged-pipeline-id)
-    llvm-project: $(llvm-project-tagged-pipeline-id)
-    MIOpen: $(miopen-tagged-pipeline-id)
-    MIVisionX: $(mivisionx-tagged-pipeline-id)
-    rccl: $(rccl-tagged-pipeline-id)
-    rdc: $(rdc-tagged-pipeline-id)
-    rocAL: $(rocal-tagged-pipeline-id)
-    rocALUTION: $(rocalution-tagged-pipeline-id)
-    rocBLAS: $(rocblas-tagged-pipeline-id)
-    ROCdbgapi : $(rocdbgapi-tagged-pipeline-id)
-    rocDecode: $(rocdecode-tagged-pipeline-id)
-    rocFFT: $(rocfft-tagged-pipeline-id)
-    ROCgdb: $(rocgdb-tagged-pipeline-id)
-    rocm-cmake: $(rocm-cmake-tagged-pipeline-id)
-    rocm-core: $(rocm-core-tagged-pipeline-id)
-    rocm-examples: $(rocm-examples-tagged-pipeline-id)
-    rocminfo: $(rocminfo-tagged-pipeline-id)
-    rocMLIR: $(rocmlir-tagged-pipeline-id)
-    ROCmValidationSuite: $(rocmvalidationsuite-tagged-pipeline-id)
-    rocm_bandwidth_test: $(rocm-bandwidth-test-tagged-pipeline-id)
-    rocm_smi_lib: $(rocm-smi-lib-tagged-pipeline-id)
-    rocPRIM: $(rocprim-tagged-pipeline-id)
-    rocprofiler-register: $(rocprofiler-register-tagged-pipeline-id)
-    rocprofiler: $(rocprofiler-tagged-pipeline-id)
-    ROCR-Runtime: $(rocr-runtime-tagged-pipeline-id)
-    rocRAND: $(rocrand-tagged-pipeline-id)
-    rocr_debug_agent: $(rocr-debug-agent-tagged-pipeline-id)
-    rocSOLVER: $(rocsolver-tagged-pipeline-id)
-    rocSPARSE: $(rocsparse-tagged-pipeline-id)
-    ROCT-Thunk-Interface: $(roct-thunk-interface-tagged-pipeline-id)
-    rocThrust: $(rocthrust-tagged-pipeline-id)
-    roctracer: $(roctracer-tagged-pipeline-id)
-    rocWMMA: $(rocwmma-tagged-pipeline-id)
-    rpp: $(rpp-tagged-pipeline-id)
+    AMDMIGraphX: $(AMDMIGRAPHX_TAGGED_PIPELINE_ID)
+    amdsmi: $(AMDSMI_TAGGED_PIPELINE_ID)
+    aomp-extras: $(AOMP_EXTRAS_TAGGED_PIPELINE_ID)
+    aomp: $(AOMP_TAGGED_PIPELINE_ID)
+    clr: $(CLR_TAGGED_PIPELINE_ID)
+    composable_kernel: $(COMPOSABLE_KERNEL_TAGGED_PIPELINE_ID)
+    half: $(HALF_TAGGED_PIPELINE_ID)
+    HIP: $(HIP_TAGGED_PIPELINE_ID)
+    hipBLAS: $(HIPBLAS_TAGGED_PIPELINE_ID)
+    hipBLAS-common: $(HIPBLAS_COMMON_TAGGED_PIPELINE_ID)
+    hipBLASLt: $(HIPBLASLT_TAGGED_PIPELINE_ID)
+    hipCUB: $(HIPCUB_TAGGED_PIPELINE_ID)
+    hipFFT: $(HIPFFT_TAGGED_PIPELINE_ID)
+    hipfort: $(HIPFORT_TAGGED_PIPELINE_ID)
+    HIPIFY: $(HIPIFY_TAGGED_PIPELINE_ID)
+    hipRAND: $(HIPRAND_TAGGED_PIPELINE_ID)
+    hipSOLVER: $(HIPSOLVER_TAGGED_PIPELINE_ID)
+    hipSPARSE: $(HIPSPARSE_TAGGED_PIPELINE_ID)
+    hipSPARSELt: $(HIPSPARSELT_TAGGED_PIPELINE_ID)
+    hipTensor: $(HIPTENSOR_TAGGED_PIPELINE_ID)
+    llvm-project: $(LLVM_PROJECT_TAGGED_PIPELINE_ID)
+    MIOpen: $(MIOpen_TAGGED_PIPELINE_ID)
+    MIVisionX: $(MIVISIONX_TAGGED_PIPELINE_ID)
+    rccl: $(RCCL_TAGGED_PIPELINE_ID)
+    rdc: $(RDC_TAGGED_PIPELINE_ID)
+    rocAL: $(ROCAL_TAGGED_PIPELINE_ID)
+    rocALUTION: $(ROCALUTION_TAGGED_PIPELINE_ID)
+    rocBLAS: $(ROCBLAS_TAGGED_PIPELINE_ID)
+    ROCdbgapi: $(ROCDBGAPI_TAGGED_PIPELINE_ID)
+    rocDecode: $(ROCDECODE_TAGGED_PIPELINE_ID)
+    rocFFT: $(ROCFFT_TAGGED_PIPELINE_ID)
+    ROCgdb: $(ROCGDB_TAGGED_PIPELINE_ID)
+    rocm-cmake: $(ROCM_CMAKE_TAGGED_PIPELINE_ID)
+    rocm-core: $(ROCM_CORE_TAGGED_PIPELINE_ID)
+    rocm-examples: $(ROCM_EXAMPLES_TAGGED_PIPELINE_ID)
+    rocminfo: $(ROCMINFO_TAGGED_PIPELINE_ID)
+    rocMLIR: $(ROCMLIR_TAGGED_PIPELINE_ID)
+    ROCmValidationSuite: $(ROCMVALIDATIONSUITE_TAGGED_PIPELINE_ID)
+    rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_TAGGED_PIPELINE_ID)
+    rocm_smi_lib: $(ROCM_SMI_LIB_TAGGED_PIPELINE_ID)
+    rocPRIM: $(ROCPRIM_TAGGED_PIPELINE_ID)
+    rocprofiler-register: $(ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID)
+    rocprofiler: $(ROCPROFILER_TAGGED_PIPELINE_ID)
+    ROCR-Runtime: $(ROCR_RUNTIME_TAGGED_PIPELINE_ID)
+    rocRAND: $(ROCRAND_TAGGED_PIPELINE_ID)
+    rocr_debug_agent: $(ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID)
+    rocSOLVER: $(ROCSOLVER_TAGGED_PIPELINE_ID)
+    rocSPARSE: $(ROCSPARSE_TAGGED_PIPELINE_ID)
+    ROCT-Thunk-Interface: $(ROCT_THUNK_INTERFACE_TAGGED_PIPELINE_ID)
+    rocThrust: $(ROCTHRUST_TAGGED_PIPELINE_ID)
+    roctracer: $(ROCTRACER_TAGGED_PIPELINE_ID)
+    rocWMMA: $(ROCWMMA_TAGGED_PIPELINE_ID)
+    rpp: $(RPP_TAGGED_PIPELINE_ID)
 # set to true if you're calling this template file multiple files in same pipeline
 # only leave last call false to optimize sequence
 - name: skipLibraryLinking
@@ -241,6 +241,14 @@ steps:
     inputs:
       targetType: inline
       script: sudo ln -s $(Agent.BuildDirectory)/rocm/llvm $(Agent.BuildDirectory)/rocm/lib/llvm
+  - task: Bash@3
+    displayName: Symlink executables from rocm/llvm/bin to rocm/bin
+    inputs:
+      targetType: inline
+      script: |
+        for file in amdclang amdclang++ amdclang-cl amdclang-cpp amdflang amdlld aompcc mygpu mycpu offload-arch; do
+          sudo ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/$file $(Agent.BuildDirectory)/rocm/bin/$file
+        done
 - task: Bash@3
   displayName: 'List downloaded ROCm files'
   inputs:
@@ -257,15 +265,15 @@ steps:
 # OS ignores if the ROCm lib folder shows up more than once
       ${{ if eq(parameters.extractToMnt, true) }}:
         script: |
-          echo /mnt/rocm/lib | sudo tee -a /etc/ld.so.conf
-          echo /mnt/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf
-          sudo cat /etc/ld.so.conf
+          echo /mnt/rocm/lib | sudo tee /etc/ld.so.conf.d/rocm-ci.conf
+          echo /mnt/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
+          sudo cat /etc/ld.so.conf.d/rocm-ci.conf
           sudo ldconfig -v
           ldconfig -p
       ${{ else }}:
         script: |
-          echo $(Agent.BuildDirectory)/rocm/lib | sudo tee -a /etc/ld.so.conf
-          echo $(Agent.BuildDirectory)/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf
-          sudo cat /etc/ld.so.conf
+          echo $(Agent.BuildDirectory)/rocm/lib | sudo tee /etc/ld.so.conf.d/rocm-ci.conf
+          echo $(Agent.BuildDirectory)/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
+          sudo cat /etc/ld.so.conf.d/rocm-ci.conf
           sudo ldconfig -v
           ldconfig -p

.azuredevops/templates/steps/local-artifact-download.yml

@@ -0,0 +1,45 @@
+parameters:
+  - name: gpuTarget
+    type: string
+    default: ''
+  # Set buildType to specific to download artifacts from previous builds, useful for saving time when debugging
+  - name: buildType
+    type: string
+    default: current
+    values:
+      - current
+      - specific
+  # Must be set if buildType == specific
+  # Set definitionId to the pipeline ID and buildId to the specific build ID
+  - name: definitionId
+    type: string
+    default: 0
+  - name: buildId
+    type: string
+    default: 0
+
+steps:
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Build'
+    inputs:
+      ${{ if eq(parameters.buildType, 'specific') }}:
+        buildType: specific
+        buildVersionToDownload: specific
+        project: ROCm-CI
+        definition: ${{ parameters.definitionId }}
+        buildId: ${{ parameters.buildId }}
+      itemPattern: '**/*${{ parameters.gpuTarget }}*'
+      targetPath: $(System.ArtifactsDirectory)
+  - task: ExtractFiles@1
+    displayName: 'Extract Pipeline Build'
+    inputs:
+      archiveFilePatterns: '$(System.ArtifactsDirectory)/**/*.tar.gz'
+      destinationFolder: '$(Agent.BuildDirectory)/rocm'
+      cleanDestinationFolder: false
+      overwriteExistingFiles: true
+  - task: DeleteFiles@1
+    displayName: 'Clean up Compressed Pipeline Build'
+    inputs:
+      SourceFolder: '$(System.ArtifactsDirectory)'
+      Contents: '/**/*.tar.xz'
+      RemoveDotFiles: true

.azuredevops/templates/steps/test.yml

@@ -10,15 +10,36 @@ parameters:
   default: 'ctest'
 - name: testParameters
   type: string
-  default: '--output-on-failure --force-new-ctest-process --output-junit test_output.xml'
+  default: '-VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml'
 - name: testOutputFile
   type: string
   default: test_output.xml
 - name: testOutputFormat
   type: string
-  default: 'junit'
+  default: 'JUnit'
+  values:
+    - JUnit
+    - NUnit
+    - VSTest
+    - XUnit
+    - CTest
+- name: testPublishResults
+  type: boolean
+  default: true
+- name: reloadAMDGPU
+  type: boolean
+  default: false
 
 steps:
+# Avoids occasional AMDGPU driver issues with opening /dev/kfd
+- ${{ if parameters.reloadAMDGPU }}:
+  - task: Bash@3
+    displayName: Unload and reload AMDGPU
+    inputs:
+      targetType: inline
+      script: |
+        sudo modprobe -r amdgpu
+        sudo modprobe amdgpu
 # run test, continue on failure to publish results
 # and to publish build artifacts
 - task: Bash@3
@@ -28,9 +49,11 @@ steps:
     targetType: inline
     script: ${{ parameters.testExecutable }} ${{ parameters.testParameters }}
     workingDirectory: ${{ parameters.testDir }}
-- task: PublishTestResults@2
-  displayName: '${{ parameters.componentName }} Publish Results'
-  inputs:
-    testResultsFormat: ${{ parameters.testOutputFormat }}
-    testResultsFiles: '**/${{ parameters.testOutputFile }}'
-  condition: succeededOrFailed()
+- ${{ if parameters.testPublishResults }}:
+  - task: PublishTestResults@2
+    displayName: '${{ parameters.componentName }} Publish Results'
+    inputs:
+      searchFolder: ${{ parameters.testDir }}
+      testResultsFormat: ${{ parameters.testOutputFormat }}
+      testResultsFiles: '**/${{ parameters.testOutputFile }}'
+    condition: succeededOrFailed()

.azuredevops/variables-global.yml

@@ -23,6 +23,8 @@ variables:
   value: rocm-ci_build_pool
 - name: LARGE_DISK_BUILD_POOL
   value: rocm-ci_larger_base_disk_pool
+- name: GFX942_TEST_POOL
+  value: gfx942_test_pool
 - name: LATEST_RELEASE_TAG
   value: rocm-6.1.0
 - name: DOCKER_IMAGE_NAME
@@ -31,3 +33,275 @@ variables:
   value: 6.1
 - name: KEYRING_VERSION
   value: 6.1
+- name: AMDMIGRAPHX_GFX942_TEST_PIPELINE_ID
+  value: 197
+- name: AMDMIGRAPHX_PIPELINE_ID
+  value: 113
+- name: AMDMIGRAPHX_TAGGED_PIPELINE_ID
+  value: 60
+- name: AMDSMI_PIPELINE_ID
+  value: 99
+- name: AMDSMI_TAGGED_PIPELINE_ID
+  value: 33
+- name: AOMP_EXTRAS_PIPELINE_ID
+  value: 111
+- name: AOMP_EXTRAS_TAGGED_PIPELINE_ID
+  value: 75
+- name: AOMP_PIPELINE_ID
+  value: 115
+- name: AOMP_TAGGED_PIPELINE_ID
+  value: 76
+- name: CCACHE_DIR
+  value: $(Pipeline.Workspace)/ccache
+- name: CLR_PIPELINE_ID
+  value: 145
+- name: CLR_TAGGED_PIPELINE_ID
+  value: 71
+- name: COMPOSABLE_KERNEL_GFX942_TEST_PIPELINE_ID
+  value: 179
+- name: COMPOSABLE_KERNEL_PIPELINE_ID
+  value: 86
+- name: COMPOSABLE_KERNEL_TAGGED_PIPELINE_ID
+  value: 38
+- name: FLANG_LEGACY_PIPELINE_ID
+  value: 77
+- name: FLANG_LEGACY_TAGGED_PIPELINE_ID
+  value: 77
+- name: HALF_PIPELINE_ID
+  value: 101
+- name: HALF_TAGGED_PIPELINE_ID
+  value: 11
+- name: HALF560_PIPELINE_ID
+  value: 66
+- name: HALF560_TAGGED_PIPELINE_ID
+  value: 66
+- name: HIP_PIPELINE_ID
+  value: 93
+- name: HIP_TAGGED_PIPELINE_ID
+  value: 31
+- name: HIPBLAS_COMMON_PIPELINE_ID
+  value: 223
+- name: HIPBLAS_COMMON_TAGGED_PIPELINE_ID
+  value: 224
+- name: HIPBLAS_GFX942_TEST_PIPELINE_ID
+  value: 202
+- name: HIPBLAS_PIPELINE_ID
+  value: 87
+- name: HIPBLAS_TAGGED_PIPELINE_ID
+  value: 44
+- name: HIPBLASLT_GFX942_TEST_PIPELINE_ID
+  value: 187
+- name: HIPBLASLT_PIPELINE_ID
+  value: 112
+- name: HIPBLASLT_TAGGED_PIPELINE_ID
+  value: 45
+- name: HIPCUB_GFX942_TEST_PIPELINE_ID
+  value: 186
+- name: HIPCUB_PIPELINE_ID
+  value: 97
+- name: HIPCUB_TAGGED_PIPELINE_ID
+  value: 46
+- name: HIPFFT_GFX942_TEST_PIPELINE_ID
+  value: 198
+- name: HIPFFT_PIPELINE_ID
+  value: 121
+- name: HIPFFT_TAGGED_PIPELINE_ID
+  value: 12
+- name: HIPFORT_PIPELINE_ID
+  value: 102
+- name: HIPFORT_TAGGED_PIPELINE_ID
+  value: 34
+- name: HIPIFY_PIPELINE_ID
+  value: 92
+- name: HIPIFY_TAGGED_PIPELINE_ID
+  value: 13
+- name: HIPRAND_GFX942_TEST_PIPELINE_ID
+  value: 188
+- name: HIPRAND_PIPELINE_ID
+  value: 90
+- name: HIPRAND_TAGGED_PIPELINE_ID
+  value: 42
+- name: HIPSOLVER_GFX942_TEST_PIPELINE_ID
+  value: 201
+- name: HIPSOLVER_PIPELINE_ID
+  value: 84
+- name: HIPSOLVER_TAGGED_PIPELINE_ID
+  value: 52
+- name: HIPSPARSE_GFX942_TEST_PIPELINE_ID
+  value: 195
+- name: HIPSPARSE_PIPELINE_ID
+  value: 83
+- name: HIPSPARSE_TAGGED_PIPELINE_ID
+  value: 14
+- name: HIPSPARSELT_GFX942_TEST_PIPELINE_ID
+  value: 200
+- name: HIPSPARSELT_PIPELINE_ID
+  value: 104
+- name: HIPSPARSELT_TAGGED_PIPELINE_ID
+  value: 53
+- name: HIPTENSOR_GFX942_TEST_PIPELINE_ID
+  value: 192
+- name: HIPTENSOR_PIPELINE_ID
+  value: 105
+- name: HIPTENSOR_TAGGED_PIPELINE_ID
+  value: 56
+- name: LAST_RELEASE
+  value: 6.1.0
+- name: LLVM_PROJECT_PIPELINE_ID
+  value: 2
+- name: LLVM_PROJECT_TAGGED_PIPELINE_ID
+  value: 8
+- name: MIOPEN_PIPELINE_ID
+  value: 108
+- name: MIOPEN_TAGGED_PIPELINE_ID
+  value: 58
+- name: MIVISIONX_PIPELINE_ID
+  value: 80
+- name: MIVISIONX_TAGGED_PIPELINE_ID
+  value: 18
+- name: RCCL_GFX942_TEST_PIPELINE_ID
+  value: 184
+- name: RCCL_PIPELINE_ID
+  value: 107
+- name: RCCL_TAGGED_PIPELINE_ID
+  value: 15
+- name: RDC_PIPELINE_ID
+  value: 100
+- name: RDC_TAGGED_PIPELINE_ID
+  value: 59
+- name: REIMAGE_ORG
+  value: AGS-ROCm-CI
+- name: REIMAGE_REPO
+  value: cirrascale-reimage-automation
+- name: ROCAL_PIPELINE_ID
+  value: 151
+- name: ROCALUTION_GFX942_TEST_PIPELINE_ID
+  value: 196
+- name: ROCALUTION_PIPELINE_ID
+  value: 89
+- name: ROCALUTION_TAGGED_PIPELINE_ID
+  value: 16
+- name: ROCBLAS_GFX942_TEST_PIPELINE_ID
+  value: 185
+- name: ROCBLAS_PIPELINE_ID
+  value: 85
+- name: ROCBLAS_TAGGED_PIPELINE_ID
+  value: 32
+- name: ROCDBGAPI_PIPELINE_ID
+  value: 135
+- name: ROCDBGAPI_TAGGED_PIPELINE_ID
+  value: 17
+- name: ROCDECODE_PIPELINE_ID
+  value: 79
+- name: ROCDECODE_TAGGED_PIPELINE_ID
+  value: 21
+- name: ROCFFT_GFX942_TEST_PIPELINE_ID
+  value: 189
+- name: ROCFFT_PIPELINE_ID
+  value: 120
+- name: ROCFFT_TAGGED_PIPELINE_ID
+  value: 19
+- name: ROCGDB_PIPELINE_ID
+  value: 134
+- name: ROCGDB_TAGGED_PIPELINE_ID
+  value: 50
+- name: ROCM_BANDWIDTH_TEST_PIPELINE_ID
+  value: 88
+- name: ROCM_BANDWIDTH_TEST_TAGGED_PIPELINE_ID
+  value: 23
+- name: ROCM_CMAKE_PIPELINE_ID
+  value: 6
+- name: ROCM_CMAKE_TAGGED_PIPELINE_ID
+  value: 7
+- name: ROCM_CORE_PIPELINE_ID
+  value: 103
+- name: ROCM_CORE_TAGGED_PIPELINE_ID
+  value: 22
+- name: ROCM_EXAMPLES_GFX942_TEST_PIPELINE_ID
+  value: 204
+- name: ROCM_SMI_LIB_PIPELINE_ID
+  value: 96
+- name: ROCM_SMI_LIB_TAGGED_PIPELINE_ID
+  value: 47
+- name: ROCMINFO_PIPELINE_ID
+  value: 91
+- name: ROCMINFO_TAGGED_PIPELINE_ID
+  value: 27
+- name: ROCMLIR_PIPELINE_ID
+  value: 229
+- name: ROCMLIR_TAGGED_PIPELINE_ID
+  value: 62
+- name: ROCMVALIDATIONSUITE_PIPELINE_ID
+  value: 106
+- name: ROCMVALIDATIONSUITE_TAGGED_PIPELINE_ID
+  value: 43
+- name: ROCPRIM_GFX942_TEST_PIPELINE_ID
+  value: 180
+- name: ROCPRIM_PIPELINE_ID
+  value: 82
+- name: ROCPRIM_TAGGED_PIPELINE_ID
+  value: 20
+- name: ROCPROFILER_GFX942_TEST_PIPELINE_ID
+  value: 190
+- name: ROCPROFILER_PIPELINE_ID
+  value: 143
+- name: ROCPROFILER_REGISTER_PIPELINE_ID
+  value: 1
+- name: ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID
+  value: 25
+- name: ROCPROFILER_TAGGED_PIPELINE_ID
+  value: 28
+- name: ROCR_DEBUG_AGENT_PIPELINE_ID
+  value: 136
+- name: ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID
+  value: 29
+- name: ROCR_RUNTIME_PIPELINE_ID
+  value: 10
+- name: ROCR_RUNTIME_TAGGED_PIPELINE_ID
+  value: 24
+- name: ROCRAND_GFX942_TEST_PIPELINE_ID
+  value: 183
+- name: ROCRAND_PIPELINE_ID
+  value: 95
+- name: ROCRAND_TAGGED_PIPELINE_ID
+  value: 41
+- name: ROCSOLVER_GFX942_TEST_PIPELINE_ID
+  value: 199
+- name: ROCSOLVER_PIPELINE_ID
+  value: 81
+- name: ROCSOLVER_TAGGED_PIPELINE_ID
+  value: 55
+- name: ROCSPARSE_GFX942_TEST_PIPELINE_ID
+  value: 191
+- name: ROCSPARSE_PIPELINE_ID
+  value: 98
+- name: ROCSPARSE_TAGGED_PIPELINE_ID
+  value: 67
+- name: ROCT_THUNK_INTERFACE_PIPELINE_ID
+  value: 3
+- name: ROCT_THUNK_INTERFACE_TAGGED_PIPELINE_ID
+  value: 9
+- name: ROCTHRUST_GFX942_TEST_PIPELINE_ID
+  value: 194
+- name: ROCTHRUST_PIPELINE_ID
+  value: 94
+- name: ROCTHRUST_TAGGED_PIPELINE_ID
+  value: 26
+- name: ROCTRACER_GFX942_TEST_PIPELINE_ID
+  value: 181
+- name: ROCTRACER_PIPELINE_ID
+  value: 141
+- name: ROCTRACER_TAGGED_PIPELINE_ID
+  value: 30
+- name: ROCWMMA_GFX942_TEST_PIPELINE_ID
+  value: 193
+- name: ROCWMMA_PIPELINE_ID
+  value: 109
+- name: ROCWMMA_TAGGED_PIPELINE_ID
+  value: 57
+- name: RPP_GFX942_TEST_PIPELINE_ID
+  value: 182
+- name: RPP_PIPELINE_ID
+  value: 78
+- name: RPP_TAGGED_PIPELINE_ID
+  value: 39

.wordlist.txt

@@ -53,6 +53,7 @@ CSC
 CSE
 CSV
 CSn
+CTest
 CTests
 CU
 CUDA
@@ -97,6 +98,7 @@ ENDPGM
 EPYC
 ESXi
 EoS
+FBGEMM
 FFT
 FFTs
 FFmpeg
@@ -110,6 +112,7 @@ Flang
 Fortran
 Fuyu
 GALB
+GCC
 GCD
 GCDs
 GCN
@@ -175,6 +178,7 @@ Interop
 Intersphinx
 Intra
 Ioffe
+Jinja
 JSON
 Jupyter
 KFD
@@ -221,6 +225,7 @@ Megatron
 Mellanox
 Mellanox's
 Meta's
+Miniconda
 MirroredStrategy
 Multicore
 Multithreaded
@@ -383,6 +388,7 @@ UAC
 UC
 UCC
 UCX
+UE
 UIF
 UMC
 USM
@@ -451,6 +457,7 @@ blit
 bootloader
 boson
 bosons
+br
 buildable
 bursty
 bzip
@@ -578,6 +585,7 @@ invariants
 invocating
 ipo
 kdb
+kfd
 latencies
 libfabric
 libjpeg
@@ -620,6 +628,7 @@ performant
 perl
 pragma
 pre
+prebuild
 prebuilt
 precompiled
 preconditioner
@@ -646,6 +655,7 @@ quasirandom
 queueing
 rccl
 rdc
+rdma
 reStructuredText
 redirections
 refactorization
@@ -711,8 +721,10 @@ subexpression
 subfolder
 subfolders
 submodule
+submodules
 supercomputing
 symlink
+symlinks
 td
 tensorfloat
 th

README.md

@@ -50,7 +50,7 @@ The following example shows how to use the repo tool to download the ROCm source
 ```bash
 mkdir -p ~/ROCm/
 cd ~/ROCm/
-~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.0.x
+~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.2.x
 ~/bin/repo sync
 ```
 
@@ -76,7 +76,7 @@ The Build time will reduce significantly if we limit the GPU Architecture/s agai
 
 mkdir -p ~/WORKSPACE/      # Or any folder name other than WORKSPACE
 cd ~/WORKSPACE/
-export ROCM_VERSION=6.2.0   
+export ROCM_VERSION=6.2.1
 ~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.2.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
 ~/bin/repo sync
 

default.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <manifest>
     <remote name="rocm-org" fetch="https://github.com/ROCm/" />
-    <default revision="refs/tags/rocm-6.2.0"
+    <default revision="refs/tags/rocm-6.2.1"
      remote="rocm-org"
      sync-c="true"
      sync-j="4" />
@@ -25,7 +25,6 @@
 <!--HIP Projects-->
     <project name="HIP" />
     <project name="hip-tests" />
-    <project name="HIP-Examples" />
     <project name="HIPIFY" />
     <project name="clr" />
     <project name="hipother" />

docs/compatibility/compatibility-matrix-historical-6.0.csv

@@ -0,0 +1,117 @@
+ROCm Version,6.2.1,6.2.0, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
+      :ref:`Operating Systems & kernels <OS-kernel-versions>`,Ubuntu 24.04,Ubuntu 24.04,,,,,
+      ,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3, 22.04.2","Ubuntu 22.04.4, 22.04.3, 22.04.2"
+      ,,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
+      ,"RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
+      ,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
+      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
+      ,,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
+      ,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,,,
+      ,.. _architecture-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
+      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
+      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
+      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
+      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
+      ,.. _gpu-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
+      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
+      ,gfx942 [#mi300_621-past-60]_,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
+      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
+      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
+      ,,,,,,,
+      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
+      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1","2.14.0, 2.13.1, 2.12.1"
+      :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
+      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
+      ,,,,,,,
+      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix-past-60:,,,,,,
+      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
+      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
+      ,,,,,,,
+      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix-past-60:,,,,,,
+      Thrust,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+      CUB,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+      ,,,,,,,
+      KFD & USER SPACE [#kfd_support-past-60]_,.. _kfd-userspace-support-compatibility-matrix-past-60:,,
+      Tested user space versions,"6.1.x, 6.0.x","6.1.x, 6.0.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x, 5.6.x","6.2.x, 6.0.x, 5.7.x, 5.6.x"
+      ,,,,,,,
+      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
+      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.10.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
+      :doc:`MIOpen <miopen:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`MIVisionX <mivisionx:index>`,3.0.0,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
+      :doc:`rocAL <rocal:index>`,2.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
+      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,0.1.0,N/A,N/A,N/A,N/A,N/A
+      :doc:`RPP <rpp:index>`,1.8.0,1.8.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
+      ,,,,,,,
+      COMMUNICATION,.. _commlibs-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`RCCL <rccl:index>`,2.20.5,2.20.5,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
+      ,,,,,,,
+      MATH LIBS,.. _mathlibs-support-compatibility-matrix-past-60:,,,,,,
+      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
+      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
+      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.8.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
+      :doc:`hipFFT <hipfft:index>`,1.0.15,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
+      :doc:`hipFORT <hipfort:index>`,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
+      :doc:`hipRAND <hiprand:index>`,2.11.0,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
+      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.2.0,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
+      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.1.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.1,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
+      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.2.0,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
+      :doc:`rocBLAS <rocblas:index>`,4.2.1,4.2.0,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
+      :doc:`rocFFT <rocfft:index>`,1.0.29,1.0.28,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
+      :doc:`rocRAND <rocrand:index>`,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
+      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.26.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
+      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.2.0,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
+      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.5.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
+      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
+      ,,,,,,,
+      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`hipCUB <hipcub:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.3.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
+      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`rocThrust <rocthrust:index>`,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      ,,,,,,,
+      SUPPORT LIBS,,,,,,,
+      `hipother <https://github.com/ROCm/hipother>`_,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.1,6.2.0,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
+      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.4.05,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
+      ,,,,,,,
+      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`AMD SMI <amdsmi:index>`,24.6.3,24.6.2,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
+      :doc:`ROCm Data Center Tool <rdc:index>`,1.0.0,1.0.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
+      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.3.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
+      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.1,rocm-6.2.0,rocm-6.1.2,rocm-6.1.1,rocm-6.1.0,rocm-6.0.2,rocm-6.0.0
+      ,,,,,,,
+      PERFORMANCE TOOLS,,,,,,,
+      :doc:`Omniperf <omniperf:index>`,2.0.1,2.0.1,N/A,N/A,N/A,N/A,N/A
+      :doc:`Omnitrace <omnitrace:index>`,1.11.2,1.11.2,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
+      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60201,2.0.60200,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
+      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,0.4.0,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCTracer <roctracer:index>`,4.1.60201,4.1.60200,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
+      ,,,,,,,
+      DEVELOPMENT TOOLS,,,,,,,
+      :doc:`HIPIFY <hipify:index>`,18.0.0.24332,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.13.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
+      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
+      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.2.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
+      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.3.0,0.3.0,0.3.0,N/A,N/A
+      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
+      ,,,,,,,
+      COMPILERS,.. _compilers-support-compatibility-matrix-past-60:,,,,,,
+      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
+      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24332,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`llvm-project <llvm-project:index>`,18.0.0.24332,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24332,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      ,,,,,,,
+      RUNTIMES,.. _runtime-support-compatibility-matrix-past-60:,,,,,,
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      :doc:`HIP <hip:index>`,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
+      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.14.0,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0

docs/compatibility/compatibility-matrix.rst

@@ -1,144 +1,189 @@
 .. meta::
     :description: ROCm compatibility matrix
-    :keywords: AMD, GPU, architecture, hardware, compatibility, requirements
+    :keywords: GPU, architecture, hardware, compatibility, system, requirements, components, libraries
 
 **************************************************************************************
 Compatibility matrix
 **************************************************************************************
 
-Use this matrix to view the ROCm compatibility across successive major and minor releases.
+Use this matrix to view the ROCm compatibility and system requirements across successive major and minor releases.
 
 You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-rocm-compatibility-matrix>`.
 
+.. |br| raw:: html
+
+   <br/>
+
 .. container:: format-big-table
 
   .. csv-table:: 
-      :header: "ROCm Version", "6.2.0", "6.1.2", "6.0.0"
+      :header: "ROCm Version", "6.2.1", "6.2.0", "6.1.0"
       :stub-columns: 1
 
-      :doc:`Operating Systems <rocm-install-on-linux:reference/system-requirements>`, "Ubuntu 24.04","",""
-      ,"Ubuntu 22.04.5 [#Ubuntu220405]_, 22.04.4","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3"
-      ,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
-      ,"RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94]_, 9.3, 9.2","RHEL 9.3, 9.2"
-      ,"RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
-      ,"SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
-      ,,CentOS 7.9,CentOS 7.9
-      ,"Oracle Linux 8.9 [#oracle89]_","Oracle Linux 8.9 [#oracle89]_",""
-      ,".. _architecture-support-compatibility-matrix:",,
+      :ref:`Operating Systems & kernels <OS-kernel-versions>`,Ubuntu 24.04,Ubuntu 24.04,
+      ,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.4, 22.04.3"
+      ,,,"Ubuntu 20.04.6, 20.04.5"
+      ,"RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94]_, 9.3, 9.2"
+      ,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8"
+      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4"
+      ,,,CentOS 7.9
+      ,Oracle Linux 8.9 [#oracle89]_,Oracle Linux 8.9 [#oracle89]_,
+      ,.. _architecture-support-compatibility-matrix:,,
       :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3
       ,CDNA2,CDNA2,CDNA2
       ,CDNA,CDNA,CDNA
       ,RDNA3,RDNA3,RDNA3
       ,RDNA2,RDNA2,RDNA2
-      ,".. _gpu-support-compatibility-matrix:",,
+      ,.. _gpu-support-compatibility-matrix:,,
       :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100
       ,gfx1030,gfx1030,gfx1030
-      ,gfx942 [#mi300_620]_, gfx942 [#mi300_612]_, gfx942 [#mi300_600]_
+      ,gfx942 [#mi300_621]_,gfx942 [#mi300_620]_, gfx942 [#mi300_610]_
       ,gfx90a,gfx90a,gfx90a
       ,gfx908,gfx908,gfx908
       ,,,
-      FRAMEWORK SUPPORT,".. _framework-support-compatibility-matrix:",,
-      :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.15, 2.14, 2.13","2.14, 2.13, 2.12"
+      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix:,,
+      :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13"
+      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15, 2.14, 2.13"
       :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26
-      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.14.1
+      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3
       ,,,
-      THIRD PARTY COMMS,".. _thirdpartycomms-support-compatibility-matrix:",,
-      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.2.0
-      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.14.1,>=1.14.1
+      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix:,,
+      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0
+      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.14.1
       ,,,
-      THIRD PARTY ALGORITHM,".. _thirdpartyalgorithm-support-compatibility-matrix:",,
-      Thrust,2.2.0,2.1.0,2.0.1
-      CUB,2.2.0,2.1.0,2.0.1
+      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix:,,
+      Thrust,2.2.0,2.2.0,2.1.0
+      CUB,2.2.0,2.2.0,2.1.0
       ,,,
-      ML & COMPUTER VISION,".. _mllibs-support-compatibility-matrix:",,
+      KFD & USER SPACE [#kfd_support]_,.. _kfd-userspace-support-compatibility-matrix:,,
+      Tested user space versions,"6.1.x, 6.0.x","6.1.x, 6.0.x","6.2.x, 6.0.x, 5.7.x"
+      ,,,
+      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix:,,
       :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0
-      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.9.0,2.8.0
-      :doc:`MIOpen <miopen:index>`,3.2.0,3.1.0,3.0.0
-      :doc:`MIVisionX <mivisionx:index>`,3.0.0,2.5.0,2.5.0
-      :doc:`RPP <rpp:index>`,1.8.0,1.5.0,1.4.0
-      :doc:`rocAL <rocal:index>`,1.0.0,1.0.0,1.0.0
-      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,N/A
-      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,N/A,N/A
+      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.10.0,2.9.0
+      :doc:`MIOpen <miopen:index>`,3.2.0,3.2.0,3.1.0
+      :doc:`MIVisionX <mivisionx:index>`,3.0.0,3.0.0,2.5.0
+      :doc:`rocAL <rocal:index>`,2.0.0,1.0.0,1.0.0
+      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.5.0
+      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,0.1.0,N/A
+      :doc:`RPP <rpp:index>`,1.8.0,1.8.0,1.5.0
       ,,,
-      COMMUNICATION,".. _commlibs-support-compatibility-matrix:",,
-      :doc:`RCCL <rccl:index>`,2.20.5,2.18.6,2.18.3
+      COMMUNICATION,.. _commlibs-support-compatibility-matrix:,,
+      :doc:`RCCL <rccl:index>`,2.20.5,2.20.5,2.18.6
       ,,,
-      MATH LIBS,".. _mathlibs-support-compatibility-matrix:",,
+      MATH LIBS,.. _mathlibs-support-compatibility-matrix:,,
       `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0
-      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.1.0,2.0.0
-      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.7.0,0.6.0
-      :doc:`hipFFT <hipfft:index>`,1.0.14,1.0.14,1.0.13
+      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.2.0,2.1.0
+      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.8.0,0.7.0
+      :doc:`hipFFT <hipfft:index>`,1.0.15,1.0.14,1.0.14
       :doc:`hipFORT <hipfort:index>`,0.4.0,0.4.0,0.4.0
-      :doc:`hipRAND <hiprand:index>`,2.11.0,2.10.16,2.10.16
-      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.1.1,2.0.0
-      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.0.1,3.0.0
-      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.0,0.1.0
-      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.1.1,3.0.3
-      :doc:`rocBLAS <rocblas:index>`,4.2.0,4.1.2,4.0.0
-      :doc:`rocFFT <rocfft:index>`,1.0.28,1.0.27,1.0.23
-      :doc:`rocRAND <rocrand:index>`,3.1.0,3.0.1,2.10.17
-      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.25.0,3.24.0
-      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.1.2,3.0.2
-      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.4.0,1.3.0
-      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.39.0
+      :doc:`hipRAND <hiprand:index>`,2.11.0,2.11.0,2.10.16
+      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.2.0,2.1.0
+      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.1.1,3.0.1
+      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.1,0.1.0
+      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.2.0,3.1.1
+      :doc:`rocBLAS <rocblas:index>`,4.2.1,4.2.0,4.1.0
+      :doc:`rocFFT <rocfft:index>`,1.0.29,1.0.28,1.0.26
+      :doc:`rocRAND <rocrand:index>`,3.1.0,3.1.0,3.0.1
+      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.26.0,3.25.0
+      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.2.0,3.1.2
+      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.5.0,1.4.0
+      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.40.0
       ,,,
-      PRIMITIVES,".. _primitivelibs-support-compatibility-matrix:",,
-      :doc:`hipCUB <hipcub:index>`,3.2.0,3.1.0,3.0.0
-      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.2.0,1.1.0
-      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.1.0,3.0.0
-      :doc:`rocThrust <rocthrust:index>`,3.0.1,3.0.1,3.0.0
+      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix:,,
+      :doc:`hipCUB <hipcub:index>`,3.2.0,3.2.0,3.1.0
+      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.3.0,1.2.0
+      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.2.0,3.1.0
+      :doc:`rocThrust <rocthrust:index>`,3.1.0,3.0.1,3.0.1
       ,,,
       SUPPORT LIBS,,,
-      `hipother <https://github.com/ROCm/hipother>`_,6.2.41133,6.1.40093,6.1.32830
-      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.0,6.1.2,6.0.0
-      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.1.4246,20240125.5.08,20231016.2.245
+      `hipother <https://github.com/ROCm/hipother>`_,6.2.41134,6.2.41133,6.1.40091
+      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.1,6.2.0,6.1.0
+      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.4.05,20240607.1.4246,20240125.3.30
       ,,,
-      SYSTEM MGMT TOOLS,".. _tools-support-compatibility-matrix:",,
-      :doc:`AMD SMI <amdsmi:index>`,24.6.2,24.5.1,23.4.2
-      :doc:`ROCm Data Center Tool <rdc:index>`,1.0.0,0.3.0,0.3.0
+      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix:,,
+      :doc:`AMD SMI <amdsmi:index>`,24.6.3,24.6.2,24.4.1
+      :doc:`ROCm Data Center Tool <rdc:index>`,1.0.0,1.0.0,0.3.0
       :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0
-      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.2.0,6.0.0
-      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.0,rocm-6.1.2,rocm-6.0.0
+      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.3.0,7.0.0
+      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.1,rocm-6.2.0,rocm-6.1.0
       ,,,
       PERFORMANCE TOOLS,,,
-      :doc:`Omniperf <omniperf:index>`,2.0.1,N/A,N/A
-      :doc:`Omnitrace <omnitrace:index>`,1.11.2,N/A,N/A
+      :doc:`Omniperf <omniperf:index>`,2.0.1,2.0.1,N/A
+      :doc:`Omnitrace <omnitrace:index>`,1.11.2,1.11.2,N/A
       :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0
-      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60200,2.0.60102,2.0.60000
-      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,N/A,N/A
-      :doc:`ROCTracer <roctracer:index>`,4.1.60200,4.1.60102,4.1.60000
+      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60201,2.0.60200,2.0.60100
+      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,0.4.0,N/A
+      :doc:`ROCTracer <roctracer:index>`,4.1.60201,4.1.60200,4.1.60100
       ,,,
       DEVELOPMENT TOOLS,,,
-      :doc:`HIPIFY <hipify:index>`,18.0.0.24232,17.0.0.24193,17.0.0.23483
-      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.12.0,0.11.0
-      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.71.0,0.71.0
-      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.1.0,13.2.0
-      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.3.0,N/A
+      :doc:`HIPIFY <hipify:index>`,18.0.0.24332,18.0.0.24232,17.0.0.24103
+      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.13.0,0.12.0
+      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.76.0,0.71.0
+      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.2.0,14.1.0
+      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.3.0
       :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3
       ,,,
-      COMPILERS,".. _compilers-support-compatibility-matrix:",,
-      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,0.5.0,0.5.0
-      :doc:`hipCC <hipcc:index>`,1.1.1,1.0.0,1.0.0
-      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24232,17.0.0.24193,17.0.0.23483
-      :doc:`llvm-project <llvm-project:index>`,18.0.0.24232,17.0.0.24193,17.0.0.23483
-      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24232,17.0.0.24193,17.0.0.23483
+      COMPILERS,.. _compilers-support-compatibility-matrix:,,
+      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,0.5.0
+      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.0.0
+      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24332,18.0.0.24232,17.0.0.24103
+      :doc:`llvm-project <llvm-project:index>`,18.0.0.24332,18.0.0.24232,17.0.0.24103
+      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24332,18.0.0.24232,17.0.0.24103
       ,,,
-      RUNTIMES,".. _runtime-support-compatibility-matrix:",,
-      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41133,6.1.40093,6.1.32830
-      :doc:`HIP <hip:index>`,6.2.41133,6.1.40093,6.1.32830
+      RUNTIMES,.. _runtime-support-compatibility-matrix:,,
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41134,6.2.41133,6.1.40091
+      :doc:`HIP <hip:index>`,6.2.41134,6.2.41133,6.1.40091
       `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0
-      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.13.0,1.13.0,1.12.0
+      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.14.0,1.13.0,1.13.0
 
 
 .. rubric:: Footnotes
 
-.. [#Ubuntu220405] Preview support of Ubuntu 22.04.5 only
 .. [#red-hat94] RHEL 9.4 is supported only on AMD Instinct MI300A.
 .. [#oracle89] Oracle Linux is supported only on AMD Instinct MI300X.
+.. [#mi300_621] **For ROCm 6.2.1** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
 .. [#mi300_620] **For ROCm 6.2.0** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
-.. [#mi300_612] **For ROCm 6.1.2** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4 and Oracle Linux.
-.. [#mi300_600] **For ROCm 6.0.0** - MI300A (gfx942) is supported on Ubuntu 22.04.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.3.
+.. [#mi300_610] **For ROCm 6.1.2** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4 and Oracle Linux.
+.. [#kfd_support] ROCm provides forward and backward compatibility between the Kernel Fusion Driver (KFD) and its user space software for +/- 2 releases. These are the compatibility combinations that are currently supported.
+
+
+.. _OS-kernel-versions:
+
+Operating systems and kernel versions
+*************************************
+
+Use this look up table to confirm which operating system and kernel versions are supported with ROCm.
+
+.. csv-table:: 
+   :header: "OS", "Version", "Kernel"
+   :widths: 40, 20, 40
+   :stub-columns: 1
+
+   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 24.04, "6.8 GA"
+   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 22.04.05, "5.15 GA, 6.8 HWE"
+   , 22.04.04, "5.15 GA, 6.5 HWE"
+   , 22.04.03, "5.15 GA, 6.2 HWE"
+   , 22.04.02, "5.15 GA, 5.19 HWE"
+   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 20.04.06, "5.15 HWE"
+   , 20.04.05, "5.15 HWE"
+   ,,
+   `Red Hat Enterprise Linux (RHEL) <https://access.redhat.com/articles/3078#RHEL9>`_, 9.4, 5.14.0
+   ,9.3, 5.14.0
+   ,9.2, 5.14.0
+   ,,
+   `Red Hat Enterprise Linux (RHEL) <https://access.redhat.com/articles/3078#RHEL8>`_, 8.10, 4.18.0
+   ,8.9, 4.18.0
+   ,8.8, 4.18.0
+   ,,
+   `CentOS <https://access.redhat.com/articles/3078#RHEL7>`_, 7.9, 3.10
+   ,,
+   `SUSE Linux Enterprise Server (SLES) <https://www.suse.com/support/kb/doc/?id=000019587#SLE15SP4>`_, 15 SP6, 6.4.0
+   ,15 SP5, 5.14.21
+   ,15 SP4, 5.14.21
+   ,,
+   `Oracle Linux <https://blogs.oracle.com/scoter/post/oracle-linux-and-unbreakable-enterprise-kernel-uek-releases>`_, 8.9, 5.15.0
+ 
 
 ..
    Footnotes and ref anchors in below historical tables should be appended with "-past-60", to differentiate from the 
@@ -160,20 +205,20 @@ Expand for full historical view of:
    You can `download the entire .csv <../downloads/compatibility-matrix-historical-6.0.csv>`_ for offline reference.
 
    .. csv-table::
-      :file: ../data/reference/compatibility-matrix-historical-6.0.csv
-      :widths: 20,10,10,10,10,10,10
+      :file: compatibility-matrix-historical-6.0.csv
+      :widths: 20,10,10,10,10,10,10,10
       :header-rows: 1
       :stub-columns: 1
    
    .. rubric:: Footnotes
 
-   .. [#Ubuntu220405-past-60] Preview support of Ubuntu 22.04.5 only
    .. [#red-hat94-past-60] RHEL 9.4 is supported only on AMD Instinct MI300A.
    .. [#oracle89-past-60] Oracle Linux is supported only on AMD Instinct MI300X.
+   .. [#mi300_621-past-60] **For ROCm 6.2.1** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
    .. [#mi300_620-past-60] **For ROCm 6.2.0** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
    .. [#mi300_612-past-60] **For ROCm 6.1.2** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4 and Oracle Linux.
    .. [#mi300_611-past-60] **For ROCm 6.1.1** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4 and Oracle Linux.
    .. [#mi300_610-past-60] **For ROCm 6.1.0** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4.
    .. [#mi300_602-past-60] **For ROCm 6.0.2** - MI300A (gfx942) is supported on Ubuntu 22.04.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.3.
    .. [#mi300_600-past-60] **For ROCm 6.0.0** - MI300A (gfx942) is supported on Ubuntu 22.04.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.3.
-
+   .. [#kfd_support-past-60] ROCm provides forward and backward compatibility between the Kernel Fusion Driver (KFD) and its user space software for +/- 2 releases. These are the compatibility combinations that are currently supported.

docs/conf.py

@@ -10,7 +10,7 @@ import shutil
 shutil.copy2("../RELEASE.md", "./about/release-notes.md")
 
 os.system("mkdir -p ../_readthedocs/html/downloads")
-os.system("cp data/reference/compatibility-matrix-historical-6.0.csv ../_readthedocs/html/downloads/compatibility-matrix-historical-6.0.csv")
+os.system("cp compatibility/compatibility-matrix-historical-6.0.csv ../_readthedocs/html/downloads/compatibility-matrix-historical-6.0.csv")
 
 latex_engine = "xelatex"
 latex_elements = {
@@ -30,16 +30,16 @@ if os.environ.get("READTHEDOCS", "") == "True":
 project = "ROCm Documentation"
 author = "Advanced Micro Devices, Inc."
 copyright = "Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved."
-version = "6.2.0"
-release = "6.2.0"
+version = "6.2.1"
+release = "6.2.1"
 setting_all_article_info = True
 all_article_info_os = ["linux", "windows"]
 all_article_info_author = ""
 
 # pages with specific settings
 article_pages = [
-    {"file": "about/release-notes", "os": ["linux", "windows"], "date": "2024-08-02"},
-    {"file": "about/changelog", "os": ["linux", "windows"], "date": "2024-08-02"},
+    {"file": "about/release-notes", "os": ["linux", "windows"], "date": "2024-09-20"},
+    {"file": "about/changelog", "os": ["linux", "windows"], "date": "2024-09-20"},
     {"file": "how-to/deep-learning-rocm", "os": ["linux"]},
     {"file": "how-to/rocm-for-ai/index", "os": ["linux"]},
     {"file": "how-to/rocm-for-ai/install", "os": ["linux"]},

docs/data/reference/compatibility-matrix-historical-6.0.csv

@@ -1,114 +0,0 @@
-ROCm Version,6.2.0, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
-      :doc:`Operating Systems <rocm-install-on-linux:reference/system-requirements>`,Ubuntu 24.04,,,,,
-      ,"Ubuntu 22.04.5 [#Ubuntu220405-past-60]_, 22.04.4","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3"
-      ,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
-      ,"RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
-      ,"RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
-      ,"SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
-      ,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
-      ,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,,,
-      ,".. _architecture-support-compatibility-matrix-past-60:",,,,,
-      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
-      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
-      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
-      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
-      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
-      ,".. _gpu-support-compatibility-matrix-past-60:",,,,,
-      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
-      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
-      ,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
-      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
-      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
-      ,,,,,,
-      FRAMEWORK SUPPORT,".. _framework-support-compatibility-matrix-past-60:",,,,,
-      :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.15, 2.14, 2.13","2.15, 2.14, 2.13","2.15, 2.14, 2.13","2.14, 2.13, 2.12","2.14, 2.13, 2.12"
-      :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
-      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
-      ,,,,,,
-      THIRD PARTY COMMS,".. _thirdpartycomms-support-compatibility-matrix-past-60:",,,,,
-      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
-      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
-      ,,,,,,
-      THIRD PARTY ALGORITHM,".. _thirdpartyalgorithm-support-compatibility-matrix-past-60:",,,,,
-      Thrust,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-      CUB,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-      ,,,,,,
-      ML & COMPUTER VISION,".. _mllibs-support-compatibility-matrix-past-60:",,,,,
-      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
-      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
-      :doc:`MIOpen <miopen:index>`,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`MIVisionX <mivisionx:index>`,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
-      :doc:`RPP <rpp:index>`,1.8.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
-      :doc:`rocAL <rocal:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
-      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,N/A,N/A,N/A,N/A,N/A
-      ,,,,,,
-      COMMUNICATION,".. _commlibs-support-compatibility-matrix-past-60:",,,,,
-      :doc:`RCCL <rccl:index>`,2.20.5,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
-      ,,,,,,
-      MATH LIBS,".. _mathlibs-support-compatibility-matrix-past-60:",,,,,
-      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
-      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
-      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
-      :doc:`hipFFT <hipfft:index>`,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
-      :doc:`hipFORT <hipfort:index>`,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
-      :doc:`hipRAND <hiprand:index>`,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
-      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
-      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
-      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
-      :doc:`rocBLAS <rocblas:index>`,4.2.0,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
-      :doc:`rocFFT <rocfft:index>`,1.0.28,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
-      :doc:`rocRAND <rocrand:index>`,3.1.0,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
-      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
-      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
-      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
-      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
-      ,,,,,,
-      PRIMITIVES,".. _primitivelibs-support-compatibility-matrix-past-60:",,,,,
-      :doc:`hipCUB <hipcub:index>`,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
-      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`rocThrust <rocthrust:index>`,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      ,,,,,,
-      SUPPORT LIBS,,,,,,
-      `hipother <https://github.com/ROCm/hipother>`_,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.0,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
-      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
-      ,,,,,,
-      SYSTEM MGMT TOOLS,".. _tools-support-compatibility-matrix-past-60:",,,,,
-      :doc:`AMD SMI <amdsmi:index>`,24.6.2,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
-      :doc:`ROCm Data Center Tool <rdc:index>`,1.0.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
-      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
-      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.0,rocm-6.1.2,rocm-6.1.1,rocm-6.1.0,rocm-6.0.2,rocm-6.0.0
-      ,,,,,,
-      PERFORMANCE TOOLS,,,,,,
-      :doc:`Omniperf <omniperf:index>`,2.0.1,N/A,N/A,N/A,N/A,N/A
-      :doc:`Omnitrace <omnitrace:index>`,1.11.2,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
-      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60200,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
-      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCTracer <roctracer:index>`,4.1.60200,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
-      ,,,,,,
-      DEVELOPMENT TOOLS,,,,,,
-      :doc:`HIPIFY <hipify:index>`,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
-      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
-      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
-      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.3.0,0.3.0,0.3.0,N/A,N/A
-      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
-      ,,,,,,
-      COMPILERS,".. _compilers-support-compatibility-matrix-past-60:",,,,,
-      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
-      :doc:`hipCC <hipcc:index>`,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      :doc:`llvm-project <llvm-project:index>`,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      ,,,,,,
-      RUNTIMES,".. _runtime-support-compatibility-matrix-past-60:",,,,,
-      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      :doc:`HIP <hip:index>`,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
-      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0

docs/how-to/llm-fine-tuning-optimization/llm-inference-frameworks.rst

@@ -137,6 +137,12 @@ Installing vLLM
 
 Refer to :ref:`mi300x-vllm-optimization` for performance optimization tips.
 
+ROCm provides a prebuilt optimized Docker image for validating the performance of LLM inference with vLLM 
+on the MI300X accelerator. The Docker image includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 .. _fine-tuning-llms-tgi:
 
 Hugging Face TGI

docs/how-to/llm-fine-tuning-optimization/model-acceleration-libraries.rst

@@ -251,3 +251,287 @@ page describes the options.
 
 Learn more about optimizing kernels with TunableOp in
 :ref:`Optimizing Triton kernels <mi300x-tunableop>`.
+
+
+FBGEMM and FBGEMM_GPU
+=====================
+
+FBGEMM (Facebook General Matrix Multiplication) is a low-precision, high-performance CPU kernel library
+for matrix-matrix multiplications and convolutions. It is used for server-side inference
+and as a back end for PyTorch quantized operators. FBGEMM offers optimized on-CPU performance for reduced precision calculations,
+strong performance on native tensor formats, and the ability to generate
+high-performance shape- and size-specific kernels at runtime.
+
+FBGEMM_GPU collects several high-performance PyTorch GPU operator libraries  
+for use in training and inference. It provides efficient table-batched embedding functionality,
+data layout transformation, and quantization support.
+
+For more information about FBGEMM and FBGEMM_GPU, see the `PyTorch FBGEMM GitHub <https://github.com/pytorch/FBGEMM>`_
+and the `PyTorch FBGEMM documentation <https://pytorch.org/FBGEMM/>`_.
+The `Meta blog post about FBGEMM <https://engineering.fb.com/2018/11/07/ml-applications/fbgemm/>`_ provides
+additional background about the library.
+
+Installing FBGEMM_GPU
+----------------------
+
+Installing FBGEMM_GPU consists of the following steps:
+
+*  Set up an isolated Miniconda environment
+*  Install ROCm using Docker or the :doc:`package manager <rocm-install-on-linux:install/native-install/index>`
+*  Install the nightly `PyTorch <https://pytorch.org/>`_ build
+*  Complete the pre-build and build tasks
+  
+.. note::
+
+   FBGEMM_GPU doesn't require the installation of FBGEMM. To optionally install
+   FBGEMM, see the `FBGEMM install instructions <https://pytorch.org/FBGEMM/fbgemm-development/BuildInstructions.html>`_.
+
+Set up the Miniconda environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To install Miniconda, use the following commands.
+
+#. Install a `Miniconda environment <https://docs.anaconda.com/miniconda/>`_ for reproducible builds.
+   All subsequent commands run inside this environment.
+
+   .. code-block:: shell
+
+      export PLATFORM_NAME="$(uname -s)-$(uname -m)"
+
+      # Set the Miniconda prefix directory
+      miniconda_prefix=$HOME/miniconda
+
+      # Download the Miniconda installer
+      wget -q "https://repo.anaconda.com/miniconda/Miniconda3-latest-${PLATFORM_NAME}.sh" -O miniconda.sh
+
+      # Run the installer
+      bash miniconda.sh -b -p "$miniconda_prefix" -u
+
+      # Load the shortcuts
+      . ~/.bashrc
+
+      # Run updates
+      conda update -n base -c defaults -y conda
+
+#. Create a Miniconda environment with Python 3.12:
+
+   .. code-block:: shell
+
+      env_name=<ENV NAME>
+      python_version=3.12
+
+      # Create the environment
+      conda create -y --name ${env_name} python="${python_version}"
+
+      # Upgrade PIP and pyOpenSSL package
+      conda run -n ${env_name} pip install --upgrade pip
+      conda run -n ${env_name} python -m pip install pyOpenSSL>22.1.0
+
+#. Install additional build tools:
+
+   .. code-block:: shell
+
+      conda install -n ${env_name} -y \
+         click \
+         cmake \
+         hypothesis \
+         jinja2 \
+         make \
+         ncurses \
+         ninja \
+         numpy \
+         scikit-build \
+         wheel
+
+Install the ROCm components
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+FBGEMM_GPU can run in a ROCm Docker container or in conjunction with the full ROCm installation.
+The Docker method is recommended because it requires fewer steps and provides a stable environment.
+
+To run FBGEMM_GPU in the Docker container, pull the `Minimal Docker image for ROCm <https://hub.docker.com/r/rocm/rocm-terminal>`_.
+This image includes all preinstalled ROCm packages required to integrate FBGEMM. To pull
+and run the ROCm Docker image, use this command:
+
+.. code-block:: shell
+
+   # Run for ROCm 6.2.0
+   docker run -it --network=host --shm-size 16G --device=/dev/kfd --device=/dev/dri --group-add video \
+   --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --ipc=host rocm/rocm-terminal:6.2 /bin/bash
+
+.. note::
+
+   The `Full Docker image for ROCm <https://hub.docker.com/r/rocm/dev-ubuntu-20.04>`_, which includes all
+   ROCm packages, can also be used. However, it results in a very large container, so the minimal
+   Docker image is recommended.
+
+You can also install ROCm using the package manager. FBGEMM_GPU requires the installation of the full ROCm package.
+For more information, see :doc:`the ROCm installation guide <rocm-install-on-linux:install/detailed-install>`.
+The ROCm package also requires the :doc:`MIOpen <miopen:index>` component as a dependency. 
+To install MIOpen, use the ``apt install`` command.
+
+.. code-block:: shell
+
+   apt install hipify-clang miopen-hip miopen-hip-dev
+
+Install PyTorch
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Install `PyTorch <https://pytorch.org/>`_ using ``pip`` for the most reliable and consistent results.
+
+#. Install the nightly PyTorch build using ``pip``.
+
+   .. code-block:: shell
+
+      # Install the latest nightly, ROCm variant
+      conda run -n ${env_name} pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/rocm6.2/
+
+#. Ensure PyTorch loads correctly. Verify the version and variant of the installation using an ``import`` test.
+
+   .. code-block:: shell
+
+      # Ensure that the package loads properly
+      conda run -n ${env_name} python -c "import torch.distributed"
+
+      # Verify the version and variant of the installation
+      conda run -n ${env_name} python -c "import torch; print(torch.__version__)"
+
+Perform the prebuild and build
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+#. Clone the FBGEMM repository and the relevant submodules. Use ``pip`` to install the 
+   components in ``requirements.txt``. Run the following commands inside the Miniconda environment.
+
+   .. code-block:: shell
+
+      # Select a version tag
+      FBGEMM_VERSION=v0.8.0
+
+      # Clone the repo along with its submodules
+      git clone https://github.com/pytorch/FBGEMM.git --branch=v0.8.0 --recursive fbgemm_${FBGEMM_VERSION}
+
+      # Install additional required packages for building and testing
+      cd fbgemm_${FBGEMM_VERSION}/fbgemm_gpu
+      pip install requirements.txt
+
+#. Clear the build cache to remove stale build information.
+
+   .. code-block:: shell
+
+      # !! Run in fbgemm_gpu/ directory inside the Conda environment !!
+
+      python setup.py clean
+
+#. Set the wheel build variables, including the package name, Python version tag, and Python platform name.
+
+   .. code-block:: shell
+
+      # Set the package name depending on the build variant
+      export package_name=fbgemm_gpu_rocm
+
+      # Set the Python version tag.  It should follow the convention `py<major><minor>`,
+      # for example, Python 3.12 --> py312
+      export python_tag=py312
+
+      # Determine the processor architecture
+      export ARCH=$(uname -m)
+
+      # Set the Python platform name for the Linux case
+      export python_plat_name="manylinux2014_${ARCH}"
+
+#. Build FBGEMM_GPU for the ROCm platform. Set ``ROCM_PATH`` to the path to your ROCm installation. 
+   Run these commands from the ``fbgemm_gpu/`` directory inside the Miniconda environment.
+
+   .. code-block:: shell
+
+      # !! Run in the fbgemm_gpu/ directory inside the Conda environment !!
+
+      export ROCM_PATH=</path/to/rocm>
+
+      # Build for the target architecture of the ROCm device installed on the machine (for example, 'gfx942;gfx90a')
+      # See :doc:`The Linux system requirements <../../reference/system-requirements>` for a list of supported GPUs.
+      export PYTORCH_ROCM_ARCH=$(${ROCM_PATH}/bin/rocminfo | grep -o -m 1 'gfx.*')
+
+      # Build the wheel artifact only
+      python setup.py bdist_wheel \
+         --package_variant=rocm \
+         --python-tag="${python_tag}" \
+         --plat-name="${python_plat_name}" \
+         -DHIP_ROOT_DIR="${ROCM_PATH}" \
+         -DCMAKE_C_FLAGS="-DTORCH_USE_HIP_DSA" \
+         -DCMAKE_CXX_FLAGS="-DTORCH_USE_HIP_DSA"
+
+      # Build and install the library into the Conda environment
+      python setup.py install \
+         --package_variant=rocm \
+         -DHIP_ROOT_DIR="${ROCM_PATH}" \
+         -DCMAKE_C_FLAGS="-DTORCH_USE_HIP_DSA" \
+         -DCMAKE_CXX_FLAGS="-DTORCH_USE_HIP_DSA"  
+
+Post-build validation
+----------------------
+
+After building FBGEMM_GPU, run some verification checks to ensure the build is correct. Continue
+to run all commands inside the ``fbgemm_gpu/`` directory inside the Miniconda environment.
+
+#. The build process generates many build artifacts and C++ templates, so
+   it is important to confirm no undefined symbols remain.
+
+   .. code-block:: shell
+
+      # !! Run in fbgemm_gpu/ directory inside the Conda environment !!
+
+      # Locate the built .SO file
+      fbgemm_gpu_lib_path=$(find . -name fbgemm_gpu_py.so)
+
+      # Check that the undefined symbols don't include fbgemm_gpu-defined functions
+      nm -gDCu "${fbgemm_gpu_lib_path}" | sort
+
+#. Verify the referenced version number of ``GLIBCXX`` and the presence of certain function symbols:
+
+   .. code-block:: shell
+
+      # !! Run in fbgemm_gpu/ directory inside the Conda environment !!
+
+      # Locate the built .SO file
+      fbgemm_gpu_lib_path=$(find . -name fbgemm_gpu_py.so)
+
+      # Note the versions of GLIBCXX referenced by the .SO
+      # The libstdc++.so.6 available on the install target must support these versions
+      objdump -TC "${fbgemm_gpu_lib_path}" | grep GLIBCXX | sed 's/.*GLIBCXX_\([.0-9]*\).*/GLIBCXX_\1/g' | sort -Vu | cat
+
+      # Test for the existence of a given function symbol in the .SO
+      nm -gDC "${fbgemm_gpu_lib_path}" | grep " fbgemm_gpu::merge_pooled_embeddings("
+      nm -gDC "${fbgemm_gpu_lib_path}" | grep " fbgemm_gpu::jagged_2d_to_dense("
+
+Testing FBGEMM
+----------------------
+
+FBGEMM includes tests and benchmarks to validate performance. To run these tests,
+you must use ROCm 5.7 or a more recent version on the host and container. To run FBGEMM tests,
+follow these instructions:
+
+.. code-block:: shell
+
+   # !! Run inside the Conda environment !!
+
+   # From the /fbgemm_gpu/ directory
+   cd test
+
+   export FBGEMM_TEST_WITH_ROCM=1
+   # Enable for debugging failed kernel executions
+   export HIP_LAUNCH_BLOCKING=1
+
+   # Run the test
+   python -m pytest -v -rsx -s -W ignore::pytest.PytestCollectionWarning split_table_batched_embeddings_test.py
+
+To run the FBGEMM_GPU ``uvm`` test, use these commands. These tests only support the AMD MI210 and 
+more recent accelerators. 
+
+.. code-block:: shell
+
+   # Run this inside the Conda environment from the /fbgemm_gpu/ directory
+   export HSA_XNACK=1
+   cd test
+
+   python -m pytest -v -rsx -s -W ignore::pytest.PytestCollectionWarning ./uvm/uvm_test.py

docs/how-to/llm-fine-tuning-optimization/model-quantization.rst

@@ -189,6 +189,9 @@ Installing bitsandbytes
       # Use -DBNB_ROCM_ARCH to specify target GPU arch
       cmake -DBNB_ROCM_ARCH="gfx942" -DCOMPUTE_BACKEND=hip -S .
 
+      # Compile the project
+      make
+
       # Install 
       python setup.py install
 
@@ -219,12 +222,11 @@ To get started with bitsandbytes primitives, use the following code as reference
 Using bitsandbytes with Hugging Face Transformers
 -------------------------------------------------
 
-To load a Transformers model in 4-bit, set ``load_int_4bt=true`` in ``BitsAndBytesConfig``.
+To load a Transformers model in 4-bit, set ``load_in_4bit=true`` in ``BitsAndBytesConfig``.
 
 .. code-block:: python
 
-   from transformers import AutoModelForCausalLM
-   from bitsandbytes import BitsAndBytesConfig
+   from transformers import AutoModelForCausalLM, BitsAndBytesConfig
    
    base_model_name = "NousResearch/Llama-2-7b-hf"
    quantization_config = BitsAndBytesConfig(load_in_4bit=True)
@@ -240,8 +242,7 @@ To load a model in 8-bit for inference, use the ``load_in_8bit`` option.
 
 .. code-block:: python
 
-   from transformers import AutoModelForCausalLM, AutoTokenizer
-   from bitsandbytes import BitsAndBytesConfig
+   from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
    
    base_model_name = "NousResearch/Llama-2-7b-hf"
    

docs/how-to/rocm-for-ai/deploy-your-model.rst

@@ -41,6 +41,15 @@ vLLM walkthrough
 Refer to this developer blog for guidance on serving with vLLM `Inferencing and serving with vLLM on AMD GPUs — ROCm
 Blogs <https://rocm.blogs.amd.com/artificial-intelligence/vllm/README.html>`_
 
+Validating vLLM performance
+---------------------------
+
+ROCm provides a prebuilt optimized Docker image for validating the performance of LLM inference with vLLM 
+on the MI300X accelerator. The Docker image includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 .. _rocm-for-ai-serve-hugging-face-tgi:
 
 Serving using Hugging Face TGI

docs/how-to/rocm-for-ai/install.rst

@@ -22,25 +22,25 @@ If you’re new to ROCm, refer to the :doc:`ROCm quick start install guide for L
 <rocm-install-on-linux:install/quick-start>`.
 
 If you’re using a Radeon GPU for graphics-accelerated applications, refer to the
-:doc:`Radeon installation instructions <radeon:docs/install/install-radeon>`.
+`Radeon installation instructions <https://rocm.docs.amd.com/projects/radeon/en/docs-6.1.3/docs/install/native_linux/install-radeon.html>`_.
 
-ROCm supports two methods for installation. There is no difference in the final ROCm installation between these two
-methods. You can also opt for :ref:`single-version or multi-version installation
-<rocm-install-on-linux:installation-types>`.
+ROCm supports multiple :doc:`installation methods <rocm-install-on-linux:install/install-overview>`:
 
-*  :doc:`Using your Linux distribution's package manager <rocm-install-on-linux:how-to/native-install/index>`
+* :doc:`Using your Linux distribution's package manager <rocm-install-on-linux:install/native-install/index>`
 
-*  :doc:`Using the AMDGPU installer <rocm-install-on-linux:how-to/amdgpu-install>`
+* :doc:`Using the AMDGPU installer <rocm-install-on-linux:install/amdgpu-install>`
+
+* :ref:`Multi-version installation <rocm-install-on-linux:installation-types>`.
 
 .. grid:: 1
 
    .. grid-item-card:: Post-install
 
-      Follow the :doc:`post-installation instructions <rocm-install-on-linux:how-to/native-install/post-install>` to
+      Follow the :doc:`post-installation instructions <rocm-install-on-linux:install/post-install>` to
       configure your system linker, PATH, and verify the installation.
 
       If you encounter any issues during installation, refer to the
-      :doc:`Installation troubleshooting <rocm-install-on-linux:how-to/native-install/install-faq>` guide.
+      :doc:`Installation troubleshooting <rocm-install-on-linux:reference/install-faq>` guide.
 
 Machine learning frameworks
 ===========================

docs/how-to/system-optimization/mi100.md

@@ -365,9 +365,9 @@ installed.
 ## System management
 
 For a complete guide on how to install/manage/uninstall ROCm on Linux, refer to
-{doc}`Quick-start (Linux)<rocm-install-on-linux:tutorial/quick-start>`. To verify that the installation was
+{doc}`Quick-start (Linux)<rocm-install-on-linux:install/quick-start>`. To verify that the installation was
 successful, refer to the
-{doc}`post-install instructions<rocm-install-on-linux:how-to/native-install/post-install>` and
+{doc}`post-install instructions<rocm-install-on-linux:install/post-install>` and
 [system tools](../../reference/rocm-tools.md). Should verification
 fail, consult the [System Debugging Guide](../system-debugging.md).
 

docs/how-to/system-optimization/mi200.md

@@ -350,9 +350,9 @@ installed.
 ## System management
 
 For a complete guide on how to install/manage/uninstall ROCm on Linux, refer to
-{doc}`Quick-start (Linux)<rocm-install-on-linux:tutorial/quick-start>`. For verifying that the
+{doc}`Quick-start (Linux)<rocm-install-on-linux:install/quick-start>`. For verifying that the
 installation was successful, refer to the
-{doc}`post-install instructions<rocm-install-on-linux:how-to/native-install/post-install>` and
+{doc}`post-install instructions<rocm-install-on-linux:install/post-install>` and
 [system tools](../../reference/rocm-tools.md). Should verification
 fail, consult the [System Debugging Guide](../system-debugging.md).
 

docs/how-to/system-optimization/mi300a.rst

@@ -136,7 +136,7 @@ This section describes performance-based settings.
 
   .. code-block:: shell
 
-     cat /sys/module/ttm/parameters/pages_limit 
+     cat /sys/module/amdttm/parameters/pages_limit 
   
   To set the amount of allocatable memory to all available memory on all four APU devices, run these commands:
 
@@ -145,20 +145,31 @@ This section describes performance-based settings.
      sudo modprobe amdttm pages_limit=134217728 page_pool_size=134217728
      sudo modprobe amdgpu
 
-  These settings can also be hardcoded in the ``/etc/modprobe.d/amdttm.conf`` file. To use this method, 
+  These settings can also be hardcoded in the ``/etc/modprobe.d/amdttm.conf`` file or specified as boot
+  parameters.
+  
+  To use the hardcoded method, 
   the filesystem must already be set up when the kernel driver is loaded.
-  Add the following lines to ``/etc/modprobe.d/amdttm.conf``:
+  To hardcode the settings, add the following lines to ``/etc/modprobe.d/amdttm.conf``:
 
   .. code-block:: shell
 
      options amdttm pages_limit=134217728
      options amdttm page_pool_size=134217728
 
+  If the filesystem is not already set up when the kernel driver is loaded, then the options
+  must be specified as boot parameters. To specify the settings
+  as boot parameters when loading the kernel, use this example as a guideline:
+
+  .. code-block:: shell
+
+     vmlinux-[...] amdttm.pages_limit=134217728 amdttm.page_pool_size=134217728 [...]
+
   To verify the new settings and confirm the change, use this command:
 
   .. code-block:: shell
 
-     cat /sys/module/ttm/parameters/pages_limit 
+     cat /sys/module/amdttm/parameters/pages_limit 
 
   .. note::
 
@@ -240,10 +251,13 @@ This section describes performance-based settings.
 
 * **Change affinity of ROCm helper threads**
   
-  This change prevents internal ROCm threads from having their CPU core affinity mask 
+  Changing the affinity prevents internal ROCm threads from having their CPU core affinity mask 
   set to all CPU cores available. With this setting, the threads inherit their parent's 
-  CPU core affinity mask. If you have any questions regarding this setting, 
-  contact your MI300A platform vendor. To enable this setting, enter the following command:
+  CPU core affinity mask. Before adjusting this setting, ensure you thoroughly understand 
+  your system topology and how the application, runtime environment, and batch system
+  set the thread-to-core affinity. If you have any questions regarding this setting, 
+  contact your MI300A platform vendor or the AMD support team. 
+  To enable this setting, enter the following command:
 
   .. code-block:: shell
 
@@ -258,9 +272,9 @@ System management
 ========================================
 
 For a complete guide on installing, managing, and uninstalling ROCm on Linux, see
-:doc:`Quick-start (Linux)<rocm-install-on-linux:tutorial/quick-start>`. To verify that the
+:doc:`Quick-start (Linux)<rocm-install-on-linux:install/quick-start>`. To verify that the
 installation was successful, see the
-:doc:`Post-installation instructions<rocm-install-on-linux:install/native-install/post-install>` and 
+:doc:`Post-installation instructions<rocm-install-on-linux:install/post-install>` and 
 :doc:`ROCm tools <../../reference/rocm-tools>` guides. If verification
 fails, consult the :doc:`System debugging guide <../system-debugging>`.
 

docs/how-to/system-optimization/mi300x.rst

@@ -534,7 +534,7 @@ optimizing user applications.
 For a complete guide on how to install, manage, or uninstall ROCm on Linux, refer to
 :doc:`rocm-install-on-linux:install/quick-start`. For verifying that the
 installation was successful, refer to the
-:doc:`rocm-install-on-linux:install/native-install/post-install`.
+:doc:`rocm-install-on-linux:install/post-install`.
 Should verification fail, consult :doc:`/how-to/system-debugging`.
 
 Hardware verification with ROCm

docs/how-to/tuning-guides/mi300x/workload.rst

@@ -150,6 +150,12 @@ the workload to validate improvements and ensure that the changes have had the
 desired effect. Continuous iteration helps refine the performance gains and
 address any new bottlenecks that may emerge.
 
+ROCm provides a prebuilt optimized Docker image that has everything required to implement
+the tips in this section. It includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 .. _mi300x-profiling-tools:
 
 Profiling tools
@@ -372,6 +378,12 @@ Refer to `vLLM documentation <https://docs.vllm.ai/en/latest/models/performance.
 for additional performance tips. :ref:`fine-tuning-llms-vllm` describes vLLM
 usage with ROCm.
 
+ROCm provides a prebuilt optimized Docker image for validating the performance of LLM inference with vLLM 
+on the MI300X accelerator. The Docker image includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
+format. For more information, see the guide to 
+`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
+on the ROCm GitHub repository.
+
 Maximize throughput
 -------------------
 

docs/index.md

@@ -23,57 +23,40 @@ Installation instructions are available from:
 ROCm documentation is organized into the following categories:
 
 ::::{grid} 1 2 2 2
+:gutter: 3
 :class-container: rocm-doc-grid
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ./data/banner-compatibility.jpg
-:img-alt: Compatibility information
-:padding: 2
+:::{grid-item-card} Compatibility
+:class-body: rocm-card-banner rocm-hue-2
 
 * [Compatibility matrix](./compatibility/compatibility-matrix.rst)
 * {doc}`Linux system requirements<rocm-install-on-linux:reference/system-requirements>`
 * {doc}`Windows system requirements<rocm-install-on-windows:reference/system-requirements>`
 * {doc}`Third-party support<rocm-install-on-linux:reference/3rd-party-support-matrix>`
-* {doc}`User/kernel space<rocm-install-on-linux:reference/user-kernel-space-compat-matrix>`
-* {doc}`Docker<rocm-install-on-linux:reference/docker-image-support-matrix>`
-* {doc}`OpenMP<llvm-project:conceptual/openmp>`
-* [Precision support](./compatibility/precision-support.rst)
-* {doc}`ROCm on Radeon GPUs<radeon:index>`
+* {doc}`User and kernel-space support matrix<rocm-install-on-linux:reference/user-kernel-space-compat-matrix>`
+* {doc}`Docker image support matrix<rocm-install-on-linux:reference/docker-image-support-matrix>`
+* {doc}`Use ROCm on Radeon GPUs<radeon:index>`
 :::
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ./data/banner-howto.jpg
-:img-alt: How-to documentation
-:padding: 2
+:::{grid-item-card} How to
+:class-body: rocm-card-banner rocm-hue-12
 
 * [Using ROCm for AI](./how-to/rocm-for-ai/index.rst)
 * [Using ROCm for HPC](./how-to/rocm-for-hpc/index.rst)
 * [Fine-tuning LLMs and inference optimization](./how-to/llm-fine-tuning-optimization/index.rst)
 * [System optimization](./how-to/system-optimization/index.rst)
-  * [AMD Instinct MI300X](./how-to/system-optimization/mi300x.rst)
-  * [AMD Instinct MI300A](./how-to/system-optimization/mi300a.rst)
-  * [AMD Instinct MI200](./how-to/system-optimization/mi200.md)
-  * [AMD Instinct MI100](./how-to/system-optimization/mi100.md)
-  * [AMD Instinct RDNA2](./how-to/system-optimization/w6000-v620.md)
 * [AMD Instinct MI300X tuning guides](./how-to/tuning-guides/mi300x/index.rst)
-  * [System tuning](./how-to/tuning-guides/mi300x/system.rst)
-  * [Workload tuning](./how-to/tuning-guides/mi300x/workload.rst)
 * [System debugging](./how-to/system-debugging.md)
-* [GPU-enabled MPI](./how-to/gpu-enabled-mpi.rst)
+* [Using MPI](./how-to/gpu-enabled-mpi.rst)
 * [Using advanced compiler features](./conceptual/compiler-topics.md)
 * [Setting the number of CUs](./how-to/setting-cus)  
-* [GitHub examples](https://github.com/amd/rocm-examples)
+* [ROCm examples](https://github.com/amd/rocm-examples)
 :::
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ./data/banner-conceptual.jpg
-:img-alt: Conceptual documentation
-:padding: 2
+:::{grid-item-card} Conceptual
+:class-body: rocm-card-banner rocm-hue-8
 
-* [GPU architecture](./conceptual/gpu-arch.md)
+* [GPU architecture overview](./conceptual/gpu-arch.md)
 * [GPU memory](./conceptual/gpu-memory.md)
 * [File structure (Linux FHS)](./conceptual/file-reorg.md)
 * [GPU isolation techniques](./conceptual/gpu-isolation.md)
@@ -84,21 +67,13 @@ ROCm documentation is organized into the following categories:
 :::
 
 <!-- markdownlint-disable MD051 -->
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ./data/banner-reference.jpg
-:img-alt: Reference documentation
-:padding: 2
+:::{grid-item-card} Reference
+:class-body: rocm-card-banner rocm-hue-6
 
-* [Libraries](./reference/api-libraries.md)
-  * [Artificial intelligence](#artificial-intelligence-apis)
-  * [C++ primitives](#cpp-primitives)
-  * [Communication](#communication-libraries)
-  * [Math](#math-apis)
-  * [Random number generators](#random-number-apis)
-  * [HIP runtime](#hip-runtime)
-* [ROCm tools and compilers](./reference/rocm-tools.md)
-* [GPU hardware specifications](./reference/gpu-arch-specs.rst)
+* [ROCm libraries](./reference/api-libraries.md)
+* [ROCm tools, compilers, and runtimes](./reference/rocm-tools.md)
+* [Accelerator and  GPU hardware specifications](./reference/gpu-arch-specs.rst)
+* [Precision support](./reference/precision-support.rst)
 :::
 <!-- markdownlint-enable MD051 -->
 

docs/reference/api-libraries.md

@@ -9,15 +9,13 @@
 # ROCm libraries
 
 ::::{grid} 1 2 2 2
+:gutter: 3
 :class-container: rocm-doc-grid
 
 (artificial-intelligence-apis)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-ai.jpg
-:img-alt: Artificial intelligence APIs
-:padding: 2
+:::{grid-item-card} Machine Learning and Computer Vision
+:class-body: rocm-card-banner rocm-hue-3
 
 * {doc}`Composable Kernel <composable_kernel:index>`
 * {doc}`MIGraphX <amdmigraphx:index>`
@@ -25,16 +23,14 @@
 * {doc}`MIVisionX <mivisionx:index>`
 * {doc}`rocAL <rocal:index>`
 * {doc}`rocDecode <rocdecode:index>`
+* {doc}`rocPyDecode <rocpydecode:index>`
 * {doc}`ROCm Performance Primitives (RPP) <rpp:index>`
 :::
 
 (cpp-primitives)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-cpp-primitives.jpg
-:img-alt: C++ primitives
-:padding: 2
+:::{grid-item-card} Primitives
+:class-body: rocm-card-banner rocm-hue-12
 
 * {doc}`hipCUB <hipcub:index>`
 * {doc}`hipTensor <hiptensor:index>`
@@ -44,40 +40,23 @@
 
 (communication-libraries)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-communication.jpg
-:img-alt: Communication APIs
-:padding: 2
+:::{grid-item-card} Communication
+:class-body: rocm-card-banner rocm-hue-7
 
 * {doc}`RCCL <rccl:index>`
 :::
 
-(hip-runtime)=
-
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-hip.jpg
-:img-alt: HIP APIs
-:padding: 2
-
-* {doc}`HIP runtime <hip:index>`
-* {doc}`HIPIFY <hipify:index>`
-:::
-
 (math-apis)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-math.jpg
-:img-alt: Math APIs
-:padding: 2
+:::{grid-item-card} Math
+:class-body: rocm-card-banner rocm-hue-6
 
 * [half](https://github.com/ROCm/half)
 * {doc}`hipBLAS <hipblas:index>` / {doc}`rocBLAS <rocblas:index>`
 * {doc}`hipBLASLt <hipblaslt:index>`
 * {doc}`hipFFT <hipfft:index>` / {doc}`rocFFT <rocfft:index>`
 * {doc}`hipfort <hipfort:index>`
+* {doc}`hipRAND <hiprand:index>` / {doc}`rocRAND <rocrand:index>`
 * {doc}`hipSOLVER <hipsolver:index>` / {doc}`rocSOLVER <rocsolver:index>`
 * {doc}`hipSPARSE <hipsparse:index>` / {doc}`rocSPARSE <rocsparse:index>`
 * {doc}`hipSPARSELt <hipsparselt:index>`
@@ -86,16 +65,4 @@
 * [Tensile](https://github.com/ROCm/Tensile)
 :::
 
-(random-number-apis)=
-
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-random-number.jpg
-:img-alt: Random number APIs
-:padding: 2
-
-* {doc}`hipRAND <hiprand:index>`
-* {doc}`rocRAND <rocrand:index>`
-:::
-
 ::::

docs/reference/precision-support.rst

@@ -410,7 +410,7 @@ description, refer to the corresponding library data type support page.
         - ❌/❌
         - ❌/❌
       *
-        - rocRAND (:doc:`details <rocrand:data-type-support>`)
+        - rocRAND (:doc:`details <rocrand:api-reference/data-type-support>`)
         - -/✅
         - -/✅
         - -/✅

docs/reference/rocm-tools.md

@@ -9,15 +9,13 @@
 # ROCm tools, compilers, and runtimes
 
 ::::{grid} 1 2 2 2
+:gutter: 3
 :class-container: rocm-doc-grid
 
 (system-tools)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-system.jpg
-:img-alt: System tools
-:padding: 2
+:::{grid-item-card} System Management
+:class-body: rocm-card-banner rocm-hue-1
 
 * {doc}`AMD SMI <amdsmi:index>`
 * {doc}`ROCm Data Center Tool <rdc:index>`
@@ -28,11 +26,8 @@
 
 (performance-tools)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-performance.jpg
-:img-alt: Performance tools
-:padding: 2
+:::{grid-item-card} Performance
+:class-body: rocm-card-banner rocm-hue-6
 
 * {doc}`Omniperf <omniperf:index>`
 * {doc}`Omnitrace <omnitrace:index>`
@@ -44,11 +39,8 @@
 
 (development-tools)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-development.jpg
-:img-alt: Development tools
-:padding: 2
+:::{grid-item-card} Development
+:class-body: rocm-card-banner rocm-hue-1
 
 * {doc}`ROCm CMake <rocmcmakebuildtools:index>`
 * {doc}`HIPIFY <hipify:index>`
@@ -59,11 +51,8 @@
 
 (compilers)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-compilers.jpg
-:img-alt: Compilers
-:padding: 2
+:::{grid-item-card} Compilers
+:class-body: rocm-card-banner rocm-hue-8
 
 * {doc}`ROCm Compilers <llvm-project:index>`
 * {doc}`HIPCC <hipcc:index>`
@@ -72,11 +61,8 @@
 
 (runtimes)=
 
-:::{grid-item-card}
-:class-card: sd-text-black
-:img-top: ../data/reference/banner-runtimes.jpg
-:img-alt: Runtimes
-:padding: 2
+:::{grid-item-card} Runtimes
+:class-body: rocm-card-banner rocm-hue-12
 
 * {doc}`AMD Common Language Runtime (CLR) <hip:understand/amd_clr>`
 * {doc}`HIP <hip:index>`

docs/release/versions.md

@@ -8,6 +8,7 @@
 
 | Version | Release date |
 | ------- | ------------ |
+| [6.2.1](https://rocm.docs.amd.com/en/docs-6.2.1/) | September 20, 2024 |
 | [6.2.0](https://rocm.docs.amd.com/en/docs-6.2.0/) | August 2, 2024 |
 | [6.1.2](https://rocm.docs.amd.com/en/docs-6.1.2/) | June 4, 2024 |
 | [6.1.1](https://rocm.docs.amd.com/en/docs-6.1.1/) | May 8, 2024 |

docs/sphinx/_toc.yml.in

@@ -100,13 +100,17 @@ subtrees:
   - file: compatibility/compatibility-matrix.rst
     title: Compatibility matrix
   - url: https://rocm.docs.amd.com/projects/install-on-linux/en/${branch}/reference/system-requirements.html
-    title: Linux
+    title: Linux system requirements
   - url: https://rocm.docs.amd.com/projects/install-on-windows/en/${branch}/reference/system-requirements.html
-    title: Windows
-  - file: compatibility/precision-support.rst
-    title: Precision support
+    title: Windows system requirements
   - url: https://rocm.docs.amd.com/projects/install-on-linux/en/${branch}/reference/3rd-party-support-matrix.html
-    title: Third-party
+    title: Third-party support
+  - url: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/user-kernel-space-compat-matrix.html
+    title: User and kernel-space support matrix
+  - url: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/docker-image-support-matrix.html
+    title: Docker image support matrix
+  - url: https://rocm.docs.amd.com/projects/radeon/en/latest/index.html
+    title: Use ROCm on Radeon GPUs
 
 - caption: Conceptual
   entries:
@@ -163,6 +167,8 @@ subtrees:
       title: ROCm tools, compilers, and runtimes
     - file: reference/gpu-arch-specs.rst
       title: Hardware specifications
+    - file: reference/precision-support.rst
+      title: Precision support
 
 - caption: Contribute
   entries:

docs/sphinx/requirements.in

@@ -1,2 +1,2 @@
-rocm-docs-core==1.7.1
+rocm-docs-core==1.8.1
 sphinx-reredirects

docs/sphinx/requirements.txt

@@ -26,7 +26,7 @@ charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via sphinx-external-toc
-cryptography==42.0.8
+cryptography==43.0.1
     # via pyjwt
 deprecated==1.2.14
     # via pygithub
@@ -92,7 +92,7 @@ requests==2.32.3
     # via
     #   pygithub
     #   sphinx
-rocm-docs-core==1.7.1
+rocm-docs-core==1.8.1
     # via -r requirements.in
 smmap==5.0.1
     # via gitdb

docs/temp/conceptual-index.md

@@ -1,54 +0,0 @@
-# Conceptual documentation
-
-:::::{grid} 1 1 2 2
-:gutter: 1
-
-:::{grid-item-card}
-**[Compiler nomenclature](./compiler-disambiguation.md)**
-
-ROCm ships multiple compilers of varying origins and purposes. This article
-disambiguates compiler naming used throughout the documentation.
-
-:::
-
-:::{grid-item-card}
-**[Using CMake](./cmake-packages.rst)**
-
-ROCm components ship with 1st party CMake support. This article details how that
-support works and how to use it.
-
-:::
-
-:::{grid-item-card}
-**[Linux folder structure reorganization](./file-reorg.md)**
-
-ROCm™ packages have adopted the Linux foundation file system hierarchy standard
-to ensure ROCm components follow open source conventions for Linux-based
-distributions.
-
-:::
-
-:::{grid-item-card}
-**[GPU isolation techniques](./gpu-isolation.md)**
-
-Restricting the access of applications to a subset of GPUs, aka isolating GPUs
-allows users to hide GPU resources from programs.
-
-:::
-
-:::{grid-item-card}
-**[GPU architectures](./gpu-arch.md)**
-
-AMD documentation around architectural details from both the CDNA and RDNA
-product lines.
-
-:::
-
-:::{grid-item-card}
-**[GPU memory](./gpu-memory.md)**
-
-Learn about the different types of memory allocations.
-
-:::
-
-:::::

docs/temp/gpu_libraries.md

@@ -1 +0,0 @@
-# GPU Libraries

docs/temp/how-to-index.md

@@ -1,34 +0,0 @@
-# How-to documentation
-
-:::::{grid} 1 1 2 2
-:gutter: 1
-
-:::{grid-item-card}
-**[Tuning guides](./tuning-guides/index.md)**
-
-Use case-specific system setup and tuning guides.
-
-:::
-
-:::{grid-item-card}
-**[Deep-learning guide](./deep-learning-rocm.md)**
-
-Installation of various deep learning frameworks and applications.
-
-:::
-
-:::{grid-item-card}
-**[GPU-enabled MPI](./gpu-enabled-mpi.rst)**
-
-This chapter exemplifies how to set up Open MPI with the ROCm platform.
-
-:::
-
-:::{grid-item-card}
-**[System debugging guide](./system-debugging.md)**
-
-Useful commands to debug misbehaving ROCm installations.
-
-:::
-
-:::::

docs/temp/install-index.md

@@ -1,30 +0,0 @@
-# ROCm tutorials
-
-:::::{grid} 1 1 2 2
-:gutter: 1
-
-:::{grid-item-card}
-**[Installing ROCm](./install/index.md)**
-
-Learn how to install ROCm on Linux and Windows.
-
-:::
-
-:::{grid-item-card}
-**[ROCm examples](https://github.com/amd/rocm-examples)**
-
-Sample code demonstrating the HIP API and ROCm-accelerated domain libraries.
-
-:::
-
-:::{grid-item-card}
-**Artificial intelligence**
-
-Detailed walkthroughs of specific artificial intelligence use cases using ROCm acceleration.
-
-* [Implementing Inception v3 on ROCm with PyTorch](../conceptual/ai-pytorch-inception.md)
-* [Optimizing inference with MIGraphX](../conceptual/ai-migraphx-optimization.md)
-
-:::
-
-:::::

docs/temp/kernel_userspace.md

@@ -1 +0,0 @@
-# Kernel and User Space Compatibility

docs/temp/packaging_guidelines.md

@@ -1 +0,0 @@
-# Packaging Guidelines

docs/temp/rocm-a-z.md

@@ -1,64 +0,0 @@
-# ROCm A-Z
-
-:::{table}
-:name: rocm-a-z
-
-| ROCm product | Description |
-| :---------------- | :------------ |
-| [AMD Compute Language Runtimes (CLR)](https://github.com/ROCm/clr) | Contains source code for AMD's compute languages runtimes: {doc}`HIP <hip:index>` and OpenCL |
-| [AMDMIGraphX](https://github.com/ROCm/AMDMIGraphX/) | A graph inference engine that accelerates machine learning model inference |
-| [AOMP](https://github.com/ROCm/aomp/) | A scripted build of [LLVM](https://github.com/ROCm/llvm-project) and supporting software |
-| [Asynchronous Task and Memory Interface (ATMI)](https://github.com/ROCm/atmi/) | A runtime framework for efficient task management in heterogeneous CPU-GPU systems |
-| [Composable Kernel](https://rocm.docs.amd.com/projects/composable_kernel/en/latest/) | A library that aims to provide a programming model for writing performance critical kernels for machine learning workloads across multiple architectures |
-| [Flang](https://github.com/ROCm/flang/) | An out-of-tree Fortran compiler targeting LLVM |
-| [Half-precision floating point library (half)](https://github.com/ROCm/half/) | A C++ header-only library that provides an IEEE 754 conformant, 16-bit half-precision floating-point type along with corresponding arithmetic operators, type conversions, and common mathematical functions |
-| {doc}`HIP <hip:index>` | AMD’s GPU programming language extension and the GPU runtime |
-| [hipBLAS](https://github.com/ROCm/hipBLAS/) | A BLAS-marshaling library that supports [rocBLAS](https://rocm.docs.amd.com/projects/rocBLAS/en/latest/) and cuBLAS backends |
-| [HIPCC](https://rocm.docs.amd.com/projects/HIPCC/en/latest/) | A compiler driver utility that calls Clang or NVCC and passes the appropriate include and library options for the target compiler and HIP infrastructure |
-| [hipCUB](https://rocm.docs.amd.com/projects/hipCUB/en/latest/) | A thin header-only wrapper library on top of [rocPRIM](https://rocm.docs.amd.com/projects/rocPRIM/en/latest/) or CUB that allows project porting using the CUB library to the HIP layer |
-| [hipFFT](https://rocm.docs.amd.com/projects/hipFFT/en/latest/) | An FFT-marshalling library that supports rocFFT or cuFFT backends |
-| [hipfort](https://rocm.docs.amd.com/projects/hipfort/en/latest/) | A Fortran interface library for accessing GPU Kernels |
-| {doc}`HIPIFY <hipify:index>` | A set of tools for translating CUDA source code into portable HIP C++ |
-| [hipify-clang](https://rocm.docs.amd.com/projects/HIPIFY/en/latest/hipify-clang.html) | A Clang-based tool for translating CUDA sources into HIP sources |
-| [hipify-perl](https://rocm.docs.amd.com/projects/HIPIFY/en/latest/hipify-perl.html) | An autogenerated, perl-based script that translates CUDA source code into portable HIP C++ |
-| [hipSOLVER](https://rocm.docs.amd.com/projects/hipSOLVER/en/latest/) | A LAPACK-marshalling library that supports [rocSOLVER](https://rocm.docs.amd.com/projects/rocSOLVER/en/latest/) and cuSOLVER backends |
-| [hipSPARSE](https://rocm.docs.amd.com/projects/hipSPARSE/en/latest/)  | A SPARSE-marshalling library that supports [rocSPARSE](https://rocm.docs.amd.com/projects/rocSPARSE/en/latest/) and cuSPARSE backends |
-| [hipTensor](https://github.com/ROCm/hipTensor) | AMD's C++ library for accelerating tensor primitives based on the composable kernel library |
-| [LLVM](https://github.com/ROCm/llvm-project) | A toolkit for the construction of highly optimized compilers, optimizers, and run-time environments |
-| [MIGraphX](https://rocm.docs.amd.com/projects/AMDMIGraphX/en/latest/) | A graph inference engine that accelerates machine learning model inference |
-| [MIOpen](https://rocm.docs.amd.com/projects/MIOpen/en/latest/) | An open source deep-learning library |
-| [MIOpenGEMM](https://github.com/ROCm/MIOpenGEMM) | An OpenCL general matrix multiplication (GEMM) API and kernel generator |
-| [MIOpenTensile](https://github.com/ROCm/MIOpenTensile) | Provides host-callable interfaces to Tensile library |
-| [MIVisionX](https://rocm.docs.amd.com/projects/MIVisionX/en/latest/doxygen/html/index.html) | A set of comprehensive computer vision and machine learning libraries, utilities, and applications |
-| [Radeon Compute Profiler (RCP)](https://github.com/GPUOpen-Tools/radeon_compute_profiler/) | A performance analysis tool that gathers data from the API run-time and GPU for OpenCL and ROCm/HSA applications |
-| [RCCL](https://rocm.docs.amd.com/projects/rccl/en/latest/) | A standalone library that provides multi-GPU and multi-node collective communication primitives |
-| [rocAL](https://rocm.docs.amd.com/projects/rocAL/en/latest/doxygen/html/index.html) | An augmentation library designed to decode and process images and videos |
-| [rocALUTION](https://rocm.docs.amd.com/projects/rocALUTION/en/latest/) | A sparse linear algebra library for exploring fine-grained parallelism on ROCm runtime and toolchains |
-| [RocBandwidthTest](https://github.com/ROCm/rocm_bandwidth_test/) | Captures the performance characteristics of buffer copying and kernel read/write operations |
-| [rocBLAS](https://rocm.docs.amd.com/projects/rocBLAS/en/latest/)| A BLAS implementation (in the HIP programming language) on the ROCm runtime and toolchains |
-| [rocFFT](https://rocm.docs.amd.com/projects/rocFFT/en/latest/) | A software library for computing fast Fourier transforms (FFTs) written in HIP |
-| [ROCK-Kernel-Driver](https://github.com/ROCm/ROCK-Kernel-Driver/) | An AMDGPU Driver with KFD that is used by ROCm |
-| [ROCm Augmentation Library (rocAL)](https://rocm.docs.amd.com/projects/rocAL/en/latest/doxygen/html/index.html) | An augmentation library designed to decode and process images and videos |
-| [ROCmCC](https://rocm.docs.amd.com/en/latest/reference/rocmcc/rocmcc.html) | A Clang/LLVM-based compiler |
-| [ROCm cmake](https://github.com/ROCm/rocm-cmake) | A collection of CMake modules for common build and development tasks |
-| [ROCm Data Center Tool](https://rocm.docs.amd.com/projects/rdc/en/latest/) | Simplifies administration and addresses key infrastructure challenges in AMD GPUs in cluster and data-center environments |
-| [ROCm Debug Agent Library (ROCdebug-agent)](https://github.com/ROCm/rocr_debug_agent/) | A library that can print the state of all AMD GPU wavefronts that caused a queue error by sending a SIGQUIT signal to the process while the program is running |
-| [ROCm Debugger (ROCgdb)](https://rocm.docs.amd.com/projects/ROCgdb/en/latest/) | A source-level debugger for Linux, based on the GNU Debugger (GDB) |
-| [ROCm Debugger API (ROCdbgapi)](https://rocm.docs.amd.com/projects/ROCdbgapi/en/latest/) | The ROCm debugger library |
-| [rocminfo](https://github.com/ROCm/rocminfo/) | Reports system information |
-| [ROCm SMI](https://github.com/ROCm/rocm_smi_lib/) | A C library for Linux that provides a user space interface for applications to monitor and control GPU applications |
-| [ROCm Validation Suite](https://rocm.docs.amd.com/projects/ROCmValidationSuite/en/latest/) | A tool for detecting and troubleshooting common problems affecting AMD GPUs running in a high-performance computing environment |
-| [rocPRIM](https://rocm.docs.amd.com/projects/rocPRIM/en/latest/) | A header-only library for HIP parallel primitives |
-| [ROCProfiler](https://rocm.docs.amd.com/projects/rocprofiler/en/latest/profiler_home_page.html) | A profiling tool for HIP applications |
-| [rocRAND](https://rocm.docs.amd.com/projects/rocRAND/en/latest/) | Provides functions that generate pseudorandom and quasirandom numbers |
-| [ROCR-Runtime](https://github.com/ROCm/ROCR-Runtime/) | User-mode API interfaces and libraries necessary for host applications to launch compute kernels on available HSA ROCm kernel agents |
-| [rocSOLVER](https://rocm.docs.amd.com/projects/rocSOLVER/en/latest/) | An implementation of LAPACK routines on the ROCm platform, implemented in the HIP programming language and optimized for AMD’s latest discrete GPUs |
-| [rocSPARSE](https://rocm.docs.amd.com/projects/rocSPARSE/en/latest/) | Exposes a common interface that provides BLAS for sparse computation implemented on ROCm runtime and toolchains (in the HIP programming language) |
-| [rocThrust](https://rocm.docs.amd.com/projects/rocThrust/en/latest/) | A parallel algorithm library |
-| [ROCT-Thunk-Interface](https://github.com/ROCm/ROCT-Thunk-Interface/) | User-mode API interfaces used to interact with the ROCk driver |
-| [ROCTracer](https://rocm.docs.amd.com/projects/roctracer/en/latest/) | Intercepts runtime API calls and traces asynchronous activity |
-| [rocWMMA](https://rocm.docs.amd.com/projects/rocWMMA/en/latest/index.html) | A C++ library for accelerating mixed-precision matrix multiply-accumulate (MMA) operations |
-| [Tensile](https://github.com/ROCm/Tensile) | A tool for creating benchmark-driven backend libraries for GEMMs, GEMM-like problems, and general N-dimensional tensor contractions |
-| [TransferBench](https://rocm.docs.amd.com/projects/TransferBench/en/latest/) | A utility to benchmark simultaneous transfers between user-specified devices (CPUs/GPUs) |
-
-:::

docs/temp/troubleshooting.md

@@ -1,50 +0,0 @@
-# Troubleshooting
-
-**Q: What do I do if I get this error when trying to run PyTorch:**
-
-```bash
-hipErrorNoBinaryForGPU: Unable to find code object for all current devices!
-```
-
-Ans: The error denotes that the installation of PyTorch and/or other
-dependencies or libraries do not support the current GPU.
-
-**Workaround:**
-
-To implement a workaround, follow these steps:
-
-1. Confirm that the hardware supports the ROCm stack. Refer to
-{ref}`linux-support` and {ref}`windows-support`.
-
-2. Determine the gfx target.
-
-    ```bash
-    rocminfo | grep gfx
-    ```
-
-3. Check if PyTorch is compiled with the correct gfx target.
-
-    ```bash
-    TORCHDIR=$( dirname $( python3 -c 'import torch; print(torch.__file__)' ) )
-    roc-obj-ls -v $TORCHDIR/lib/libtorch_hip.so # check for gfx target
-    ```
-
-:::{note}
-    Recompile PyTorch with the right gfx target if compiling from the source if
-    the hardware is not supported. For wheels or Docker installation, contact
-    ROCm support [^ROCm_issues].
-:::
-
-**Q: Why am I unable to access Docker or GPU in user accounts?**
-
-Ans: Ensure that the user is added to docker, video, and render Linux groups as
-described in the ROCm Installation Guide at {ref}`linux_group_permissions`.
-
-**Q: Can I install PyTorch directly on bare metal?**
-
-Ans: Bare-metal installation of PyTorch is supported through wheels. Refer to
-Option 2: Install PyTorch Using Wheels Package. See {doc}`PyTorch for ROCm<rocm-install-on-linux:pytorch-install>` for more information.
-
-**Q: How do I profile PyTorch workloads?**
-
-Ans: Use the PyTorch Profiler to profile GPU kernels on ROCm.