Merge pull request #4011 from ROCm/develop

Merge develop into roc-6.2.x

.azuredevops/ci-builds/aomp-mainline.yml

@@ -0,0 +1,42 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+resources:
+  repositories:
+  - repository: aomp_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp
+    ref: amd-mainline-open
+  - repository: aomp-extras_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp-extras
+    ref: amd-mainline-open
+  - repository: flang_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/flang
+    ref: amd-mainline-open
+  - repository: llvm-project_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/llvm-project
+    ref: amd-mainline-open
+  pipelines:
+  - pipeline: rocr-runtime_pipeline
+    source: \ROCR-Runtime
+    trigger:
+      branches:
+        include:
+        - amd-master
+# this job will only be triggered after successful build sequence of llvm-project and ROCR-Runtime
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/aomp.yml
+    parameters:
+      checkoutRepo: aomp_repo

.azuredevops/ci-builds/aomp-staging.yml

@@ -27,7 +27,10 @@ resources:
   pipelines:
   - pipeline: rocr-runtime_pipeline
     source: \ROCR-Runtime
-    trigger: true
+    trigger:
+      branches:
+        include:
+        - amd-staging
 # this job will only be triggered after successful build sequence of llvm-project and ROCR-Runtime
 
 trigger: none

.azuredevops/components/AMDMIGraphX.yml

@@ -8,30 +8,31 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - git
     - cmake
-    - ninja-build
+    - git
+    - libdnnl-dev
     - libdrm-dev
-    - libnuma-dev
-    - python3-pip
-    - python3-venv
-    - libtbb-dev
-    - nlohmann-json3-dev
     - libmsgpack-dev
-    - libsqlite3-dev
+    - libnuma-dev
     - libprotobuf-dev
+    - libsqlite3-dev
+    - libtbb-dev
+    - ninja-build
+    - nlohmann-json3-dev
     - protobuf-compiler
+    - python3-pip
     - python3-pybind11
+    - python3-venv
 - name: pipModules
   type: object
   default:
     - https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
-    - onnx==1.14.1
-    - numpy==1.21.6
-    - typing==3.7.4
-    - pytest==6.0.1
-    - packaging==23.0
-    - protobuf==3.20.2
+    - onnx>=1.14.1
+    - numpy>=1.21.6
+    - typing>=3.7.4
+    - pytest>=6.0.1
+    - packaging>=23.0
+    - protobuf>=3.20.2
 - name: rocmDependencies
   type: object
   default:
@@ -50,6 +51,26 @@ parameters:
     - rocminfo
     - rocMLIR
     - rocprofiler-register
+    - roctracer
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - aomp-extras
+    - clr
+    - composable_kernel
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - MIOpen
+    - rocm-cmake
+    - ROCR-Runtime
+    - rocBLAS
+    - rocminfo
+    - rocMLIR
+    - rocprofiler-register
+    - roctracer
 
 jobs:
 - job: AMDMIGraphX
@@ -63,8 +84,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -95,17 +114,84 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
         -DCMAKE_BUILD_TYPE=Release
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
         -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
-        -DMIGRAPHX_USE_COMPOSABLEKERNEL=OFF
         -DBUILD_TESTING=ON
         -GNinja
-# REFERENCE: https://github.com/ROCm/composable_kernel/issues/782
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: AMDMIGraphX_testing
+  dependsOn: AMDMIGraphX
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: MIGRAPHX_TRACE_BENCHMARKING
+    value: 1
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+# half version should be fixed to 5.6.0
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencySource: fixed
+      fixedComponentName: half
+      fixedPipelineIdentifier: ${{ variables.HALF560_PIPELINE_ID }}
+      skipLibraryLinking: true
+      skipLlvmSymlink: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: CMake@1
+    displayName: MIGraphXTest CMake Flags
+    inputs:
+      cmakeArgs: >-
+        -DCMAKE_BUILD_TYPE=Release
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
+        -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
+        -DBUILD_TESTING=ON
+        -DMIGRAPHX_ENABLE_C_API_TEST=ON
+        ..
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: AMDMIGraphX
+      testExecutable: make
+      testParameters: -j$(nproc) check
+      testPublishResults: false

.azuredevops/components/MIOpen.yml

@@ -8,15 +8,24 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - software-properties-common
-    - python3-pip
     - cmake
-    - ninja-build
-    - libsqlite3-dev
     - libbz2-dev
-    - nlohmann-json3-dev
-    - libgtest-dev
     - libdrm-dev
+    - libeigen3-dev
+    - libgmock-dev
+    - libgtest-dev
+    - libsqlite3-dev
+    - libstdc++-12-dev
+    - libzstd-dev
+    - ninja-build
+    - nlohmann-json3-dev
+    - python3-pip
+    - software-properties-common
+    - zstd
+- name: pipModules
+  type: object
+  default:
+    - cget
 - name: rocmDependencies
   type: object
   default:
@@ -35,6 +44,24 @@ parameters:
     - clr
     - rocminfo
     - roctracer
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - composable_kernel
+    - half
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - rocMLIR
+    - ROCR-Runtime
+    - rocprofiler-register
+    - rocRAND
+    - roctracer
 
 jobs:
 - job: MIOpen
@@ -48,28 +75,17 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
-  # The default boost library from apt is 1.74, which does not satisfy MIOpen's build requirement (1.79+)
-  # Upgrade boost from apt by following https://launchpad.net/~mhier/+archive/ubuntu/libboost-latest
-  - task: Bash@3
-    displayName: 'update boost version'
-    inputs:
-      targetType: inline
-      script: sudo add-apt-repository ppa:mhier/libboost-latest -y
-  - task: Bash@3
-    displayName: 'install boost'
-    inputs:
-      targetType: inline
-      script: sudo apt-get --yes install libboost1.83-dev libboost-system1.83-dev libboost-filesystem1.83-dev
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+  # The default boost library from apt is 1.74, which does not satisfy MIOpen's build requirement (1.79+)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-boost.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
@@ -85,7 +101,8 @@ jobs:
       extraBuildFlags: >-
         -DMIOPEN_BACKEND=HIP
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/boost
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DMIOPEN_ENABLE_AI_KERNEL_TUNING=OFF
         -DMIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK=OFF
         -DCMAKE_BUILD_TYPE=Release
@@ -94,3 +111,102 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: MIOpen_testing
+  dependsOn: MIOpen
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  # The default boost library from apt is 1.74, which does not satisfy MIOpen's build requirement (1.79+)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-boost.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+# MIOpen depends on a specific version of frugally-deep which is forked here: https://github.com/ROCm/frugally-deep
+# https://github.com/ROCm/frugally-deep/blob/master/INSTALL.md
+  - task: Bash@3
+    displayName: Add Python site-packages binaries to path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.prependpath]$USER_BASE/bin"
+  - task: Bash@3
+    displayName: Install FunctionalPlus
+    inputs:
+      targetType: inline
+      script: cget install Dobiasd/FunctionalPlus
+  - task: Bash@3
+    displayName: Remove Python site-packages binaries from path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$USER_BASE/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: git clone frugally-deep
+    inputs:
+      targetType: inline
+      script: git clone https://github.com/ROCm/frugally-deep --depth=1 --shallow-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      componentName: frugally-deep
+      cmakeBuildDir: $(Build.SourcesDirectory)/frugally-deep/build
+      installDir: $(Build.SourcesDirectory)/bin
+      extraBuildFlags: -DCMAKE_PREFIX_PATH=$(Build.SourcesDirectory)/cget/cget/pkg/Dobiasd__FunctionalPlus/install
+  - task: CMake@1
+    displayName: 'MIOpen Test CMake Flags'
+    inputs:
+      cmakeArgs: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Build.SourcesDirectory)/bin;$(Build.SourcesDirectory)/cget/cget/pkg/Dobiasd__FunctionalPlus/install;$(Agent.BuildDirectory)/boost
+        -DCMAKE_INSTALL_PREFIX=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DMIOPEN_BACKEND=HIP
+        -DMIOPEN_TEST_FLAGS=" --disable-verification-cache"
+        -DCMAKE_BUILD_TYPE=release
+        -DBUILD_DEV=OFF
+        -DMIOPEN_USE_MLIR=ON
+        -DMIOPEN_GPU_SYNC=OFF
+        ..
+  - task: Bash@3
+    displayName: 'MIOpen Test Build'
+    inputs:
+      targetType: inline
+      script: |
+        cmake --build . --target tests -- -j$(nproc)
+      workingDirectory: $(Build.SourcesDirectory)/build
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: MIOpen

.azuredevops/components/MIVisionX.yml

@@ -86,8 +86,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -120,7 +118,7 @@ jobs:
 
 - job: MIVisionX_testing
   dependsOn: MIVisionX
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -162,6 +160,7 @@ jobs:
       mkdir mivisionx-tests
       cd mivisionx-tests
       cmake /opt/rocm/share/mivisionx/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: MIVisionX

.azuredevops/components/ROCR-Runtime.yml

@@ -8,15 +8,21 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - libelf-dev
     - g++
     - libdrm-dev
+    - libelf-dev
     - libnuma-dev
 - name: rocmDependencies
   type: object
   default:
     - llvm-project
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - llvm-project
+    - rocm_smi_lib
+    - rocprofiler-register
 
 jobs:
 - job: ROCR_Runtime
@@ -50,3 +56,128 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_SHARED_LIBS=ON
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: ROCR_Runtime_testing
+  dependsOn: ROCR_Runtime
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - task: Bash@3
+    displayName: Install libhwloc5
+    inputs:
+      targetType: 'inline'
+      script: |
+        wget http://ftp.us.debian.org/debian/pool/main/h/hwloc/libhwloc5_1.11.12-3_amd64.deb
+        wget http://ftp.us.debian.org/debian/pool/main/h/hwloc/libhwloc-dev_1.11.12-3_amd64.deb
+        sudo apt install -y --allow-downgrades ./libhwloc5_1.11.12-3_amd64.deb ./libhwloc-dev_1.11.12-3_amd64.deb
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
+  - task: Bash@3
+    displayName: Build kfdtest
+    continueOnError: true
+    inputs:
+      targetType: 'inline'
+      workingDirectory: $(Build.SourcesDirectory)/libhsakmt/tests/kfdtest
+      script: |
+        mkdir build && cd build
+        cmake -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm ..
+        make
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: kfdtest
+      testExecutable: BIN_DIR=$(Build.SourcesDirectory)/libhsakmt/tests/kfdtest/build ./run_kfdtest.sh
+      testParameters: '-p core --gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: $(Build.SourcesDirectory)/libhsakmt/tests/kfdtest/scripts
+  - task: Bash@3
+    displayName: Build rdmatest app
+    continueOnError: true
+    inputs:
+      targetType: 'inline'
+      workingDirectory: $(Build.SourcesDirectory)/libhsakmt/tests/rdma/simple/app
+      script: |
+        cmake -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm -DLIBHSAKMT_PATH=$(Agent.BuildDirectory)/rocm -DDRM_AMDGPU_INCLUDE_DIRS=$(Agent.BuildDirectory)/rocm/include .
+        cmake --build .
+  - task: Bash@3
+    displayName: Build rdmatest driver
+    continueOnError: true
+    inputs:
+      targetType: 'inline'
+      workingDirectory: $(Build.SourcesDirectory)/libhsakmt/tests/rdma/simple/drv
+      script: |
+        sed -i 's/HSAKMT_PAGE_SHIFT/PAGE_SHIFT/g' amdp2ptest.c
+        sed -i 's/"MIT"/"GPL"/' amdp2ptest.c
+        RDMA_HEADER_DIR=/usr/src/amdgpu-*/include make all
+  - task: Bash@3
+    displayName: Install rdmatest driver
+    continueOnError: true
+    inputs:
+      targetType: 'inline'
+      workingDirectory: $(Build.SourcesDirectory)/libhsakmt/tests/rdma/simple/drv
+      script: |
+        sudo rmmod amdp2ptest.ko
+        sudo insmod amdp2ptest.ko
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rdmatest
+      testExecutable: yes | ./rdma_test
+      testParameters: ''
+      testDir: $(Build.SourcesDirectory)/libhsakmt/tests/rdma/simple/app
+      testPublishResults: false
+  - task: Bash@3
+    displayName: Build rocrtst
+    continueOnError: true
+    inputs:
+      targetType: 'inline'
+      workingDirectory: $(Build.SourcesDirectory)/rocrtst/suites/test_common
+      script: |
+        sudo rm -rf $(Agent.BuildDirectory)/external/llvm-project
+        mkdir -p $(Agent.BuildDirectory)/external/llvm-project/clang/lib
+        sudo ln -s $(Agent.BuildDirectory)/rocm/llvm/lib/clang/20/include $(Agent.BuildDirectory)/external/llvm-project/clang/lib/Headers
+        mkdir build && cd build
+        cmake .. \
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm \
+          -DTARGET_DEVICES=$(JOB_GPU_TARGET) \
+          -DROCM_DIR=$(Agent.BuildDirectory)/rocm \
+          -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm/bin \
+          -DOPENCL_DIR=$(Agent.BuildDirectory)/rocm/llvm/bin
+        make
+        make rocrtst_kernels
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocrtst
+      testExecutable: ./rocrtst64
+      testParameters: '--gtest_filter="-rocrtstNeg.Memory_Negative_Tests" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: $(Build.SourcesDirectory)/rocrtst/suites/test_common/build/$(JOB_GPU_TARGET)

.azuredevops/components/ROCgdb.yml

@@ -5,24 +5,47 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# reference: https://github.com/ROCm/ROCgdb/blob/amd-staging/README-ROCM.md
 - name: aptPackages
   type: object
   default:
-    - libgmp-dev
-    - libmpfr-dev
-    - texinfo
     - bison
+    - dejagnu
     - flex
+    - libbabeltrace-dev
+    - libexpat-dev
+    - libgmp-dev
+    - liblzma-dev
+    - libmpfr-dev
+    - ncurses-dev
+    - texinfo
+    - zlib1g-dev
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - ROCdbgapi
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
-- job: rocgdb
+- job: ROCgdb
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool:
-    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  - name: PKG_CONFIG_PATH
+    value: $(Agent.BuildDirectory)/rocm/share/pkgconfig
+  pool: $(JOB_TEST_POOL)
   workspace:
     clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -31,5 +54,64 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-autotools.yml
+    parameters:
+      configureFlags: >-
+        --program-prefix=roc
+        --enable-64-bit-bfd
+        --enable-targets="x86_64-linux-gnu,amdgcn-amd-amdhsa"
+        --disable-ld
+        --disable-gas
+        --disable-gdbserver
+        --disable-sim
+        --enable-tui
+        --disable-gdbtk
+        --disable-shared
+        --disable-gprofng
+        --with-expat
+        --with-system-zlib
+        --without-guile
+        --with-babeltrace
+        --with-lzma
+        --with-python=python3
+        --with-rocm-dbgapi=$(Agent.BuildDirectory)/rocm
+        LDFLAGS="-Wl,--enable-new-dtags,-rpath=$(Agent.BuildDirectory)/rocm/lib"
+      makeCallPrefix: LD_RUN_PATH='${ORIGIN}/../lib'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        echo "##vso[task.prependpath]/opt/rocm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - task: Bash@3
+    displayName: check-gdb
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: make check-gdb TESTS=gdb.rocm/simple.exp
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: print gdb log
+    inputs:
+      targetType: inline
+      script: find -name gdb.log -exec cat {} \;
+      workingDirectory: $(Build.SourcesDirectory)

.azuredevops/components/ROCmValidationSuite.yml

@@ -13,6 +13,7 @@ parameters:
     - libyaml-cpp-dev
     - libpci-dev
     - libpci3
+    - libgst-dev
     - libgtest-dev
     - git
 - name: rocmDependencies
@@ -40,9 +41,11 @@ parameters:
     - llvm-project
     - rocBLAS
     - rocm_smi_lib
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
     - rocRAND
+    - roctracer
 
 jobs:
 - job: ROCmValidationSuite
@@ -63,8 +66,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -97,7 +98,7 @@ jobs:
 
 - job: ROCmValidationSuite_testing
   dependsOn: ROCmValidationSuite
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -131,6 +132,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: ROCmValidationSuite

.azuredevops/components/Tensile.yml

@@ -0,0 +1,175 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - python3-pip
+    - cmake
+    - libmsgpack-dev
+    - libboost-program-options-dev
+- name: pipModules
+  type: object
+  default:
+    - tox
+    - pytest
+- name: rocmDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - rocm_smi_lib
+    - rocprofiler-register
+    - ROCR-Runtime
+
+jobs:
+- job: Tensile
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Create wheel file
+    inputs:
+      targetType: inline
+      script: python3 setup.py bdist_wheel
+      workingDirectory: $(Build.SourcesDirectory)
+    env:
+      ROCM_PATH: $(Agent.BuildDirectory)/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.SourcesDirectory)/dist
+      contentsString: '*.whl'
+      targetDir: $(Build.ArtifactStagingDirectory)
+      clean: false
+  - task: PublishPipelineArtifact@1
+    displayName: 'wheel file Publish'
+    retryCountOnTaskFailure: 3
+    inputs:
+      targetPath: $(Build.ArtifactStagingDirectory)
+
+- job: Tensile_testing
+  dependsOn: Tensile
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Wheel Files'
+    inputs:
+      itemPattern: '**/*.whl'
+      targetPath: $(Agent.BuildDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: pip install
+    inputs:
+      targetType: inline
+      script: find -name *.whl -exec pip install {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: Add Python site-packages binaries to path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.prependpath]$USER_BASE/bin"
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - task: Bash@3
+    displayName: tox test
+    inputs:
+      targetType: inline
+      script: tox run -v -e ci -- -m pre_checkin
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Remove Python site-packages binaries from path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$USER_BASE/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"

.azuredevops/components/amdsmi.yml

@@ -35,7 +35,7 @@ jobs:
 
 - job: amdsmi_testing
   dependsOn: amdsmi
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -53,6 +53,9 @@ jobs:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: amdsmi

.azuredevops/components/aomp.yml

@@ -413,7 +413,7 @@ jobs:
 
 - job: aomp_testing
   dependsOn: aomp
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -472,6 +472,7 @@ jobs:
       Contents: FileCheck
       TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests/bin
       retryCount: 3
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - task: Bash@3
     displayName: Test AOMP
     continueOnError: true

.azuredevops/components/composable_kernel.yml

@@ -48,8 +48,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -90,6 +88,8 @@ jobs:
         -DCMAKE_C_COMPILER_LAUNCHER=ccache
         -DCMAKE_HIP_FLAGS="-Wno-missing-include-dirs"
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCK_BUILD_JIT_LIB=ON
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
         -DCMAKE_BUILD_TYPE=Release
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
@@ -99,7 +99,7 @@ jobs:
 
 - job: composable_kernel_testing
   dependsOn: composable_kernel
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -135,6 +135,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - task: Bash@3
     displayName: Iterate through test scripts
     inputs:

.azuredevops/components/copyHIP.yml

@@ -25,7 +25,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-download.yml
     parameters:
       componentName: HIP
-      pipelineId: $(hip-pipeline-id)
+      pipelineId: $(HIP_PIPELINE_ID)
   - template:  ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
       sourceDir: $(Agent.BuildDirectory)/rocm

.azuredevops/components/hip-tests.yml

@@ -52,8 +52,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -89,9 +87,9 @@ jobs:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hip_tests_testing
-  timeoutInMinutes: 180
+  timeoutInMinutes: 240
   dependsOn: hip_tests
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -130,13 +128,14 @@ jobs:
     inputs:
       targetType: inline
       script: |
+        sudo rm -rf /opt/rocm
         sudo mkdir -p /opt/rocm/bin
         sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rocm_agent_enumerator /opt/rocm/bin/rocm_agent_enumerator
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hip_tests
       testDir: $(Agent.BuildDirectory)/rocm/share/hip
-      reloadAMDGPU: true
   - task: Bash@3
     displayName: Clean up symlink
     inputs:

.azuredevops/components/hipBLAS-common.yml

@@ -29,7 +29,7 @@ jobs:
   - name: ROCM_PATH
     value: $(Agent.BuildDirectory)/rocm
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/components/hipBLAS.yml

@@ -16,7 +16,6 @@ parameters:
     - libgtest-dev
     - wget
     - python3-pip
-    - libomp-dev
 - name: rocmDependencies
   type: object
   default:
@@ -46,6 +45,7 @@ parameters:
     - rocBLAS
     - rocSPARSE
     - rocSOLVER
+    - roctracer
 
 jobs:
 - job: hipBLAS
@@ -59,8 +59,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -98,7 +96,7 @@ jobs:
 
 - job: hipBLAS_testing
   dependsOn: hipBLAS
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -132,6 +130,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipBLAS

.azuredevops/components/hipBLASLt.yml

@@ -16,13 +16,13 @@ parameters:
     - python3-pip
     - python3-venv
     - gfortran
-    - libomp-dev
     - libblas-dev
     - ccache
 - name: pipModules
   type: object
   default:
     - joblib
+    - packaging
 - name: rocmDependencies
   type: object
   default:
@@ -66,6 +66,8 @@ jobs:
     value: $(Agent.BuildDirectory)/rocm/bin/hipcc
   - name: PATH
     value: $(Agent.BuildDirectory)/rocm/llvm/bin:$(Agent.BuildDirectory)/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin
+  - name: DAY_STRING
+    value: $[format('{0:ddMMyyyy}', pipeline.startTime)]
   pool: ${{ variables.ULTRA_BUILD_POOL }}
   workspace:
     clean: all
@@ -73,8 +75,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -149,7 +149,7 @@ jobs:
 
 - job: hipBLASLt_testing
   dependsOn: hipBLASLt
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -165,6 +165,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
     parameters:
@@ -183,6 +184,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipBLASLt

.azuredevops/components/hipCUB.yml

@@ -43,8 +43,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -78,7 +76,7 @@ jobs:
 
 - job: hipCUB_testing
   dependsOn: hipCUB
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -112,6 +110,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipCUB

.azuredevops/components/hipFFT.yml

@@ -14,11 +14,11 @@ parameters:
     - libgtest-dev
     - libfftw3-dev
     - python3-pip
-    - libomp-14-dev
 # rocm dependencies should match dependencies-rocm.yml
 - name: rocmDependencies
   type: object
   default:
+    - aomp
     - rocRAND
     - hipRAND
     - llvm-project
@@ -55,8 +55,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -96,7 +94,7 @@ jobs:
 
 - job: hipFFT_testing
   dependsOn: hipFFT
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -130,6 +128,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipFFT

.azuredevops/components/hipRAND.yml

@@ -45,8 +45,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -82,7 +80,7 @@ jobs:
 
 - job: hipRAND_testing
   dependsOn: hipRAND
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -116,6 +114,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipRAND

.azuredevops/components/hipSOLVER.yml

@@ -42,6 +42,7 @@ parameters:
     - ROCR-Runtime
     - rocSOLVER
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: hipSOLVER
@@ -56,8 +57,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -102,7 +101,7 @@ jobs:
 
 - job: hipSOLVER_testing
   dependsOn: hipSOLVER
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -136,6 +135,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipSOLVER

.azuredevops/components/hipSPARSE.yml

@@ -37,6 +37,7 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: hipSPARSE
@@ -51,8 +52,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -96,7 +95,7 @@ jobs:
 
 - job: hipSPARSE_testing
   dependsOn: hipSPARSE
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -130,6 +129,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipSPARSE

.azuredevops/components/hipSPARSELt.yml

@@ -15,7 +15,6 @@ parameters:
     - python3-pip
     - gfortran
     - libgfortran5
-    - libomp-dev
     - libopenblas-dev
 - name: pipModules
   type: object
@@ -124,7 +123,7 @@ jobs:
 
 - job: hipSPARSELt_testing
   dependsOn: hipSPARSELt
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -152,6 +151,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipSPARSELt

.azuredevops/components/hipTensor.yml

@@ -42,8 +42,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -79,7 +77,7 @@ jobs:
 - job: hipTensor_testing
   timeoutInMinutes: 90
   dependsOn: hipTensor
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -113,8 +111,9 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipTensor
       testDir: '$(Agent.BuildDirectory)/rocm/bin/hiptensor'
-      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testParameters: '-E ".*-extended" -VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml'

.azuredevops/components/hipfort.yml

@@ -51,8 +51,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -94,7 +92,7 @@ jobs:
 
 - job: hipfort_testing
   dependsOn: hipfort
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -139,6 +137,7 @@ jobs:
         sudo rm -rf /opt/rocm
         sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
       workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - task: Bash@3
     displayName: 'Test hipfort'
     inputs:

.azuredevops/components/omnitrace.yml

@@ -0,0 +1,140 @@
+# largely referenced from: https://github.com/ROCm/omnitrace/blob/main/.github/workflows/ubuntu-jammy.yml
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - autoconf
+    - autotools-dev
+    - bison
+    - build-essential
+    - bzip2
+    - clang
+    - cmake
+    - environment-modules
+    - g++-12
+    - libdrm-dev
+    - libfabric-dev
+    - libiberty-dev
+    - libpapi-dev
+    - libpfm4-dev
+    - libtool
+    - libopenmpi-dev
+    - m4
+    - openmpi-bin
+    - software-properties-common
+    - python3-pip
+    - texinfo
+    - zlib1g-dev
+- name: pipModules
+  type: object
+  default:
+    - numpy
+    - perfetto
+    - dataclasses
+- name: rocmDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - llvm-project
+    - rccl
+    - rocm-core
+    - rocm_smi_lib
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler
+    - rocprofiler-register
+    - roctracer
+
+jobs:
+- job: omnitrace
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+# build flags reference: https://rocm.docs.amd.com/projects/omnitrace/en/latest/install/install.html
+      extraBuildFlags: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DOMNITRACE_BUILD_TESTING=ON
+        -DOMNITRACE_BUILD_DYNINST=ON
+        -DOMNITRACE_BUILD_LIBUNWIND=ON
+        -DDYNINST_BUILD_TBB=ON
+        -DDYNINST_BUILD_ELFUTILS=ON
+        -DDYNINST_BUILD_LIBIBERTY=ON
+        -DDYNINST_BUILD_BOOST=ON
+        -DOMNITRACE_USE_PAPI=ON
+        -DOMNITRACE_USE_MPI=ON
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j32
+  - task: Bash@3
+    displayName: Set up omnitrace env
+    inputs:
+      targetType: inline
+      script: source share/omnitrace/setup-env.sh
+      workingDirectory: build
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: omnitrace
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rccl.yml

@@ -60,8 +60,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -102,7 +100,7 @@ jobs:
 - job: rccl_testing
   timeoutInMinutes: 120
   dependsOn: rccl
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -136,6 +134,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rccl

.azuredevops/components/rdc.yml

@@ -60,8 +60,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -113,7 +111,7 @@ jobs:
 
 - job: rdc_testing
   dependsOn: rdc
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -131,6 +129,8 @@ jobs:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
     parameters:
       ${{ if eq(parameters.checkoutRef, '') }}:
@@ -156,6 +156,7 @@ jobs:
         sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rdcd /usr/sbin/rdcd
         echo $(Agent.BuildDirectory)/rocm/lib/rdc/grpc/lib | sudo tee /etc/ld.so.conf.d/grpc.conf
         sudo ldconfig -v
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - task: Bash@3
     displayName: Test rdc
     inputs:

.azuredevops/components/rocAL.yml

@@ -53,6 +53,7 @@ parameters:
     - half
     - llvm-project
     - MIVisionX
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
     - rpp
@@ -70,8 +71,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - task: Bash@3
     displayName: 'Register libjpeg-turbo packages'
@@ -157,7 +156,7 @@ jobs:
 
 - job: rocAL_testing
   dependsOn: rocAL
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -220,11 +219,11 @@ jobs:
         mkdir rocAL-tests
         cd rocAL-tests
         cmake $(Agent.BuildDirectory)/rocm/share/rocal/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocAL
       testDir: rocAL-tests
-      reloadAMDGPU: true
   - task: Bash@3
     displayName: Clean up libjpeg-turbo
     inputs:

.azuredevops/components/rocALUTION.yml

@@ -43,6 +43,7 @@ parameters:
     - ROCR-Runtime
     - rocRAND
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: rocALUTION
@@ -59,8 +60,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -97,7 +96,7 @@ jobs:
 
 - job: rocALUTION_testing
   dependsOn: rocALUTION
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -131,6 +130,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocALUTION

.azuredevops/components/rocBLAS.yml

@@ -40,6 +40,7 @@ parameters:
     - aomp-extras
     - hipBLAS-common
     - hipBLASLt
+    - roctracer
 - name: rocmTestDependencies
   type: object
   default:
@@ -52,6 +53,7 @@ parameters:
     - rocprofiler-register
     - rocm_smi_lib
     - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: rocBLAS
@@ -73,8 +75,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -120,7 +120,7 @@ jobs:
 
 - job: rocBLAS_testing
   dependsOn: rocBLAS
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -155,6 +155,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocBLAS

.azuredevops/components/rocDecode.yml

@@ -17,7 +17,7 @@ parameters:
     - libavformat-dev
     - libavutil-dev
     - libstdc++-12-dev
-    - libva-dev
+    - libva-amdgpu-dev
     - mesa-amdgpu-va-drivers
     - libdrm-dev
 - name: rocmDependencies
@@ -35,6 +35,7 @@ parameters:
   default:
     - clr
     - llvm-project
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
 
@@ -88,11 +89,13 @@ jobs:
 
 - job: rocDecode_testing
   dependsOn: rocDecode
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+  pool:
+    name: $(JOB_TEST_POOL)
+    demands: firstRenderDeviceAccess
   workspace:
     clean: all
   strategy:
@@ -101,6 +104,18 @@ jobs:
         JOB_GPU_TARGET: gfx942
         JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
+  # Since mesa-amdgpu-multimedia-devel is not directly available from apt, register it
+  - task: Bash@3
+    displayName: 'Register ROCm packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/${{ variables.KEYRING_VERSION }}/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/amdgpu.list
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${{ variables.KEYRING_VERSION }} jammy main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
+        echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+        sudo apt update
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
@@ -128,6 +143,7 @@ jobs:
       mkdir rocDecode-tests
       cd rocDecode-tests
       cmake /opt/rocm/share/rocdecode/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocDecode

.azuredevops/components/rocFFT.yml

@@ -55,8 +55,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -95,7 +93,7 @@ jobs:
 
 - job: rocFFT_testing
   dependsOn: rocFFT
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -129,6 +127,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocFFT

.azuredevops/components/rocMLIR.yml

@@ -13,6 +13,7 @@ parameters:
     - git
     - python3-pip
     - libdrm-dev
+    - libstdc++-12-dev
 - name: pipModules
   type: object
   default:
@@ -28,7 +29,7 @@ parameters:
     - ROCR-Runtime
 
 jobs:
-- job: rocMLIR_library
+- job: rocMLIR
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -66,16 +67,19 @@ jobs:
 
 # compiling and running test on the test system together
 - job: rocMLIR_testing
+  dependsOn: rocMLIR
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.GFX942_TEST_POOL }}
+  pool: $(JOB_TEST_POOL)
   workspace:
     clean: all
   strategy:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -105,7 +109,9 @@ jobs:
     displayName: ROCm symbolic link
     inputs:
       targetType: inline
-      script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       installEnabled: false
@@ -117,6 +123,7 @@ jobs:
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DROCM_TEST_CHIPSET=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocMLIR

.azuredevops/components/rocPRIM.yml

@@ -42,8 +42,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -77,7 +75,7 @@ jobs:
 
 - job: rocPRIM_testing
   dependsOn: rocPRIM
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -111,6 +109,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocPRIM

.azuredevops/components/rocPyDecode.yml

@@ -8,25 +8,22 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - python3-pip
     - cmake
-    - ninja-build
-    - pkg-config
     - ffmpeg
     - libavcodec-dev
     - libavformat-dev
     - libavutil-dev
-    - libva-dev
-    - libdrm-dev
-    - pybind11-dev
-    - python3-pybind11
     - libdlpack-dev
+    - libdrm-dev
+    - libva-dev
+    - ninja-build
+    - pkg-config
+    - python3-pip
 - name: pipModules
   type: object
   default:
-    - -i
-    - https://test.pypi.org/simple
-    - hip-python
+    - numpy
+    - pybind11
 - name: rocmDependencies
   type: object
   default:
@@ -52,13 +49,16 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
       pipModules: ${{ parameters.pipModules }}
+  - task: Bash@3
+    displayName: 'pip install hip-python'
+    inputs:
+      targetType: inline
+      script: pip install -i https://test.pypi.org/simple hip-python
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
@@ -73,11 +73,26 @@ jobs:
       # manual build case: triggered by ROCm/ROCm repo
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - task: Bash@3
+    displayName: 'ROCm symbolic link'
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: 'Save Python Package Paths'
+    inputs:
+      targetType: inline
+      script: |
+        echo "##vso[task.setvariable variable=PYTHON_USER_SITE;]$(python3 -m site --user-site)"
+        echo "##vso[task.setvariable variable=PYTHON_DIST_PACKAGES;]$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
+        echo "##vso[task.setvariable variable=PYBIND11_PATH;]$(python3 -c 'import pybind11; print(pybind11.get_cmake_dir())')"
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(PYTHON_USER_SITE)/pybind11;$(PYTHON_DIST_PACKAGES)/pybind11;$(PYBIND11_PATH)
         -DCMAKE_BUILD_TYPE=Release
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DCMAKE_INSTALL_PREFIX_PYTHON=$(Build.BinariesDirectory)
@@ -85,3 +100,126 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+      publish: false
+  - task: Bash@3
+    displayName: Create wheel file
+    inputs:
+      targetType: inline
+      script: |
+        export ROCM_PATH=$(Agent.BuildDirectory)/rocm
+        export HIP_INCLUDE_DIRS=$(Agent.BuildDirectory)/rocm/include/hip
+        python3 setup.py bdist_wheel
+      workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.SourcesDirectory)/dist
+      contentsString: '*.whl'
+      targetDir: $(Build.ArtifactStagingDirectory)
+      clean: false
+  - task: PublishPipelineArtifact@1
+    displayName: 'wheel file Publish'
+    retryCountOnTaskFailure: 3
+    inputs:
+      targetPath: $(Build.ArtifactStagingDirectory)
+
+- job: rocPyDecode_testing
+  dependsOn: rocPyDecode
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    name: $(JOB_TEST_POOL)
+    demands: firstRenderDeviceAccess
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - task: Bash@3
+    displayName: Ensure pybind11-dev is not installed
+    inputs:
+      targetType: inline
+      script: |
+        if dpkg -l | grep -q pybind11-dev; then
+          echo "Removing pybind11-dev..."
+          sudo apt remove -y pybind11-dev
+        else
+          echo "pybind11-dev is not installed."
+        fi
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - task: Bash@3
+    displayName: 'pip install hip-python'
+    inputs:
+      targetType: inline
+      script: pip install -i https://test.pypi.org/simple hip-python
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Wheel Files'
+    inputs:
+      itemPattern: '**/*.whl'
+      targetPath: $(Agent.BuildDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: pip install
+    inputs:
+      targetType: inline
+      script: |
+        pip uninstall -y rocPyDecode
+        find -name *.whl -exec pip install {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Setup search paths
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        echo "##vso[task.setvariable variable=PYTHON_USER_SITE;]$(python3 -m site --user-site)"
+        echo "##vso[task.setvariable variable=PYTHON_DIST_PACKAGES;]$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
+        echo "##vso[task.setvariable variable=PYBIND11_PATH;]$(python3 -c 'import pybind11; print(pybind11.get_cmake_dir())')"
+  - task: CMake@1
+    displayName: 'rocPyDecode Test CMake Flags'
+    inputs:
+      cmakeArgs: >-
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(PYTHON_USER_SITE)/pybind11;$(PYTHON_DIST_PACKAGES)/pybind11;$(PYBIND11_PATH)
+        -DCMAKE_BUILD_TYPE=Release
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        ..
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocPyDecode
+      testDir: $(Build.SourcesDirectory)/build
+# sudo required for pip install but screws up permissions for next pipeline run
+  - task: Bash@3
+    displayName: Clean up test environment
+    condition: always()
+    inputs:
+      targetType: inline
+      script: |
+        pip uninstall -y rocPyDecode
+        pip uninstall -y hip-python

.azuredevops/components/rocRAND.yml

@@ -45,8 +45,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -79,7 +77,7 @@ jobs:
 
 - job: rocRAND_testing
   dependsOn: rocRAND
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -113,6 +111,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocRAND

.azuredevops/components/rocSOLVER.yml

@@ -44,6 +44,7 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: rocSOLVER
@@ -57,8 +58,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -112,7 +111,7 @@ jobs:
 
 - job: rocSOLVER_testing
   dependsOn: rocSOLVER
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -146,6 +145,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocSOLVER

.azuredevops/components/rocSPARSE.yml

@@ -40,6 +40,7 @@ parameters:
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: rocSPARSE
@@ -55,8 +56,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -106,7 +105,7 @@ jobs:
 - job: rocSPARSE_testing
   timeoutInMinutes: 90
   dependsOn: rocSPARSE
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -140,6 +139,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocSPARSE

.azuredevops/components/rocThrust.yml

@@ -47,8 +47,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -82,7 +80,7 @@ jobs:
 
 - job: rocThrust_testing
   dependsOn: rocThrust
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -116,6 +114,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocThrust

.azuredevops/components/rocWMMA.yml

@@ -16,10 +16,10 @@ parameters:
     - googletest
     - libfftw3-dev
     - git
-    - libomp-dev
 - name: rocmDependencies
   type: object
   default:
+    - aomp
     - rocm-cmake
     - llvm-project
     - ROCR-Runtime
@@ -42,6 +42,7 @@ parameters:
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: rocWMMA
@@ -55,8 +56,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -93,7 +92,7 @@ jobs:
 - job: rocWMMA_testing
   timeoutInMinutes: 90
   dependsOn: rocWMMA
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -127,6 +126,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocWMMA

.azuredevops/components/rocm-examples.yml

@@ -60,6 +60,7 @@ parameters:
     - rocSOLVER
     - rocSPARSE
     - rocThrust
+    - roctracer
 
 jobs:
 - job: rocm_examples
@@ -73,8 +74,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -116,7 +115,7 @@ jobs:
 
 - job: rocm_examples_testing
   dependsOn: rocm_examples
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -135,9 +134,9 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+      checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
     parameters:
       ${{ if eq(parameters.checkoutRef, '') }}:
@@ -152,20 +151,18 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
-  - task: Bash@3
-    displayName: Unload and reload AMDGPU
-    inputs:
-      targetType: inline
-      script: |
-        sudo modprobe -r amdgpu
-        sudo modprobe amdgpu
-  - task: Bash@3
-    displayName: Iterate through examples
-    inputs:
-      targetType: inline
-      script: |
-        for file in *; do
-          echo Now running: $file
-          ./$file | tee -a $(TEST_LOG_FILE)
-        done
-      workingDirectory: $(Agent.BuildDirectory)/rocm/examples
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      # https://github.com/ROCm/HIP/issues/2203
+      extraBuildFlags: >-
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_ROOT=$(Agent.BuildDirectory)/rocm
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
+        -DCMAKE_EXE_LINKER_FLAGS=-fgpu-rdc
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm-examples
+      testDir: $(Build.SourcesDirectory)/build

.azuredevops/components/rocm_bandwidth_test.yml

@@ -27,6 +27,7 @@ parameters:
 - name: rocmTestDependencies
   type: object
   default:
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
 
@@ -72,7 +73,7 @@ jobs:
 
 - job: rocm_bandwidth_test_testing
   dependsOn: rocm_bandwidth_test
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -99,6 +100,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocm_bandwidth_test

.azuredevops/components/rocm_smi_lib.yml

@@ -29,7 +29,7 @@ jobs:
 
 - job: rocm_smi_lib_testing
   dependsOn: rocm_smi_lib
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -44,6 +44,9 @@ jobs:
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocm_smi_lib

.azuredevops/components/rocminfo.yml

@@ -49,7 +49,7 @@ jobs:
 
 - job: rocminfo_testing
   dependsOn: rocminfo
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -72,6 +72,9 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocminfo

.azuredevops/components/rocprofiler-compute.yml

@@ -0,0 +1,175 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - locales
+    - python3-pip
+- name: pipModules
+  type: object
+  default:
+    - astunparse==1.6.2
+    - colorlover
+    - dash>=1.12.0
+    - matplotlib
+    - numpy>=1.17.5
+    - pandas>=1.4.3
+    - pymongo
+    - pyyaml
+    - tabulate
+    - tqdm
+    - dash-svg
+    - dash-bootstrap-components
+    - kaleido
+    - setuptools
+    - plotille
+    - mock
+    - pytest
+    - pytest-cov
+    - pytest-xdist
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler
+    - rocprofiler-register
+    - roctracer
+
+jobs:
+- job: rocprofiler_compute
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocprofiler_compute_testing
+  dependsOn: rocprofiler_compute
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: PYTHON_VERSION
+    value: 3.10
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: Bash@3
+    displayName: Add en_US.UTF-8 locale
+    inputs:
+      targetType: inline
+      script: |
+        sudo locale-gen en_US.UTF-8
+        sudo update-locale
+        locale -a
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -DENABLE_TESTS=ON
+        -DINSTALL_TESTS=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofiler-compute
+      testDir: $(Build.BinariesDirectory)/libexec/rocprofiler-compute
+      testExecutable: export ROCPROFCOMPUTE_ARCH_OVERRIDE="MI300X"; ctest
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"

.azuredevops/components/rocprofiler-sdk.yml

@@ -0,0 +1,93 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - python3-pip
+    - libdrm-dev
+    - libdw-dev
+    - libelf-dev
+- name: pipModules
+  type: object
+  default:
+    - black
+    - clang-format
+    - clang-tidy
+    - cmake
+    - cmake-format
+    - dataclasses
+    - numpy
+    - otf2
+    - pandas
+    - perfetto
+    - pycobertura
+    - pytest
+    - pyyaml
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rccl
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler-register
+    - roctracer
+
+jobs:
+- job: rocprofilersdk
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCPROFILER_BUILD_TESTS=ON
+        -DROCPROFILER_BUILD_SAMPLES=OFF
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j2
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocprofiler-systems.yml

@@ -0,0 +1,140 @@
+# largely referenced from: https://github.com/ROCm/omnitrace/blob/main/.github/workflows/ubuntu-jammy.yml
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - autoconf
+    - autotools-dev
+    - bison
+    - build-essential
+    - bzip2
+    - clang
+    - cmake
+    - environment-modules
+    - g++-12
+    - libdrm-dev
+    - libfabric-dev
+    - libiberty-dev
+    - libpapi-dev
+    - libpfm4-dev
+    - libtool
+    - libopenmpi-dev
+    - m4
+    - openmpi-bin
+    - software-properties-common
+    - python3-pip
+    - texinfo
+    - zlib1g-dev
+- name: pipModules
+  type: object
+  default:
+    - numpy
+    - perfetto
+    - dataclasses
+- name: rocmDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - llvm-project
+    - rccl
+    - rocm-core
+    - rocm_smi_lib
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler
+    - rocprofiler-register
+    - roctracer
+
+jobs:
+- job: rocprofiler_systems
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+# build flags reference: https://rocm.docs.amd.com/projects/omnitrace/en/latest/install/install.html
+      extraBuildFlags: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCPROFSYS_BUILD_TESTING=ON
+        -DROCPROFSYS_BUILD_DYNINST=ON
+        -DROCPROFSYS_BUILD_LIBUNWIND=ON
+        -DDYNINST_BUILD_TBB=ON
+        -DDYNINST_BUILD_ELFUTILS=ON
+        -DDYNINST_BUILD_LIBIBERTY=ON
+        -DDYNINST_BUILD_BOOST=ON
+        -DROCPROFSYS_USE_PAPI=ON
+        -DROCPROFSYS_USE_MPI=ON
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j32
+  - task: Bash@3
+    displayName: Set up rocprofiler-systems env
+    inputs:
+      targetType: inline
+      script: source share/rocprofiler-systems/setup-env.sh
+      workingDirectory: build
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofiler-systems
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocprofiler.yml

@@ -57,8 +57,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -92,6 +90,66 @@ jobs:
         -DENABLE_LDCONFIG=OFF
         -DUSE_PROF_API=1
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j32
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocprofiler_testing
+  dependsOn: rocprofiler
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofilerV1
+      testExecutable: LD_LIBRARY_PATH="$(Agent.BuildDirectory)/rocm/lib/rocprofiler:$(Agent.BuildDirectory)/rocm/share/rocprofiler/tests-v1/test" ./run.sh
+      testDir: $(Agent.BuildDirectory)/rocm/share/rocprofiler/tests-v1
+      testParameters: ''
+      testPublishResults: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofilerV2
+      testExecutable: LD_LIBRARY_PATH="$(Agent.BuildDirectory)/rocm/lib/rocprofiler:$(Agent.BuildDirectory)/rocm/share/rocprofiler/tests" share/rocprofiler/tests/runUnitTests
+      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: $(Agent.BuildDirectory)/rocm

.azuredevops/components/rocr_debug_agent.yml

@@ -72,7 +72,7 @@ jobs:
 
 - job: rocr_debug_agent_testing
   dependsOn: rocr_debug_agent
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -114,6 +114,7 @@ jobs:
       cmakeBuildDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'
       cmakeSourceDir: '.'
       installEnabled: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocr_debug_agent

.azuredevops/components/roctracer.yml

@@ -30,6 +30,7 @@ parameters:
   default:
     - clr
     - llvm-project
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
 
@@ -48,8 +49,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -77,6 +76,7 @@ jobs:
         -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
@@ -84,7 +84,7 @@ jobs:
 
 - job: roctracer_testing
   dependsOn: roctracer
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -118,6 +118,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: roctracer

.azuredevops/components/rpp.yml

@@ -9,18 +9,39 @@ parameters:
   type: object
   default:
     - cmake
-    - libomp-dev # needed to pass flag step
+    - libopencv-dev
+    - libsndfile1-dev
+    - libstdc++-12-dev
+    - imagemagick
     - ninja-build
     - clang
+- name: pipModules
+  type: object
+  default:
+    - openpyxl
+    - pandas
+    - sphinx
 - name: rocmDependencies
   type: object
   default:
-    - aomp # needed to pass build step
+    - aomp
     - clr
     - half
     - llvm-project
     - rocminfo
     - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - half
+    - hipTensor
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rpp
@@ -35,8 +56,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -69,3 +88,93 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rpp_testing
+  dependsOn: rpp
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: LD_LIBRARY_PATH
+    value: $(Agent.BuildDirectory)/rocm/lib;$(Agent.BuildDirectory)/rocm/llvm/lib
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  # Dependencies from: https://github.com/ROCm/rpp/blob/develop/utilities/test_suite/README.md
+  - task: Bash@3
+    displayName: Build and install Turbo JPEG
+    inputs:
+      targetType: 'inline'
+      script: |
+        sudo apt-get install nasm
+        sudo apt-get install wget
+        git clone -b 3.0.2 https://github.com/libjpeg-turbo/libjpeg-turbo.git 
+        cd libjpeg-turbo
+        mkdir build
+        cd build
+        cmake -DCMAKE_INSTALL_PREFIX=/usr \
+              -DCMAKE_BUILD_TYPE=RELEASE  \
+              -DENABLE_STATIC=FALSE       \
+              -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib  \
+              -DWITH_JPEG8=TRUE           \
+              ..
+        make -j$nproc
+        sudo make install
+  - task: Bash@3
+    displayName: Build and install Nifti
+    inputs:
+      targetType: 'inline'
+      script: |
+        git clone https://github.com/NIFTI-Imaging/nifti_clib.git
+        cd nifti_clib
+        mkdir build
+        cd build
+        cmake ..
+        sudo make -j$nproc install
+  - task: Bash@3
+    displayName: Build rpp tests
+    inputs:
+      targetType: 'inline'
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        mkdir rpp-tests
+        cd rpp-tests
+        cmake /opt/rocm/share/rpp/test \
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ \
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rpp
+      testExecutable: 'export PATH=$(Agent.BuildDirectory)/rocm/llvm/bin:$PATH; CC=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang CMAKE_VERBOSE_MAKEFILE=ON VERBOSE=1 ctest'
+      testDir: 'rpp-tests'
+  - script: sudo rm /opt/rocm

.azuredevops/dependencies/boost.yml

@@ -0,0 +1,65 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: boostVersion
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - git
+- name: rocmDependencies
+  type: object
+  default:
+    - llvm-project
+
+jobs:
+- job: boost
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: Bash@3
+    displayName: 'git clone boost'
+    inputs:
+      targetType: inline
+      script: git clone -b ${{ parameters.boostVersion }} https://github.com/boostorg/boost --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      dependencySource: staging
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - task: Bash@3
+    displayName: 'Build Boost with clang'
+    inputs:
+      targetType: inline
+      script: |
+        export CC=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
+        export CXX=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
+        ./bootstrap.sh --with-toolset=clang --prefix=$(Build.BinariesDirectory)
+        ./b2 --toolset=clang threading=multi link=shared --prefix=$(Build.BinariesDirectory) cxxflags="-std=c++20"
+        ./b2 install
+      workingDirectory: $(Build.SourcesDirectory)/boost
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/dependencies/grpc.yml

@@ -20,7 +20,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/dependencies/gtest.yml

@@ -20,7 +20,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/dependencies/lapack.yml

@@ -21,7 +21,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/nightly/pytorch.yml

@@ -11,7 +11,6 @@ parameters:
     - ca-certificates
     - bc
     - bridge-utils
-    - cmake
     - devscripts
     - dkms
     - doxygen
@@ -62,11 +61,14 @@ parameters:
     - ffmpeg
     - libopenblas-dev
     - liblapack-dev
+    - libswscale-dev
+    - libavformat-dev
 - name: pipModules
   type: object
   default:
+    - cmake
     - astunparse
-    - expecttest!=0.2.0
+    - expecttest>=0.2.1
     - hypothesis
     - numpy
     - psutil
@@ -83,11 +85,15 @@ parameters:
     - lintrunner
     - ninja
     - packaging
-    - optree>=0.12.0
+    - optree>=0.13.0
+    - click>=8.0.3
   # list for vision
     - auditwheel
     - future
-# list from https://github.com/pytorch/builder/blob/main/manywheel/build_rocm.sh
+    - pytest
+    - pytest-azurepipelines
+    - pillow
+# list from https://github.com/pytorch/pytorch/blob/main/.ci/manywheel/build_rocm.sh
 - name: rocmDependencies
   type: object
   default:
@@ -117,6 +123,25 @@ parameters:
     - hipCUB
     - rocThrust
     - hipBLAS-common
+    - composable_kernel
+- name: rocmTestDependencies
+  type: object
+  default:
+    - rocminfo
+# Reference on what tests to run for torchvision found in private repo:
+# https://github.com/ROCm/rocAutomation/blob/jenkins-pipelines/pytorch/pytorch_ci/test_pytorch_test1.sh#L54
+# Will iterate through this list using pytest
+- name: torchTestList
+  type: object
+  default:
+    - nn
+    - torch
+#    - cuda seg faults and might need cuda installed on test system
+#    - ops takes too long
+    - unary_ufuncs
+    - binary_ufuncs
+    - autograd
+#    - inductor/torchinductor takes too long
 
 trigger: none
 pr: none
@@ -136,13 +161,10 @@ jobs:
       amd-staging-gfx942:
         ROCM_BRANCH: amd-staging
         JOB_GPU_TARGET: gfx942
-      amd-staging-gfx90a:
-        ROCM_BRANCH: amd-staging
-        JOB_GPU_TARGET: gfx90a
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-# various flags/parameters expected by bash scripts in pytorch builder repo
+# various flags/parameters expected by bash scripts in pytorch repo's .ci directory
   - name: ROCM_VERSION
     value: 6.3.0
   - name: ROCM_PATH
@@ -163,7 +185,7 @@ jobs:
   workspace:
     clean: all
   steps:
-# copy environment setup from https://github.com/pytorch/builder/blob/main/manywheel/Dockerfile
+# copy environment setup from https://github.com/pytorch/pytorch/blob/main/.ci/docker/manywheel/Dockerfile
 # but instead of centos, use ubuntu environment
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
   - task: Bash@3
@@ -200,18 +222,21 @@ jobs:
       targetType: inline
       script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
   - checkout: self
-  - task: Bash@3
-    displayName: git clone pytorch builder
-    inputs:
-      targetType: inline
-      script: git clone https://github.com/pytorch/builder.git --depth=1 --recurse-submodules
-      workingDirectory: $(Build.SourcesDirectory)
   - task: Bash@3
     displayName: git clone upstream pytorch
     inputs:
       targetType: inline
       script: git clone https://github.com/pytorch/pytorch.git --depth=1 --recurse-submodules
       workingDirectory: $(Build.SourcesDirectory)
+# builder clone still needed due to run_tests.sh at end of build_common.sh call
+  - task: Bash@3
+    displayName: git clone pytorch builder
+    inputs:
+      targetType: inline
+      script: |
+        git clone https://github.com/pytorch/builder.git --depth=1 --recurse-submodules
+        sudo ln -s $(Build.SourcesDirectory)/builder /builder
+      workingDirectory: $(Build.SourcesDirectory)
   - task: Bash@3
     displayName: Install patchelf
     inputs:
@@ -264,8 +289,8 @@ jobs:
         PYTORCH_BUILD_VERSION=$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)
         PYTORCH_BUILD_NUMBER=$(date -u +%Y%m%d)
         SKIP_ALL_TESTS=1
-        bash ./manywheel/build_rocm.sh
-      workingDirectory: $(Build.SourcesDirectory)/builder
+        bash ./.ci/manywheel/build_rocm.sh
+      workingDirectory: $(Build.SourcesDirectory)/pytorch
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
       sourceDir: /remote/wheelhouserocm$(ROCM_VERSION)
@@ -300,7 +325,10 @@ jobs:
     inputs:
       targetType: inline
       script: >-
-        BUILD_VERSION=$(ROCM_BRANCH).$(JOB_GPU_TARGET)-$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)$(date -u +%Y%m%d)
+        TORCH_PACKAGE_NAME=torch.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
+        TORCHVISION_PACKAGE_NAME=torchvision.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
+        PYTORCH_VERSION=$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)post$(date -u +%Y%m%d)
+        BUILD_VERSION=$(cat $(Build.SourcesDirectory)/vision/version.txt | cut -da -f1)post$(date -u +%Y%m%d)
         python3 setup.py bdist_wheel
       workingDirectory: $(Build.SourcesDirectory)/vision
   - task: Bash@3
@@ -319,3 +347,191 @@ jobs:
     retryCountOnTaskFailure: 3
     inputs:
       targetPath: $(Build.BinariesDirectory)
+
+- job: torchvision_testing
+  dependsOn: pytorch
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: PYTORCH_TEST_WITH_ROCM
+    value: 1
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - task: Bash@3
+    displayName: 'Register libjpeg-turbo packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
+        sudo apt update
+        apt-cache show libjpeg-turbo-official | grep Version
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Wheel Files'
+    inputs:
+      itemPattern: '**/*$(JOB_GPU_TARGET)*.whl'
+      targetPath: $(Agent.BuildDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      dependencySource: staging
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      dependencySource: staging
+      skipLlvmSymlink: true
+# get sources to run test scripts
+  - task: Bash@3
+    displayName: git clone upstream pytorch
+    inputs:
+      targetType: inline
+      script: git clone https://github.com/pytorch/pytorch.git --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: git clone pytorch vision
+    inputs:
+      targetType: inline
+      script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Install Wheel Files
+    inputs:
+      targetType: inline
+      script: find . -name "*.whl" -exec pip install --no-index --find-links=. --no-dependencies -v {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Show Updated pip List
+    inputs:
+      targetType: inline
+      script: pip list -v
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add Python site-packages binaries to path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.prependpath]$USER_BASE/bin"
+  - task: Bash@3
+    displayName: Add torch libs to ldconfig
+    inputs:
+      targetType: inline
+      script: |
+        echo $(python3 -m site --user-site)/torch/lib | sudo tee /etc/ld.so.conf.d/torch.conf
+        sudo ldconfig -v
+        ldconfig -p
+# https://pytorch.org/get-started/locally/#linux-verification
+# https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/3rd-party/pytorch-install.html#testing-the-pytorch-installation
+  - task: Bash@3
+    displayName: Simple Import Torch Tests
+    inputs:
+      targetType: inline
+      script: |
+        python3 -c 'import torch' 2> /dev/null && echo 'Success' || echo 'Failure'
+        python3 -c 'import torch; print(torch.cuda.is_available())'
+        python3 -c 'import torch; x = torch.rand(5, 3); print(x)'
+# Test artifact build script has too many if statements for different environments
+# Based off the snippet of interest for this environment, with some adjustments
+# https://github.com/pytorch/pytorch/blob/main/.ci/pytorch/build.sh#L335-L375
+# Removing in-line comments since it does not fit with the yaml markup
+  - task: Bash@3
+    displayName: Build Pytorch Test Artifacts
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: |
+        CUSTOM_TEST_ARTIFACT_BUILD_DIR="build/custom_test_artifacts"
+        CUSTOM_TEST_USE_ROCM=ON
+        CUSTOM_TEST_MODULE_PATH="${PWD}/cmake/public"
+        mkdir -pv "${CUSTOM_TEST_ARTIFACT_BUILD_DIR}"
+
+        CUSTOM_OP_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-op-build"
+        CUSTOM_OP_TEST="${PWD}/test/custom_operator"
+        python --version
+        SITE_PACKAGES="$(python -c 'import site; print(";".join([x for x in site.getsitepackages()] + [x + "/torch" for x in site.getsitepackages()]))')"
+
+        mkdir -p "$CUSTOM_OP_BUILD"
+        pushd "$CUSTOM_OP_BUILD"
+        cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
+              -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
+        make VERBOSE=1
+        popd
+
+        JIT_HOOK_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/jit-hook-build"
+        JIT_HOOK_TEST="$PWD/test/jit_hooks"
+        python --version
+        SITE_PACKAGES="$(python -c 'import site; print(";".join([x for x in site.getsitepackages()] + [x + "/torch" for x in site.getsitepackages()]))')"
+        mkdir -p "$JIT_HOOK_BUILD"
+        pushd "$JIT_HOOK_BUILD"
+        cmake "$JIT_HOOK_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
+              -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
+        make VERBOSE=1
+        popd
+
+        CUSTOM_BACKEND_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-backend-build"
+        CUSTOM_BACKEND_TEST="${PWD}/test/custom_backend"
+        python --version
+        mkdir -p "$CUSTOM_BACKEND_BUILD"
+        pushd "$CUSTOM_BACKEND_BUILD"
+        cmake "$CUSTOM_BACKEND_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
+              -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
+        make VERBOSE=1
+        popd
+      workingDirectory: $(Build.SourcesDirectory)/pytorch
+  - ${{ each torchTest in parameters.torchTestList }}:
+    - task: Bash@3
+      displayName: Test ${{ torchTest }}
+      continueOnError: true
+      inputs:
+        targetType: inline
+        workingDirectory: $(Build.SourcesDirectory)/pytorch
+        ${{ if contains(torchTest, '/') }}:
+          script: pytest test/${{ split(torchTest, '/')[0] }}/test_${{ split(torchTest, '/')[1] }}.py
+        ${{ else }}:
+          script: pytest test/test_${{ torchTest }}.py
+# Reference on what tests to run for torchvision found in private repo:
+# https://github.com/ROCm/rocAutomation/blob/jenkins-pipelines/pytorch/pytorch_ci/test_torchvision.sh#L51
+  - task: Bash@3
+    displayName: Test vision/transforms
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: pytest test/test_transforms.py
+      workingDirectory: $(Build.SourcesDirectory)/vision
+  - task: Bash@3
+    displayName: Uninstall Wheel Files
+    inputs:
+      targetType: inline
+      script: find . -name "*.whl" -exec pip uninstall -y {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Remove Python site-packages binaries from path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$USER_BASE/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"

.azuredevops/nightly/rocm-nightly.yml

@@ -1,5 +1,5 @@
 parameters:
-# currently excludes clr and rocm-examples
+# currently excludes clr
 - name: rocmDependencies
   type: object
   default:
@@ -10,6 +10,7 @@ parameters:
     - composable_kernel
     - half
     - HIP
+    - hip-tests
     - hipBLAS
     - hipBLAS-common
     - hipBLASLt
@@ -36,14 +37,19 @@ parameters:
     - ROCgdb
     - rocm-cmake
     - rocm-core
+    - rocm-examples
     - rocminfo
     - rocMLIR
     - ROCmValidationSuite
     - rocm_bandwidth_test
     - rocm_smi_lib
     - rocPRIM
+    - rocprofiler-compute
     - rocprofiler-register
+    - rocprofiler-sdk
+    - rocprofiler-systems
     - rocprofiler
+    - rocPyDecode
     - ROCR-Runtime
     - rocRAND
     - rocr_debug_agent
@@ -76,8 +82,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - task: DeleteFiles@1
     displayName: 'Cleanup checkout space'
@@ -97,6 +101,7 @@ jobs:
       dependencyList: ${{ parameters.rocmDependencies }}
       dependencySource: staging
       skipLibraryLinking: true
+      skipLlvmSymlink: true
       gpuTarget: $(JOB_GPU_TARGET)
   - script: df -h
     displayName: System disk space after ROCm
@@ -113,7 +118,7 @@ jobs:
   - script: du -sh $(Build.ArtifactStagingDirectory)
     displayName: Compressed ROCm size
   - task: PublishPipelineArtifact@1
-    displayName: 'Public ROCm Nightly Artifact'
+    displayName: 'Publish ROCm Nightly Artifact'
     retryCountOnTaskFailure: 3
     inputs:
       targetPath: '$(Build.ArtifactStagingDirectory)'

.azuredevops/tag-builds/Tensile.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/Tensile
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/Tensile.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/boost.yml

@@ -0,0 +1,23 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: boostVersion
+  type: string
+  default: 'master'
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_DEPENDENCIES_PATH }}/boost.yml
+    parameters:
+      boostVersion: ${{ parameters.boostVersion }}

.azuredevops/tag-builds/omnitrace.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/omnitrace
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/omnitrace.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocprofiler-compute.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocprofiler-compute
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocprofiler-compute.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocprofiler-sdk.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocprofiler-sdk
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocprofiler-sdk.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocprofiler-systems.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocprofiler-systems
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocprofiler-systems.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/templates/steps/artifact-download.yml

@@ -9,6 +9,10 @@ parameters:
 - name: useDefaultBranch
   type: boolean
   default: true
+# useMainlineBranch only processed if useDefaultBranch is false
+- name: useMainlineBranch
+  type: boolean
+  default: false
 - name: latestFromBranch
   type: boolean
   default: true
@@ -22,13 +26,14 @@ parameters:
   type: object
   default:
     AMDMIGraphX: develop
-    amdsmi: develop
+    amdsmi: amd-staging
     aomp-extras: aomp-dev
     aomp: aomp-dev
     clr: amd-staging
     composable_kernel: develop
     half: rocm
     HIP: amd-staging
+    hip-tests: amd-staging
     hipBLAS: develop
     hipBLASLt: develop
     hipBLAS-common: develop
@@ -44,8 +49,10 @@ parameters:
     llvm-project: amd-staging
     MIOpen: develop
     MIVisionX: develop
+    omniperf: amd-staging
+    omnitrace: amd-staging
     rccl: develop
-    rdc: develop
+    rdc: amd-staging
     rocAL: develop
     rocALUTION: develop
     rocBLAS: develop
@@ -54,16 +61,20 @@ parameters:
     rocFFT: develop
     ROCgdb: amd-staging
     rocm-cmake: develop
-    rocm-core: master
+    rocm-core: amd-staging
     rocm-examples: develop
     rocminfo: amd-staging
     rocMLIR: develop
     ROCmValidationSuite: master
     rocm_bandwidth_test: master
-    rocm_smi_lib: develop
+    rocm_smi_lib: amd-staging
     rocPRIM: develop
-    rocprofiler-register: amd-mainline
     rocprofiler: amd-staging
+    rocprofiler-compute: amd-staging
+    rocprofiler-register: amd-staging
+    rocprofiler-sdk: amd-staging
+    rocprofiler-systems: amd-staging
+    rocPyDecode: develop
     ROCR-Runtime: amd-staging
     rocRAND: develop
     rocr_debug_agent: amd-staging
@@ -73,24 +84,84 @@ parameters:
     roctracer: amd-staging
     rocWMMA: develop
     rpp: develop
-- name: allowPartiallySucceededBuilds
+- name: mainlineBranchList
   type: object
   default:
-    - amdsmi
-    - aomp
-    - HIPIFY
-    - rccl
-    - rdc
-    - rocm-cmake
-    - rocm_smi_lib
-    - rocFFT
-    - MIVisionX
+    AMDMIGraphX: mainline
+    amdsmi: amd-mainline
+    aomp-extras: amd-mainline-open
+    aomp: amd-mainline-open
+    clr: amd-mainline
+    composable_kernel: mainline
+    half: rocm
+    HIP: amd-mainline
+    hip-tests: amd-mainline
+    hipBLAS: mainline
+    hipBLASLt: mainline
+    hipBLAS-common: mainline
+    hipCUB: mainline
+    hipFFT: mainline
+    hipfort: mainline
+    HIPIFY: amd-mainline
+    hipRAND: mainline
+    hipSOLVER: mainline
+    hipSPARSE: mainline
+    hipSPARSELt: mainline
+    hipTensor: mainline
+    llvm-project: amd-mainline-open
+    MIOpen: mainline
+    MIVisionX: mainline
+    omniperf: amd-mainline
+    omnitrace: amd-mainline
+    rccl: mainline
+    rdc: amd-mainline
+    rocAL: master # needs the yaml file
+    rocALUTION: mainline
+    rocBLAS: mainline
+    ROCdbgapi : amd-mainline
+    rocDecode: mainline
+    rocFFT: mainline
+    ROCgdb: amd-mainline-rocgdb-15 #
+    rocm-cmake: mainline
+    rocm-core: amd-master
+    rocm-examples: develop # no mainline
+    rocminfo: amd-master
+    rocMLIR: mainline # needs the yaml file
+    ROCmValidationSuite: mainline
+    rocm_bandwidth_test: master
+    rocm_smi_lib: amd-mainline
+    rocPRIM: mainline
+    rocprofiler: amd-master
+    rocprofiler-compute: amd-mainline
+    rocprofiler-register: amd-mainline
+    rocprofiler-sdk: amd-mainline
+    rocprofiler-systems: amd-mainline
+    rocPyDecode: mainline
+    ROCR-Runtime: amd-master
+    rocRAND: mainline
+    rocr_debug_agent: amd-mainline
+    rocSOLVER: mainline
+    rocSPARSE: mainline
+    rocThrust: mainline
+    roctracer: amd-master
+    rocWMMA: mainline
+    rpp: mainline
 # BELOW REQUIRED IF useDefaultBranch false
 - name: branchName
   type: string
   default: '$(Build.SourceBranchName)' # for tagged builds
 
 steps:
+- task: Bash@3
+  displayName: Set allowPartiallySucceededBuilds
+  inputs:
+    targetType: inline
+    script: |
+      if [[ ",$ALLOWED_PARTIAL_SUCCEED_BUILDS," == *",${{ parameters.componentName }},"* ]]; then
+        echo "##vso[task.setvariable variable=allowPartiallySucceededBuilds;]true"
+      else
+        echo "##vso[task.setvariable variable=allowPartiallySucceededBuilds;]false"
+      fi
 - task: DownloadPipelineArtifact@2
   displayName: Download ${{ parameters.componentName }}
   inputs:
@@ -104,10 +175,11 @@ steps:
         buildVersionToDownload: latestFromBranch # default is 'latest'
     ${{ if eq(parameters.useDefaultBranch, true) }}:
       branchName: refs/heads/${{ parameters.defaultBranchList[parameters.componentName] }}
+    ${{ elseif eq(parameters.useMainlineBranch, true) }}:
+      branchName: refs/heads/${{ parameters.mainlineBranchList[parameters.componentName] }}
     ${{ else }}:
       branchName: ${{ parameters.branchName }}
-    ${{ if containsValue(parameters.allowPartiallySucceededBuilds, parameters.componentName) }}:
-      allowPartiallySucceededBuilds: true
+    allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
     targetPath: '$(Pipeline.Workspace)/d'
 - task: ExtractFiles@1
   displayName: Extract ${{ parameters.componentName }}

.azuredevops/templates/steps/build-autotools.yml

@@ -11,6 +11,9 @@ parameters:
 - name: installDir
   type: string
   default: '$(Build.BinariesDirectory)'
+- name: makeCallPrefix
+  type: string
+  default: ''
 
 steps:
 - task: Bash@3
@@ -23,7 +26,7 @@ steps:
   displayName: '${{ parameters.componentName }} make'
   inputs:
     targetType: inline
-    script: make -j$(nproc)
+    script: ${{ parameters.makeCallPrefix }} make -j$(nproc)
     workingDirectory: ${{ parameters.buildDir }}
 - task: Bash@3
   displayName: '${{ parameters.componentName }} make install'

.azuredevops/templates/steps/dependencies-aqlprofile.yml

@@ -8,20 +8,22 @@ parameters:
 - name: repositoryUrl
   type: object
   default: 
-    staging: https://repo.radeon.com/rocm/apt/6.2/pool/main/h/hsa-amd-aqlprofile
-    tag-builds: https://repo.radeon.com/rocm/apt/6.2/pool/main/h/hsa-amd-aqlprofile
-- name: packageName
-  type: object
-  default:
-    staging: hsa-amd-aqlprofile_1.0.0.60200.60200-66~22.04_amd64.deb
-    tag-builds: hsa-amd-aqlprofile_1.0.0.60200.60200-66~22.04_amd64.deb
+    staging: https://repo.radeon.com/rocm/apt/latest/pool/main/h/hsa-amd-aqlprofile/ # end slash is important for curl!
+    tag-builds: https://repo.radeon.com/rocm/apt/$(TAGGED_RELEASE)/pool/main/h/hsa-amd-aqlprofile/
 
 steps:
+- task: Bash@3
+  displayName: Get aqlprofile package name
+  inputs:
+    targetType: inline
+    script: |
+      export packageName=$(curl -s ${{ parameters.repositoryUrl[parameters.dependencySource] }} | grep -oP "href=\"\K[^\"]*$(lsb_release -rs)[^\"]*\.deb")
+      echo "##vso[task.setvariable variable=packageName;isreadonly=true]$packageName"
 - task: Bash@3
   displayName: 'Download aqlprofile'
   inputs:
     targetType: inline
-    script: wget -nv ${{ parameters.repositoryUrl[parameters.dependencySource] }}/${{ parameters.packageName[parameters.dependencySource] }}
+    script: wget -nv ${{ parameters.repositoryUrl[parameters.dependencySource] }}$(packageName)
     workingDirectory: '$(Pipeline.Workspace)'
 - task: Bash@3
   displayName: 'Extract aqlprofile'
@@ -29,7 +31,7 @@ steps:
     targetType: inline
     script: |
       mkdir hsa-amd-aqlprofile
-      dpkg-deb -R ${{ parameters.packageName[parameters.dependencySource] }} hsa-amd-aqlprofile
+      dpkg-deb -R $(packageName) hsa-amd-aqlprofile
     workingDirectory: '$(Pipeline.Workspace)'
 - task: Bash@3
   displayName: 'Copy aqlprofile files'
@@ -43,5 +45,5 @@ steps:
   displayName: 'Clean up aqlprofile'
   inputs:
     targetType: inline
-    script: rm -rf hsa-amd-aqlprofile ${{ parameters.packageName[parameters.dependencySource] }}
+    script: rm -rf hsa-amd-aqlprofile $(packageName)
     workingDirectory: '$(Pipeline.Workspace)'

.azuredevops/templates/steps/dependencies-boost.yml

@@ -0,0 +1,35 @@
+steps:
+- task: DownloadPipelineArtifact@2
+  displayName: Download Boost
+  inputs:
+    buildType: specific
+    project: ROCm-CI
+    definition: $(BOOST_DEPENDENCY_PIPELINE_ID)
+    targetPath: $(Pipeline.Workspace)/d
+- task: ExtractFiles@1
+  displayName: Extract Boost
+  inputs:
+    archiveFilePatterns: '$(Pipeline.Workspace)/d/**/*.tar.gz'
+    destinationFolder: $(Agent.BuildDirectory)/boost
+    cleanDestinationFolder: true
+    overwriteExistingFiles: true
+- task: DeleteFiles@1
+  displayName: Cleanup Compressed Boost
+  inputs:
+    SourceFolder: $(Pipeline.Workspace)/d
+    Contents: '**/*.tar.gz'
+    RemoveDotFiles: true
+- task: Bash@3
+  displayName: 'List Boost files'
+  inputs:
+    targetType: inline
+    script: ls -1R $(Agent.BuildDirectory)/boost
+- task: Bash@3
+  displayName: 'Link Boost shared libraries'
+  inputs:
+    targetType: inline
+    script: |
+      echo $(Agent.BuildDirectory)/boost/lib | sudo tee /etc/ld.so.conf.d/boost.conf
+      sudo cat /etc/ld.so.conf.d/boost.conf
+      sudo ldconfig -v
+      ldconfig -p

.azuredevops/templates/steps/dependencies-other.yml

@@ -8,6 +8,12 @@ parameters:
   default: []
 
 steps:
+# firefox takes time to upgrade and is not needed for CI workloads, hold version
+- task: Bash@3
+  displayName: 'sudo apt-mark hold firefox'
+  inputs:
+    targetType: inline
+    script: sudo apt-mark hold firefox
 - task: Bash@3
   displayName: 'sudo apt-get update'
   inputs:
@@ -34,4 +40,4 @@ steps:
     displayName: 'pip install  ...'
     inputs:
       targetType: inline
-      script: pip install ${{ join(' ', parameters.pipModules) }}
+      script: pip install -v ${{ join(' ', parameters.pipModules) }}

.azuredevops/templates/steps/dependencies-rocm.yml

@@ -9,6 +9,7 @@ parameters:
   default: staging
   values:
     - staging
+    - mainline
     - tag-builds
     - fixed
 - name: extractToMnt
@@ -37,6 +38,7 @@ parameters:
     composable_kernel: $(COMPOSABLE_KERNEL_PIPELINE_ID)
     half: $(HALF_PIPELINE_ID)
     HIP: $(HIP_PIPELINE_ID)
+    hip-tests: $(HIP_TESTS_PIPELINE_ID)
     hipBLAS: $(HIPBLAS_PIPELINE_ID)
     hipBLAS-common: $(HIPBLAS_COMMON_PIPELINE_ID)
     hipBLASLt: $(HIPBLASLT_PIPELINE_ID)
@@ -52,6 +54,8 @@ parameters:
     llvm-project: $(LLVM_PROJECT_PIPELINE_ID)
     MIOpen: $(MIOpen_PIPELINE_ID)
     MIVisionX: $(MIVISIONX_PIPELINE_ID)
+    omniperf: $(OMNIPERF_PIPELINE_ID)
+    omnitrace: $(OMNITRACE_PIPELINE_ID)
     rccl: $(RCCL_PIPELINE_ID)
     rdc: $(RDC_PIPELINE_ID)
     rocAL: $(ROCAL_PIPELINE_ID)
@@ -70,8 +74,12 @@ parameters:
     rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_PIPELINE_ID)
     rocm_smi_lib: $(ROCM_SMI_LIB_PIPELINE_ID)
     rocPRIM: $(ROCPRIM_PIPELINE_ID)
+    rocprofiler-compute: $(ROCPROFILER_COMPUTE_PIPELINE_ID)
     rocprofiler-register: $(ROCPROFILER_REGISTER_PIPELINE_ID)
+    rocprofiler-sdk: $(ROCPROFILER_SDK_PIPELINE_ID)
+    rocprofiler-systems: $(ROCPROFILER_SYSTEMS_PIPELINE_ID)
     rocprofiler: $(ROCPROFILER_PIPELINE_ID)
+    rocPyDecode: $(ROCPYDECODE_PIPELINE_ID)
     ROCR-Runtime: $(ROCR_RUNTIME_PIPELINE_ID)
     rocRAND: $(ROCRAND_PIPELINE_ID)
     rocr_debug_agent: $(ROCR_DEBUG_AGENT_PIPELINE_ID)
@@ -93,6 +101,7 @@ parameters:
     composable_kernel: $(COMPOSABLE_KERNEL_TAGGED_PIPELINE_ID)
     half: $(HALF_TAGGED_PIPELINE_ID)
     HIP: $(HIP_TAGGED_PIPELINE_ID)
+    hip-tests: $(HIP_TESTS_TAGGED_PIPELINE_ID)
     hipBLAS: $(HIPBLAS_TAGGED_PIPELINE_ID)
     hipBLAS-common: $(HIPBLAS_COMMON_TAGGED_PIPELINE_ID)
     hipBLASLt: $(HIPBLASLT_TAGGED_PIPELINE_ID)
@@ -108,6 +117,8 @@ parameters:
     llvm-project: $(LLVM_PROJECT_TAGGED_PIPELINE_ID)
     MIOpen: $(MIOpen_TAGGED_PIPELINE_ID)
     MIVisionX: $(MIVISIONX_TAGGED_PIPELINE_ID)
+    omniperf: $(OMNIPERF_TAGGED_PIPELINE_ID)
+    omnitrace: $(OMNITRACE_TAGGED_PIPELINE_ID)
     rccl: $(RCCL_TAGGED_PIPELINE_ID)
     rdc: $(RDC_TAGGED_PIPELINE_ID)
     rocAL: $(ROCAL_TAGGED_PIPELINE_ID)
@@ -126,8 +137,12 @@ parameters:
     rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_TAGGED_PIPELINE_ID)
     rocm_smi_lib: $(ROCM_SMI_LIB_TAGGED_PIPELINE_ID)
     rocPRIM: $(ROCPRIM_TAGGED_PIPELINE_ID)
+    rocprofiler-compute: $(ROCPROFILER_COMPUTE_TAGGED_PIPELINE_ID)
     rocprofiler-register: $(ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID)
+    rocprofiler-sdk: $(ROCPROFILER_SDK_TAGGED_PIPELINE_ID)
+    rocprofiler-systems: $(ROCPROFILER_SYSTEMS_PIPELINE_ID)
     rocprofiler: $(ROCPROFILER_TAGGED_PIPELINE_ID)
+    rocPyDecode: $(ROCPYDECODE_TAGGED_PIPELINE_ID)
     ROCR-Runtime: $(ROCR_RUNTIME_TAGGED_PIPELINE_ID)
     rocRAND: $(ROCRAND_TAGGED_PIPELINE_ID)
     rocr_debug_agent: $(ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID)
@@ -166,13 +181,18 @@ parameters:
     - hipRAND
     - hipSPARSELt
     - hipTensor
+    - omnitrace
     - rccl
     - rocALUTION
     - rocBLAS
     - rocFFT
     - rocm-examples
     - rocPRIM
+    - rocprofiler-compute
+    - rocprofiler-sdk
+    - rocprofiler-systems
     - rocprofiler
+    - rocPyDecode
     - rocRAND
     - rocSOLVER
     - rocSPARSE
@@ -186,6 +206,7 @@ parameters:
     - hipSPARSE
     - MIOpen
     - MIVision
+    - omniperf
     - rocAL
     - ROCmValidationSuite
 
@@ -205,6 +226,10 @@ steps:
         ${{ if eq(parameters.dependencySource, 'staging') }}:
           pipelineId: ${{ parameters.stagingPipelineIdentifiers[ split(dependency, ':')[0] ] }}
           latestFromBranch: ${{ parameters.latestFromBranch }}
+        ${{ elseif eq(parameters.dependencySource, 'mainline') }}:
+          pipelineId: ${{ parameters.stagingPipelineIdentifiers[ split(dependency, ':')[0] ] }}
+          useMainlineBranch: true
+          latestFromBranch: ${{ parameters.latestFromBranch }}
         ${{ elseif eq(parameters.dependencySource, 'tag-builds') }}:
           pipelineId: ${{ parameters.taggedPipelineIdentifiers[ split(dependency, ':')[0] ] }}
           latestFromBranch: false
@@ -221,6 +246,10 @@ steps:
         ${{ if eq(parameters.dependencySource, 'staging') }}:
           pipelineId: ${{ parameters.stagingPipelineIdentifiers[dependency] }}
           latestFromBranch: ${{ parameters.latestFromBranch }}
+        ${{ elseif eq(parameters.dependencySource, 'mainline') }}:
+          pipelineId: ${{ parameters.stagingPipelineIdentifiers[dependency] }}
+          useMainlineBranch: true
+          latestFromBranch: ${{ parameters.latestFromBranch }}
         ${{ elseif eq(parameters.dependencySource, 'tag-builds') }}:
           pipelineId: ${{ parameters.taggedPipelineIdentifiers[dependency] }}
           latestFromBranch: false

.azuredevops/templates/steps/gpu-diagnostics.yml

@@ -0,0 +1,54 @@
+# Diagnostics for GPU-enabled systems
+parameters:
+- name: runRocminfo
+  type: boolean
+  default: true
+
+steps:
+- ${{ if eq(parameters.runRocminfo, true) }}:
+  - task: Bash@3
+    displayName: 'rocminfo'
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: $(Agent.BuildDirectory)/rocm/bin/rocminfo || true
+  - task: Bash@3
+    displayName: 'rocm_agent_enumerator'
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: $(Agent.BuildDirectory)/rocm/bin/rocm_agent_enumerator || true
+- task: Bash@3
+  displayName: 'List DRI devices'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: ls -la /dev/dri/ || true
+- task: Bash@3
+  displayName: 'List amdgpu/rocm/mesa packages'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: apt list --installed | grep -E 'amdgpu|rocm|mesa' || true
+- task: Bash@3
+  displayName: 'List GPU processes'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      ls /sys/class/kfd/kfd/proc/ || true
+      sudo lsof | grep amdgpu || true
+- task: Bash@3
+  displayName: 'System snapshot'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: top -bn1 || true
+- task: Bash@3
+  displayName: 'List dmesg'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      echo 'rocm-ci: $(Build.DefinitionName) $(System.DefinitionId)' | sudo tee /dev/kmsg || true
+      sudo dmesg || true

.azuredevops/variables-global.yml

@@ -79,6 +79,10 @@ variables:
   value: 93
 - name: HIP_TAGGED_PIPELINE_ID
   value: 31
+- name: HIP_TESTS_PIPELINE_ID
+  value: 233
+- name: HIP_TESTS_TAGGED_PIPELINE_ID
+  value: 220
 - name: HIPBLAS_COMMON_PIPELINE_ID
   value: 223
 - name: HIPBLAS_COMMON_TAGGED_PIPELINE_ID
@@ -159,6 +163,14 @@ variables:
   value: 80
 - name: MIVISIONX_TAGGED_PIPELINE_ID
   value: 18
+- name: OMNIPERF_PIPELINE_ID
+  value: 241
+- name: OMNIPERF_TAGGED_PIPELINE_ID
+  value: 242
+- name: OMNITRACE_PIPELINE_ID
+  value: 253
+- name: OMNITRACE_TAGGED_PIPELINE_ID
+  value: 252
 - name: RCCL_GFX942_TEST_PIPELINE_ID
   value: 184
 - name: RCCL_PIPELINE_ID
@@ -219,6 +231,10 @@ variables:
   value: 22
 - name: ROCM_EXAMPLES_GFX942_TEST_PIPELINE_ID
   value: 204
+- name: ROCM_EXAMPLES_PIPELINE_ID
+  value: 216
+- name: ROCM_EXAMPLES_TAGGED_PIPELINE_ID
+  value: 245
 - name: ROCM_SMI_LIB_PIPELINE_ID
   value: 96
 - name: ROCM_SMI_LIB_TAGGED_PIPELINE_ID
@@ -243,14 +259,30 @@ variables:
   value: 20
 - name: ROCPROFILER_GFX942_TEST_PIPELINE_ID
   value: 190
-- name: ROCPROFILER_PIPELINE_ID
-  value: 143
+- name: ROCPROFILER_COMPUTE_PIPELINE_ID
+  value: 257
+- name: ROCPROFILER_COMPUTE_TAGGED_PIPELINE_ID
+  value: 258
 - name: ROCPROFILER_REGISTER_PIPELINE_ID
   value: 1
 - name: ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID
   value: 25
+- name: ROCPROFILER_SDK_PIPELINE_ID
+  value: 246
+- name: ROCPROFILER_SDK_TAGGED_PIPELINE_ID
+  value: 234
+- name: ROCPROFILER_SYSTEMS_PIPELINE_ID
+  value: 255
+- name: ROCPROFILER_SYSTEMS_TAGGED_PIPELINE_ID
+  value: 254
+- name: ROCPROFILER_PIPELINE_ID
+  value: 143
 - name: ROCPROFILER_TAGGED_PIPELINE_ID
   value: 28
+- name: ROCPYDECODE_PIPELINE_ID
+  value: 239
+- name: ROCPYDECODE_TAGGED_PIPELINE_ID
+  value: 232
 - name: ROCR_DEBUG_AGENT_PIPELINE_ID
   value: 136
 - name: ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID
@@ -305,3 +337,5 @@ variables:
   value: 78
 - name: RPP_TAGGED_PIPELINE_ID
   value: 39
+- name: BOOST_DEPENDENCY_PIPELINE_ID
+  value: 250

.github/CODEOWNERS

@@ -5,3 +5,4 @@ docs/ @amd-aakash @jlgreathouse @samjwu @yhuiYH @ROCm/rocm-documentation
 *.rst @amd-aakash @jlgreathouse @samjwu @yhuiYH @ROCm/rocm-documentation
 # External CI
 /.azuredevops/ @ROCm/external-ci
+tools/rocm-build/ @ROCm/rocm-devops

.wordlist.txt

@@ -36,6 +36,7 @@ Bluefield
 Bootloader
 CCD
 CDNA
+CHTML
 CIFAR
 CLI
 CLion
@@ -70,6 +71,7 @@ Concretized
 Conda
 ConnectX
 CuPy
+Dashboarding
 DDR
 DF
 DGEMM
@@ -227,6 +229,7 @@ Mellanox's
 Meta's
 Miniconda
 MirroredStrategy
+Mixtral
 Multicore
 Multithreaded
 MyEnvironment
@@ -272,6 +275,8 @@ OpenMPI
 OpenSSL
 OpenVX
 OpenXLA
+Oversubscription
+PagedAttention
 PCC
 PCI
 PCIe
@@ -293,6 +298,7 @@ PowerShell
 PyPi
 PyTorch
 Qcycles
+Qwen
 RAII
 RAS
 RCCL
@@ -490,6 +496,7 @@ cuLIB
 cuRAND
 cuSOLVER
 cuSPARSE
+customizations
 cTDP
 dataset
 datasets
@@ -561,6 +568,7 @@ hipfort
 hipify
 hipsolver
 hipsparse
+hlist
 hotspotting
 hpc
 hpp
@@ -584,6 +592,7 @@ intra
 invariants
 invocating
 ipo
+jax
 kdb
 kfd
 latencies
@@ -604,6 +613,7 @@ migraphx
 miopen
 miopengemm
 mivisionx
+mjx
 mkdir
 mlirmiopen
 mtypes
@@ -619,6 +629,7 @@ openmp
 openssl
 optimizers
 os
+oversubscription
 pageable
 parallelization
 parameterization
@@ -698,6 +709,7 @@ rocsolver
 rocsparse
 rocthrust
 roctracer
+rst
 runtime
 runtimes
 sL

README.md

@@ -76,7 +76,7 @@ The Build time will reduce significantly if we limit the GPU Architecture/s agai
 
 mkdir -p ~/WORKSPACE/      # Or any folder name other than WORKSPACE
 cd ~/WORKSPACE/
-export ROCM_VERSION=6.2.1
+export ROCM_VERSION=6.2.4 # Or 6.2.0, 6.2.1, 6.2.2
 ~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.2.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
 ~/bin/repo sync
 
@@ -199,4 +199,4 @@ cmake --build build --target=doc
 ## Older ROCm releases
 
 For release information for older ROCm releases, refer to the
-[CHANGELOG](./CHANGELOG.md).
+[ROCm release history](https://rocm.docs.amd.com/en/latest/release/versions.html).

RELEASE.md

@@ -1,73 +1,69 @@
-# ROCm 6.2.1 release notes
+# ROCm 6.2.4 release notes
 
 The release notes provide a summary of notable changes since the previous ROCm release.
 
-- [Release highlights](release-highlights)
+- [Release highlights](#release-highlights)
 
-- [Operating system and hardware support changes](operating-system-and-hardware-support-changes)
+- [Operating system and hardware support changes](#operating-system-and-hardware-support-changes)
 
-- [ROCm components versioning](rocm-components)
+- [ROCm components versioning](#rocm-components)
 
-- [Detailed component changes](detailed-component-changes)
+- [Detailed component changes](#detailed-component-changes)
 
-- [ROCm known issues](rocm-known-issues)
+- [ROCm known issues](#rocm-known-issues)
 
-- [ROCm upcoming changes](rocm-upcoming-changes)
+- [ROCm upcoming changes](#rocm-upcoming-changes)
 
-The [Compatibility matrix](https://rocm.docs.amd.com/en/docs-6.2.1/compatibility/compatibility-matrix.html)
-provides the full list of supported hardware, operating systems, ecosystems, third-party components, and ROCm components for each ROCm release.
-
-Release notes for previous ROCm releases are available in earlier versions of the documentation.
-See the [ROCm documentation release history](https://rocm.docs.amd.com/en/latest/release/versions.html).
+```{note}
+If you’re using Radeon™ PRO or Radeon GPUs in a workstation setting with a
+display connected, continue to use ROCm 6.2.3. See the [Use ROCm on Radeon
+GPUs](https://rocm.docs.amd.com/projects/radeon/en/latest/index.html)
+documentation to verify compatibility and system requirements.
+```
 
 ## Release highlights
 
-The following are notable new features and improvements in ROCm 6.2.1. For changes to individual components, see [Detailed component changes](#detailed-component-changes).
+The following are notable new features and improvements in ROCm 6.2.4. For changes to individual components, see
+[Detailed component changes](#detailed-component-changes).
 
-### rocAL major version change
+#### ROCm documentation updates
 
-The new version of rocAL introduces many new features, but does not modify any of the existing public API functions. However, the version number was incremented from 1.3 to 2.0.
-Applications linked to version 1.3 must be recompiled to link against version 2.0.
+ROCm documentation continues to be updated to provide clearer and more comprehensive guidance for
+a wider variety of user needs and use cases.
 
-See [the rocAL detailed changes](#rocal-2-0-0) for more information.
+* Added a new GPU cluster networking guide. See
+  [Cluster network performance validation for AMD Instinct accelerators](https://rocm.docs.amd.com/projects/gpu-cluster-networking/en/docs-6.2.4/index.html).
+  This documentation provides guidelines on validating network configurations
+  in single-node and multi-node environments to attain optimal speed and bandwidth
+  in AMD Instinct-powered clusters.
 
-### New support for FBGEMM (Facebook General Matrix Multiplication)
+* Updated the HIP runtime documentation.
 
-As of ROCm 6.2.1, ROCm supports Facebook General Matrix Multiplication (FBGEMM) and the related FBGEMM_GPU library. 
+  * Added a new section on how to use [HIP graphs](https://rocm.docs.amd.com/projects/HIP/en/docs-6.2.4/how-to/hipgraph.html).
 
-FBGEMM is a low-precision, high-performance CPU kernel library for convolution and matrix multiplication. It is used for server-side inference and as a back end for PyTorch quantized operators. FBGEMM_GPU includes a collection of PyTorch GPU operator libraries for training and inference. For more information, see the ROCm [Model acceleration libraries guide](https://rocm.docs.amd.com/en/6.2.1/how-to/llm-fine-tuning-optimization/model-acceleration-libraries.html)
-and [PyTorch's FBGEMM GitHub repository](https://github.com/pytorch/FBGEMM).
+  * Added a new section about the [Stream ordered memory allocator (SOMA)](https://rocm.docs.amd.com/projects/HIP/en/docs-6.2.4/how-to/stream_ordered_allocator.html).
 
-### ROCm Offline Installer Creator changes
+  * Updated the [Porting CUDA driver API](https://rocm.docs.amd.com/projects/HIP/en/docs-6.2.4/how-to/hip_porting_driver_api.html) section.
 
-The [ROCm Offline Installer Creator 6.2.1](https://rocm.docs.amd.com/projects/install-on-linux/en/6.2.1/install/rocm-offline-installer.html) introduces several new features and improvements including:
+* Updated the [Post-installation instructions](https://rocm.docs.amd.com/projects/install-on-linux/en/docs-6.2.4/install/post-install.html)
+  with guidance on using the `update-alternatives` utility and environment modules to help you manage multiple ROCm
+  versions and streamline PATH configuration.
 
-* Logging support for create and install logs
-* More stringent checks for Linux versions and distributions
-* Updated prerequisite repositories
-* Fixed CTest issues
-
-### ROCm documentation changes 
-
-There have been no changes to supported hardware or operating systems from ROCm 6.2.0 to ROCm 6.2.1.
-
-* The Programming Model Reference and Understanding the Programming Model topics in HIP have been consolidated into one topic,
-[HIP programming model (conceptual)](https://rocm.docs.amd.com/projects/HIP/en/6.2.1/understand/programming_model.html). 
-* The [HIP virtual memory management](https://rocm.docs.amd.com/projects/HIP/en/6.2.1/how-to/virtual_memory.html) and [HIP virtual memory management API](https://rocm.docs.amd.com/projects/HIP/en/6.2.1/reference/virtual_memory_reference.html) topics have been added.
-
-```{note}
-The ROCm documentation, like all ROCm projects, is open source and available on GitHub. To contribute to ROCm documentation, see the [ROCm documentation contribution guidelines](https://rocm.docs.amd.com/en/latest/contribute/contributing.html).
-```
+* Updated the [LLM inference performance validation on AMD Instinct
+  MI300X](https://rocm.docs.amd.com/en/docs-6.2.4/how-to/performance-validation/mi300x/vllm-benchmark.html)
+  documentation with more detailed guidance, new models, and the `float8` data type.
 
 ## Operating system and hardware support changes
 
-There are no changes to supported hardware or operating systems from ROCm 6.2.0 to ROCm 6.2.1.
+ROCm 6.2.4 adds support for the [AMD Radeon PRO V710](https://www.amd.com/en/products/accelerators/radeon-pro/amd-radeon-pro-v710.html) GPU for compute workloads. See 
+[Supported GPUs](https://rocm.docs.amd.com/projects/install-on-linux/en/docs-6.2.4/reference/system-requirements.html#supported-gpus)
+for more information.
 
-See the [Compatibility matrix](https://rocm.docs.amd.com/en/docs-6.2.1/compatibility/compatibility-matrix.html) for the full list of supported operating systems and hardware architectures.
+This release maintains the same operating system support as 6.2.2.
 
 ## ROCm components
 
-The following table lists the versions of ROCm components for ROCm 6.2.1, including any version changes from 6.2.0 to 6.2.1. 
+The following table lists the versions of ROCm components for ROCm 6.2.4, including any version changes from 6.2.2 to 6.2.4.
 
 Click the component's updated version to go to a detailed list of its changes. Click <i class="fab fa-github fa-lg"></i> to go to the component's source code on GitHub.
 
@@ -90,51 +86,51 @@ Click the component's updated version to go to a detailed list of its changes. C
             <tr>
                 <th rowspan="8">Libraries</th>
                 <th rowspan="8">Machine learning and computer vision</th>
-                <td><a href="https://rocm.docs.amd.com/projects/composable_kernel/en/docs-6.2.1">Composable Kernel</a>
+                <td><a href="https://rocm.docs.amd.com/projects/composable_kernel/en/docs-6.2.4">Composable Kernel</a>
                 </td>
                 <td>1.1.0</td>
-                <td><a href="https://github.com/ROCm/composable_kernel/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/composable_kernel/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/AMDMIGraphX/en/docs-6.2.1">MIGraphX</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/AMDMIGraphX/en/docs-6.2.4">MIGraphX</a></td>
                 <td>2.10</td>
-                <td><a href="https://github.com/ROCm/AMDMIGraphX/releases/tag/rocm-6.2.1"><i class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://github.com/ROCm/AMDMIGraphX/releases/"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/MIOpen/en/docs-6.2.1">MIOpen</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/MIOpen/en/docs-6.2.4">MIOpen</a></td>
                 <td>3.2.0</td>
-                <td><a href="https://github.com/ROCm/MIOpen/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/MIOpen/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/MIVisionX/en/docs-6.2.1">MIVisionX</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/MIVisionX/en/docs-6.2.4">MIVisionX</a></td>
                 <td>3.0.0</td>
-                <td><a href="https://github.com/ROCm/MIVisionX/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/MIVisionX/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocAL/en/docs-6.2.1">rocAL</a></td>
-                <td>1.0.0&nbsp;&Rightarrow;&nbsp;<a href="#rocal-2-0-0">2.0.0</a></td>
-                <td><a href="https://github.com/ROCm/rocAL/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/rocAL/en/docs-6.2.4">rocAL</a></td>
+                <td>2.0.0</td>
+                <td><a href="https://github.com/ROCm/rocAL/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocDecode/en/docs-6.2.1">rocDecode</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocDecode/en/docs-6.2.4">rocDecode</a></td>
                 <td>0.6.0</td>
-                <td><a href="https://github.com/ROCm/rocDecode/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/rocDecode/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocPyDecode/en/docs-6.2.1">rocPyDecode</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocPyDecode/en/docs-6.2.4">rocPyDecode</a></td>
                 <td>0.1.0</td>
-                <td><a href="https://github.com/ROCm/rocPyDecode/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/rocPyDecode/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rpp/en/docs-6.2.1">RPP</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rpp/en/docs-6.2.4">RPP</a></td>
                 <td>1.8.0</td>
-                <td><a href="https://github.com/ROCm/rpp/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/rpp/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
@@ -142,9 +138,9 @@ Click the component's updated version to go to a detailed list of its changes. C
             <tr>
                 <th rowspan="1"></th>
                 <th rowspan="1">Communication</th>
-                <td><a href="https://rocm.docs.amd.com/projects/rccl/en/docs-6.2.1">RCCL</a></td>
-                <td>2.20.5&nbsp;&Rightarrow;&nbsp;<a href="#rccl-2-20-5">2.20.5</a></td>
-                <td><a href="https://github.com/ROCm/rccl/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/rccl/en/docs-6.2.4">RCCL</a></td>
+                <td>2.20.5</td>
+                <td><a href="https://github.com/ROCm/rccl/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
@@ -152,99 +148,99 @@ Click the component's updated version to go to a detailed list of its changes. C
             <tr>
                 <th rowspan="16"></th>
                 <th rowspan="16">Math</th>
-                <td><a href="https://rocm.docs.amd.com/projects/hipBLAS/en/docs-6.2.1">hipBLAS</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipBLAS/en/docs-6.2.4">hipBLAS</a></td>
                 <td>2.2.0</td>
-                <td><a href="https://github.com/ROCm/hipBLAS/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/hipBLAS/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipBLASLt/en/docs-6.2.1">hipBLASLt</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipBLASLt/en/docs-6.2.4">hipBLASLt</a></td>
                 <td>0.8.0</td>
-                <td><a href="https://github.com/ROCm/hipBLASLt/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/hipBLASLt/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipFFT/en/docs-6.2.1">hipFFT</a></td>
-                <td>1.0.15</td>
-                <td><a href="https://github.com/ROCm/hipFFT/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/hipFFT/en/docs-6.2.4">hipFFT</a></td>
+                <td>1.0.15&nbsp;&Rightarrow;&nbsp;<a href="https://github.com/ROCm/hipFFT/blob/docs/6.2.4/CHANGELOG.md">1.0.16</a></td>
+                <td><a href="https://github.com/ROCm/hipFFT/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipfort/en/docs-6.2.1">hipfort</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipfort/en/docs-6.2.4">hipfort</a></td>
                 <td>0.4.0</td>
-                <td><a href="https://github.com/ROCm/hipfort/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/hipfort/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipRAND/en/docs-6.2.1">hipRAND</a></td>
-                <td>2.11.0</td>
-                <td><a href="https://github.com/ROCm/hipRAND/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/hipRAND/en/docs-6.2.4">hipRAND</a></td>
+                <td>2.11.0&nbsp;&Rightarrow;&nbsp;<a href="https://github.com/ROCm/hipRAND/blob/docs/6.2.4/CHANGELOG.md">2.11.1</a></td>
+                <td><a href="https://github.com/ROCm/hipRAND/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipSOLVER/en/docs-6.2.1">hipSOLVER</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipSOLVER/en/docs-6.2.4">hipSOLVER</a></td>
                 <td>2.2.0</td>
-                <td><a href="https://github.com/ROCm/hipSOLVER/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/hipSOLVER/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipSPARSE/en/docs-6.2.1">hipSPARSE</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipSPARSE/en/docs-6.2.4">hipSPARSE</a></td>
                 <td>3.1.1</td>
-                <td><a href="https://github.com/ROCm/hipSPARSE/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/hipSPARSE/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipSPARSELt/en/docs-6.2.1">hipSPARSELt</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipSPARSELt/en/docs-6.2.4">hipSPARSELt</a></td>
                 <td>0.2.1</td>
-                <td><a href="https://github.com/ROCm/hipSPARSELt/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/hipSPARSELt/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocALUTION/en/docs-6.2.1">rocALUTION</a></td>
-                <td>3.2.0</td>
-                <td><a href="https://github.com/ROCm/rocALUTION/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/rocALUTION/en/docs-6.2.4">rocALUTION</a></td>
+                <td>3.2.0&nbsp;&Rightarrow;&nbsp;<a href="https://github.com/ROCm/rocALUTION/blob/docs/6.2.4/CHANGELOG.md">3.2.1</a></td>
+                <td><a href="https://github.com/ROCm/rocALUTION/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocBLAS/en/docs-6.2.1">rocBLAS</a></td>
-                <td>4.1.2&nbsp;&Rightarrow;&nbsp;<a href="#rocblas-4-2-1">4.2.1</a></td>
-                <td><a href="https://github.com/ROCm/rocBLAS/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/rocBLAS/en/docs-6.2.4">rocBLAS</a></td>
+                <td>4.2.1&nbsp;&Rightarrow;&nbsp;<a href="https://github.com/ROCm/rocBLAS/blob/docs/6.2.4/CHANGELOG.md">4.2.4</a></td>
+                <td><a href="https://github.com/ROCm/rocBLAS/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocFFT/en/docs-6.2.1">rocFFT</a></td>
-                <td>1.0.28&nbsp;&Rightarrow;&nbsp;<a href="#rocfft-1-0-29">1.0.29</a></td>
-                <td><a href="https://github.com/ROCm/rocFFT/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/rocFFT/en/docs-6.2.4">rocFFT</a></td>
+                <td>1.0.29&nbsp;&Rightarrow;&nbsp;<a href="#rocfft-1-0-30">1.0.30</a></td>
+                <td><a href="https://github.com/ROCm/rocFFT/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocRAND/en/docs-6.2.1">rocRAND</a></td>
-                <td>3.1.0</td>
-                <td><a href="https://github.com/ROCm/rocRAND/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/rocRAND/en/docs-6.2.4">rocRAND</a></td>
+                <td>3.1.0&nbsp;&Rightarrow;&nbsp;<a href="https://github.com/ROCm/rocRAND/blob/docs/6.2.4/CHANGELOG.md">3.1.1</a></td>
+                <td><a href="https://github.com/ROCm/rocRAND/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocSOLVER/en/docs-6.2.1">rocSOLVER</a></td>
-                <td>3.26.0</td>
-                <td><a href="https://github.com/ROCm/rocSOLVER/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/rocSOLVER/en/docs-6.2.4">rocSOLVER</a></td>
+                <td>3.26.0&nbsp;&Rightarrow;&nbsp;<a href="#rocsolver-3-26-2">3.26.2</a></td>
+                <td><a href="https://github.com/ROCm/rocSOLVER/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocSPARSE/en/docs-6.2.1">rocSPARSE</a></td>
-                <td>3.2.0</td>
-                <td><a href="https://github.com/ROCm/rocSPARSE/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/rocSPARSE/en/docs-6.2.4">rocSPARSE</a></td>
+                <td>3.2.0&nbsp;&Rightarrow;&nbsp;<a href="https://github.com/ROCm/rocSPARSE/blob/docs/6.2.4/CHANGELOG.md">3.2.1</a></td>
+                <td><a href="https://github.com/ROCm/rocSPARSE/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocWMMA/en/docs-6.2.1">rocWMMA</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocWMMA/en/docs-6.2.4">rocWMMA</a></td>
                 <td>1.5.0</td>
-                <td><a href="https://github.com/ROCm/rocWMMA/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/rocWMMA/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
                 <td><a href="https://github.com/ROCm/Tensile">Tensile</a></td>
                 <td>4.41.0</td>
-                <td><a href="https://github.com/ROCm/tensile/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/tensile/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
@@ -252,27 +248,27 @@ Click the component's updated version to go to a detailed list of its changes. C
             <tr>
                 <th rowspan="4"></th>
                 <th rowspan="4">Primitives</th>
-                <td><a href="https://rocm.docs.amd.com/projects/hipCUB/en/docs-6.2.1">hipCUB</a></td>
-                <td>3.2.0</td>
-                <td><a href="https://github.com/ROCm/hipCUB/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/hipCUB/en/docs-6.2.4">hipCUB</a></td>
+                <td>3.2.0&nbsp;&Rightarrow;&nbsp;<a href="https://github.com/ROCm/hipCUB/blob/docs/6.2.4/CHANGELOG.md">3.2.1</a></td>
+                <td><a href="https://github.com/ROCm/hipCUB/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipTensor/en/docs-6.2.1">hipTensor</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipTensor/en/docs-6.2.4">hipTensor</a></td>
                 <td>1.3.0</td>
-                <td><a href="https://github.com/ROCm/hipTensor/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/hipTensor/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocPRIM/en/docs-6.2.1">rocPRIM</a></td>
-                <td>3.2.0&nbsp;&Rightarrow;&nbsp;<a href="#rocprim-3-2-1">3.2.1</a></td>
-                <td><a href="https://github.com/ROCm/rocPRIM/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/rocPRIM/en/docs-6.2.4">rocPRIM</a></td>
+                <td>3.2.1&nbsp;&Rightarrow;&nbsp;<a href="https://github.com/ROCm/rocPRIM/blob/docs/6.2.4/CHANGELOG.md">3.2.2</a></td>
+                <td><a href="https://github.com/ROCm/rocPRIM/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocThrust/en/docs-6.2.1">rocThrust</a></td>
-                <td>3.1.0</td>
-                <td><a href="https://github.com/ROCm/rocThrust/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/rocThrust/en/docs-6.2.4">rocThrust</a></td>
+                <td>3.1.0&nbsp;&Rightarrow;&nbsp;<a href="https://github.com/ROCm/rocThrust/blob/docs/6.2.4/CHANGELOG.md">3.1.1</a></td>
+                <td><a href="https://github.com/ROCm/rocThrust/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
@@ -280,33 +276,33 @@ Click the component's updated version to go to a detailed list of its changes. C
             <tr>
                 <th rowspan="6">Tools</th>
                 <th rowspan="6">System management</th>
-                <td><a href="https://rocm.docs.amd.com/projects/amdsmi/en/docs-6.2.1">AMD SMI</a></td>
-                <td>24.6.2&nbsp;&Rightarrow;&nbsp;<a href="#amd-smi-24-6-3">24.6.3</a></td>
-                <td><a href="https://github.com/ROCm/amdsmi/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/amdsmi/en/docs-6.2.4">AMD SMI</a></td>
+                <td>24.6.3&nbsp;&Rightarrow;&nbsp;<a href="#amd-smi-24-6-3">24.6.3</a></td>
+                <td><a href="https://github.com/ROCm/amdsmi/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocminfo/en/docs-6.2.1">rocminfo</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocminfo/en/docs-6.2.4">rocminfo</a></td>
                 <td>1.0.0</td>
-                <td><a href="https://github.com/ROCm/rocminfo/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/rocminfo/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rdc/en/docs-6.2.1">ROCm Data Center Tool</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rdc/en/docs-6.2.4">ROCm Data Center Tool</a></td>
+                <td>0.3.0</td>
+                <td><a href="https://github.com/ROCm/rdc/releases/"><i
+                            class="fab fa-github fa-lg"></i></a></td>
+            </tr>
+            <tr>
+                <td><a href="https://rocm.docs.amd.com/projects/rocm_smi_lib/en/docs-6.2.4">ROCm SMI</a></td>
+                <td>7.3.0</td>
+                <td><a href="https://github.com/ROCm/rocm_smi_lib/releases/"><i
+                            class="fab fa-github fa-lg"></i></a></td>
+            </tr>
+            <tr>
+                <td><a href="https://rocm.docs.amd.com/projects/ROCmValidationSuite/en/docs-6.2.4">ROCm Validation Suite</a></td>
                 <td>1.0.0</td>
-                <td><a href="https://github.com/ROCm/rdc/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
-            </tr>
-            <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocm_smi_lib/en/docs-6.2.1">ROCm SMI</a></td>
-                <td>7.3.0&nbsp;&Rightarrow;&nbsp;<a href="#rocm-smi-7-3-0">7.3.0</a></td>
-                <td><a href="https://github.com/ROCm/rocm_smi_lib/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
-            </tr>
-            <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/ROCmValidationSuite/en/docs-6.2.1">ROCm Validation Suite</a></td>
-                <td>1.0.0</td>
-                <td><a href="https://github.com/ROCm/ROCmValidationSuite/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/ROCmValidationSuite/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
@@ -314,40 +310,40 @@ Click the component's updated version to go to a detailed list of its changes. C
             <tr>
                 <th rowspan="6"></th>
                 <th rowspan="6">Performance</th>
-                <td><a href="https://rocm.docs.amd.com/projects/omniperf/en/docs-6.2.1">Omniperf</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/omniperf/en/docs-6.2.4">Omniperf</a></td>
                 <td>2.0.1</td>
-                <td><a href="https://github.com/ROCm/omniperf/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/omniperf/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/omnitrace/en/docs-6.2.1">Omnitrace</a></td>
-                <td>1.11.2&nbsp;&Rightarrow;&nbsp;<a href="#omnitrace-1-11-2">1.11.2</a></td>
-                <td><a href="https://github.com/ROCm/omnitrace/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/omnitrace/en/docs-6.2.4">Omnitrace</a></td>
+                <td>1.11.2</td>
+                <td><a href="https://github.com/ROCm/omnitrace/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocm_bandwidth_test/en/docs-6.2.1">ROCm Bandwidth
+                <td><a href="https://rocm.docs.amd.com/projects/rocm_bandwidth_test/en/docs-6.2.4">ROCm Bandwidth
                         Test</a></td>
                 <td>1.4.0</td>
-                <td><a href="https://github.com/ROCm/rocm_bandwidth_test/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/rocm_bandwidth_test/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocprofiler/en/docs-6.2.1/">ROCProfiler</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocprofiler/en/docs-6.2.4/">ROCProfiler</a></td>
                 <td>2.0.0</td>
-                <td><a href="https://github.com/ROCm/ROCProfiler/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/ROCProfiler/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocprofiler-sdk/en/docs-6.2.1">ROCprofiler-SDK</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocprofiler-sdk/en/docs-6.2.4">ROCprofiler-SDK</a></td>
                 <td>0.4.0</td>
-                <td><a href="https://github.com/ROCm/rocprofiler-sdk/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/rocprofiler-sdk/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr >
-                <td><a href="https://rocm.docs.amd.com/projects/roctracer/en/docs-6.2.1/">ROCTracer</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/roctracer/en/docs-6.2.4/">ROCTracer</a></td>
                 <td>4.1.0</td>
-                <td><a href="https://github.com/ROCm/ROCTracer/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/ROCTracer/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
@@ -355,65 +351,65 @@ Click the component's updated version to go to a detailed list of its changes. C
             <tr>
                 <th rowspan="5"></th>
                 <th rowspan="5">Development</th>
-                <td><a href="https://rocm.docs.amd.com/projects/HIPIFY/en/docs-6.2.1/">HIPIFY</a></td>
-                <td>18.0.0&nbsp;&Rightarrow;&nbsp;<a href="#hipify-18-0-0">18.0.0</a></td>
-                <td><a href="https://github.com/ROCm/HIPIFY/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/HIPIFY/en/docs-6.2.4/">HIPIFY</a></td>
+                <td>18.0.0</td>
+                <td><a href="https://github.com/ROCm/HIPIFY/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/ROCdbgapi/en/docs-6.2.1">ROCdbgapi</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/ROCdbgapi/en/docs-6.2.4">ROCdbgapi</a></td>
                 <td>0.76.0</td>
-                <td><a href="https://github.com/ROCm/ROCdbgapi/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/ROCdbgapi/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/ROCmCMakeBuildTools/en/docs-6.2.1/">ROCm CMake</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/ROCmCMakeBuildTools/en/docs-6.2.4/">ROCm CMake</a></td>
                 <td>0.13.0</td>
-                <td><a href="https://github.com/ROCm/rocm-cmake/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/rocm-cmake/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/ROCgdb/en/docs-6.2.1">ROCm Debugger (ROCgdb)</a>
+                <td><a href="https://rocm.docs.amd.com/projects/ROCgdb/en/docs-6.2.4">ROCm Debugger (ROCgdb)</a>
                 </td>
                 <td>14.2</td>
-                <td><a href="https://github.com/ROCm/ROCgdb/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/ROCgdb/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocr_debug_agent/en/docs-6.2.1">ROCr Debug Agent</a>
+                <td><a href="https://rocm.docs.amd.com/projects/rocr_debug_agent/en/docs-6.2.4">ROCr Debug Agent</a>
                 </td>
                 <td>2.0.3</td>
-                <td><a href="https://github.com/ROCm/rocr_debug_agent/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/rocr_debug_agent/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
         <tbody class="rocm-components-compilers tbody-reverse-zebra">
             <tr>
                 <th rowspan="2" colspan="2">Compilers</th>
-                <td><a href="https://rocm.docs.amd.com/projects/HIPCC/en/docs-6.2.1">HIPCC</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/HIPCC/en/docs-6.2.4">HIPCC</a></td>
                 <td>1.1.1</td>
-                <td><a href="https://github.com/ROCm/llvm-project/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/llvm-project/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/llvm-project/en/docs-6.2.1">llvm-project</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/llvm-project/en/docs-6.2.4">llvm-project</a></td>
                 <td>18.0.0</td>
-                <td><a href="https://github.com/ROCm/llvm-project/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/llvm-project/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
         <tbody class="rocm-components-runtimes tbody-reverse-zebra">
             <tr>
                 <th rowspan="2" colspan="2">Runtimes</th>
-                <td><a href="https://rocm.docs.amd.com/projects/HIP/en/docs-6.2.1">HIP</a></td>
-                <td>6.2&nbsp;&Rightarrow;&nbsp;<a href="#hip-6-2-1">6.2.1</a></td>
-                <td><a href="https://github.com/ROCm/HIP/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/HIP/en/docs-6.2.4">HIP</a></td>
+                <td>6.2.4</a></td>
+                <td><a href="https://github.com/ROCm/HIP/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/ROCR-Runtime/en/docs-6.2.1">ROCr Runtime</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/ROCR-Runtime/en/docs-6.2.4">ROCr Runtime</a></td>
                 <td>1.14.0</td>
-                <td><a href="https://github.com/ROCm/ROCR-Runtime/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/ROCR-Runtime/releases/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
@@ -426,139 +422,34 @@ The following sections describe key changes to ROCm components.
 
 ### **AMD SMI** (24.6.3)
 
-#### Changes
+#### Resolved issues
 
-* Added `amd-smi static --ras` on Guest VMs. Guest VMs can view enabled/disabled RAS features on Host cards.
+* Fixed support for the API calls `amdsmi_get_gpu_process_isolation` and
+  `amdsmi_clean_gpu_local_data`, along with the `amd-smi set
+  --process-isolation <0 or 1>` command. See issue
+  [#3500](https://github.com/ROCm/ROCm/issues/3500) on GitHub.
 
-#### Removals
+### **rocFFT** (1.0.30)
 
-* Removed `amd-smi metric --ecc` & `amd-smi metric --ecc-blocks` on Guest VMs. Guest VMs do not support getting current ECC counts from the Host cards.
+#### Optimized
+
+* Implemented 1D kernels for factorizable sizes greater than 1024 and less than 2048.
 
 #### Resolved issues
 
-* Fixed TypeError in `amd-smi process -G`.
-* Updated CLI error strings to handle empty and invalid GPU/CPU inputs.
-* Fixed Guest VM showing passthrough options.
-* Fixed firmware formatting where leading 0s were missing.
+* Fixed plan creation failure on some even-length real-complex transforms that use Bluestein's algorithm.
 
-### **HIP** (6.2.1)
+### **rocSOLVER** (3.26.2)
 
 #### Resolved issues
 
-* Soft hang when using `AMD_SERIALIZE_KERNEL`
-* Memory leak in `hipIpcCloseMemHandle`
-
-### **HIPIFY** (18.0.0)
-
-#### Changes
-
-* Added CUDA 12.5.1 support
-* Added cuDNN 9.2.1 support
-* Added LLVM 18.1.8 support
-* Added `hipBLAS` 64-bit APIs support
-* Added Support for math constants `math_constants.h`
-
-### **Omnitrace** (1.11.2)
-
-#### Known issues
-
-Perfetto can no longer open Omnitrace proto files. Loading Perfetto trace output `.proto` files in the latest version of `ui.perfetto.dev` can result in a dialog with the message, "Oops, something went wrong! Please file a bug." The information in the dialog will refer to an "Unknown field type." The workaround is to open the files with the previous version of the Perfetto UI found at [https://ui.perfetto.dev/v46.0-35b3d9845/#!/](https://ui.perfetto.dev/v46.0-35b3d9845/#!/).
-
-See [issue #3767](https://github.com/ROCm/ROCm/issues/3767) on GitHub.
-
-### **RCCL** (2.20.5)
-
-#### Known issues
-
-On systems running Linux kernel 6.8.0, such as Ubuntu 24.04, GPUDirect RDMA is disabled and impacts multi-node RCCL performance.
-This issue was reproduced with RCCL 2.20.5 (ROCm 6.2.0 and 6.2.1) on systems with Broadcom Thor-2 NICs and affects other systems with RoCE networks using Linux 6.8.0 or newer.
-Older RCCL versions are also impacted.
-
-This issue will be addressed in a future ROCm release.
-
-See [issue #3772](https://github.com/ROCm/ROCm/issues/3772) on GitHub.
-
-### **rocAL** (2.0.0)
-
-#### Changes
- 
-* The new version of rocAL introduces many new features, but does not modify any of the existing public API functions.However, the version number was incremented from 1.3 to 2.0.
-  Applications linked to version 1.3 must be recompiled to link against version 2.0.
-* Added development and test packages.
-* Added C++ rocAL audio unit test and Python script to run and compare the outputs.
-* Added Python support for audio decoders.
-* Added Pytorch iterator for audio.
-* Added Python audio unit test and support to verify outputs.
-* Added rocDecode for HW decode.
-* Added support for: 
-    * Audio loader and decoder, which uses libsndfile library to decode wav files
-    * Audio augmentation - PreEmphasis filter, Spectrogram, ToDecibels, Resample, NonSilentRegionDetection, MelFilterBank 
-    * Generic augmentation - Slice, Normalize
-    * Reading from file lists in file reader
-    * Downmixing audio channels during decoding
-    * TensorTensorAdd and TensorScalarMultiply operations
-    * Uniform and Normal distribution nodes
-* Image to tensor updates
-* ROCm install - use case graphics removed
-
-#### Known issues
- 
-* Dependencies are not installed with the rocAL package installer. Dependencies must be installed with the prerequisite setup script provided. See the [rocAL README on GitHub](https://github.com/ROCm/rocAL/blob/docs/6.2.1/README.md#prerequisites-setup-script) for details.
-
-### **rocBLAS** (4.2.1)
-
-#### Removals
-
-* Removed Device_Memory_Allocation.pdf link in documentation.
-
-#### Resolved issues
-
-* Fixed error/warning message during `rocblas_set_stream()` call.
-
-### **rocFFT** (1.0.29)
-
-#### Optimizations
-
-* Implemented 1D kernels for factorizable sizes greater than 1024.
-
-### **ROCm SMI** (7.3.0)
-
-#### Optimizations
-
-* Improved handling of UnicodeEncodeErrors with non UTF-8 locales. Non UTF-8 locales were causing crashes on UTF-8 special characters.
-
-#### Resolved issues
-
-* Fixed an issue where the Compute Partition tests segfaulted when AMDGPU was loaded with optional parameters.
-
-#### Known issues
-
-* When setting CPX as a partition mode, there is a DRM node limit of 64. This is a known limitation when multiple drivers are using the DRM nodes. The `ls /sys/class/drm` command can be used to see the number of DRM nodes, and the following steps can be used to remove unnecessary drivers:
-        
-    1. Unload AMDGPU: `sudo rmmod amdgpu`.
-    2. Remove any unnecessary drivers using `rmmod`. For example, to remove an AST driver, run `sudo rmmod ast`.
-    3. Reload AMDGPU using `modprobe`: `sudo modprobe amdgpu`.
-
-### **rocPRIM** (3.2.1)
-
-#### Optimizations
-
-* Improved performance of `block_reduce_warp_reduce` when warp size equals block size.
+* Fixed synchronization issue in STEIN.
 
 ## ROCm known issues
 
-ROCm known issues are tracked on [GitHub](https://github.com/ROCm/ROCm/labels/Verified%20Issue). Known issues related to
-individual components are listed in the [Detailed component changes](detailed-component-changes) section.
-
-### Instinct MI300X GPU recovery failure on uncorrectable errors
-
-For the AMD Instinct MI300X accelerator, GPU recovery resets triggered by uncorrectable errors (UE) might not complete
-successfully, which can result in the system being left in an undefined state. A system reboot is needed to recover from
-this state. Additionally, error logging might fail in these situations, hindering diagnostics.
-
-This issue is under investigation and will be resolved in a future ROCm release.
-
-See [issue #3766](https://github.com/ROCm/ROCm/issues/3766) on GitHub.
+ROCm known issues are tracked on [GitHub](https://github.com/ROCm/ROCm/labels/Verified%20Issue).
+Known issues related to individual components are listed in the [Detailed component changes](#detailed-component-changes)
+section.
 
 ## ROCm upcoming changes
 

default.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <manifest>
     <remote name="rocm-org" fetch="https://github.com/ROCm/" />
-    <default revision="refs/tags/rocm-6.2.1"
+    <default revision="refs/tags/rocm-6.2.4"
      remote="rocm-org"
      sync-c="true"
      sync-j="4" />

docs/about/license.md

@@ -91,8 +91,7 @@ additional licenses. Please review individual repositories for more information.
 
 Open sourced ROCm components are released via public GitHub
 repositories, packages on [https://repo.radeon.com](https://repo.radeon.com) and other distribution channels.
-Proprietary products are only available on [https://repo.radeon.com](https://repo.radeon.com). Currently, only
-one component of ROCm, `rocm-llvm-alt` is governed by a proprietary license.
+Proprietary products are only available on [https://repo.radeon.com](https://repo.radeon.com).
 Proprietary components are organized in a proprietary subdirectory in the package
 repositories to distinguish from open sourced packages.
 

docs/compatibility/compatibility-matrix-historical-6.0.csv

@@ -1,117 +1,117 @@
-ROCm Version,6.2.1,6.2.0, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
-      :ref:`Operating Systems & kernels <OS-kernel-versions>`,Ubuntu 24.04,Ubuntu 24.04,,,,,
-      ,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3, 22.04.2","Ubuntu 22.04.4, 22.04.3, 22.04.2"
-      ,,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
-      ,"RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
-      ,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
-      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
-      ,,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
-      ,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,,,
-      ,.. _architecture-support-compatibility-matrix-past-60:,,,,,,
-      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
-      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
-      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
-      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
-      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
-      ,.. _gpu-support-compatibility-matrix-past-60:,,,,,,
-      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
-      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
-      ,gfx942 [#mi300_621-past-60]_,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
-      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
-      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
-      ,,,,,,,
-      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix-past-60:,,,,,,
-      :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1","2.14.0, 2.13.1, 2.12.1"
-      :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
-      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
-      ,,,,,,,
-      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix-past-60:,,,,,,
-      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
-      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
-      ,,,,,,,
-      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix-past-60:,,,,,,
-      Thrust,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-      CUB,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-      ,,,,,,,
-      KFD & USER SPACE [#kfd_support-past-60]_,.. _kfd-userspace-support-compatibility-matrix-past-60:,,
-      Tested user space versions,"6.1.x, 6.0.x","6.1.x, 6.0.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x, 5.6.x","6.2.x, 6.0.x, 5.7.x, 5.6.x"
-      ,,,,,,,
-      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix-past-60:,,,,,,
-      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
-      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.10.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
-      :doc:`MIOpen <miopen:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`MIVisionX <mivisionx:index>`,3.0.0,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
-      :doc:`rocAL <rocal:index>`,2.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
-      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,0.1.0,N/A,N/A,N/A,N/A,N/A
-      :doc:`RPP <rpp:index>`,1.8.0,1.8.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
-      ,,,,,,,
-      COMMUNICATION,.. _commlibs-support-compatibility-matrix-past-60:,,,,,,
-      :doc:`RCCL <rccl:index>`,2.20.5,2.20.5,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
-      ,,,,,,,
-      MATH LIBS,.. _mathlibs-support-compatibility-matrix-past-60:,,,,,,
-      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
-      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
-      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.8.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
-      :doc:`hipFFT <hipfft:index>`,1.0.15,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
-      :doc:`hipFORT <hipfort:index>`,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
-      :doc:`hipRAND <hiprand:index>`,2.11.0,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
-      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.2.0,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
-      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.1.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.1,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
-      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.2.0,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
-      :doc:`rocBLAS <rocblas:index>`,4.2.1,4.2.0,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
-      :doc:`rocFFT <rocfft:index>`,1.0.29,1.0.28,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
-      :doc:`rocRAND <rocrand:index>`,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
-      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.26.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
-      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.2.0,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
-      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.5.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
-      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
-      ,,,,,,,
-      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix-past-60:,,,,,,
-      :doc:`hipCUB <hipcub:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.3.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
-      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`rocThrust <rocthrust:index>`,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      ,,,,,,,
-      SUPPORT LIBS,,,,,,,
-      `hipother <https://github.com/ROCm/hipother>`_,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.1,6.2.0,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
-      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.4.05,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
-      ,,,,,,,
-      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix-past-60:,,,,,,
-      :doc:`AMD SMI <amdsmi:index>`,24.6.3,24.6.2,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
-      :doc:`ROCm Data Center Tool <rdc:index>`,1.0.0,1.0.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
-      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.3.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
-      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.1,rocm-6.2.0,rocm-6.1.2,rocm-6.1.1,rocm-6.1.0,rocm-6.0.2,rocm-6.0.0
-      ,,,,,,,
-      PERFORMANCE TOOLS,,,,,,,
-      :doc:`Omniperf <omniperf:index>`,2.0.1,2.0.1,N/A,N/A,N/A,N/A,N/A
-      :doc:`Omnitrace <omnitrace:index>`,1.11.2,1.11.2,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
-      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60201,2.0.60200,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
-      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,0.4.0,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCTracer <roctracer:index>`,4.1.60201,4.1.60200,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
-      ,,,,,,,
-      DEVELOPMENT TOOLS,,,,,,,
-      :doc:`HIPIFY <hipify:index>`,18.0.0.24332,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.13.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
-      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
-      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.2.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
-      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.3.0,0.3.0,0.3.0,N/A,N/A
-      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
-      ,,,,,,,
-      COMPILERS,.. _compilers-support-compatibility-matrix-past-60:,,,,,,
-      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
-      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24332,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      :doc:`llvm-project <llvm-project:index>`,18.0.0.24332,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24332,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      ,,,,,,,
-      RUNTIMES,.. _runtime-support-compatibility-matrix-past-60:,,,,,,
-      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      :doc:`HIP <hip:index>`,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
-      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.14.0,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0
+ROCm Version,6.2.4,6.2.2,6.2.1,6.2.0, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
+      :ref:`Operating systems & kernels <OS-kernel-versions>`,"Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04",Ubuntu 24.04,,,,,
+      ,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3, 22.04.2","Ubuntu 22.04.4, 22.04.3, 22.04.2"
+      ,,,,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
+      ,"RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
+      ,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
+      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
+      ,,,,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
+      ,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,,,
+      ,.. _architecture-support-compatibility-matrix-past-60:,,,,,,,,
+      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
+      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
+      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
+      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
+      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
+      ,.. _gpu-support-compatibility-matrix-past-60:,,,,,,,,
+      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
+      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
+      ,gfx942 [#mi300_624-past-60]_,gfx942 [#mi300_622-past-60]_,gfx942 [#mi300_621-past-60]_,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
+      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
+      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
+      ,,,,,,,,,
+      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix-past-60:,,,,,,,,
+      :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
+      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1","2.14.0, 2.13.1, 2.12.1"
+      :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
+      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
+      ,,,,,,,,,
+      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix-past-60:,,,,,,,,
+      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
+      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
+      ,,,,,,,,,
+      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix-past-60:,,,,,,,,
+      Thrust,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+      CUB,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+      ,,,,,,,,,
+      KFD & USER SPACE [#kfd_support-past-60]_,.. _kfd-userspace-support-compatibility-matrix-past-60:,,,,,,,,
+      Tested user space versions,"6.1.x, 6.0.x","6.1.x, 6.0.x","6.1.x, 6.0.x","6.1.x, 6.0.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x, 5.6.x","6.2.x, 6.0.x, 5.7.x, 5.6.x"
+      ,,,,,,,,,
+      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix-past-60:,,,,,,,,
+      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
+      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.10.0,2.10.0,2.10.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
+      :doc:`MIOpen <miopen:index>`,3.2.0,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`MIVisionX <mivisionx:index>`,3.0.0,3.0.0,3.0.0,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
+      :doc:`rocAL <rocal:index>`,2.0.0,2.0.0,2.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.6.0,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
+      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,0.1.0,0.1.0,0.1.0,N/A,N/A,N/A,N/A,N/A
+      :doc:`RPP <rpp:index>`,1.8.0,1.8.0,1.8.0,1.8.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
+      ,,,,,,,,,
+      COMMUNICATION,.. _commlibs-support-compatibility-matrix-past-60:,,,,,,,,
+      :doc:`RCCL <rccl:index>`,2.20.5,2.20.5,2.20.5,2.20.5,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
+      ,,,,,,,,,
+      MATH LIBS,.. _mathlibs-support-compatibility-matrix-past-60:,,,,,,,,
+      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
+      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
+      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.8.0,0.8.0,0.8.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
+      :doc:`hipFFT <hipfft:index>`,1.0.16,1.0.15,1.0.15,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
+      :doc:`hipFORT <hipfort:index>`,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
+      :doc:`hipRAND <hiprand:index>`,2.11.1,2.11.0,2.11.0,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
+      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.2.0,2.2.0,2.2.0,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
+      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.1.1,3.1.1,3.1.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.1,0.2.1,0.2.1,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
+      :doc:`rocALUTION <rocalution:index>`,3.2.1,3.2.0,3.2.0,3.2.0,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
+      :doc:`rocBLAS <rocblas:index>`,4.2.4,4.2.1,4.2.1,4.2.0,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
+      :doc:`rocFFT <rocfft:index>`,1.0.30,1.0.29,1.0.29,1.0.28,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
+      :doc:`rocRAND <rocrand:index>`,3.1.1,3.1.0,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
+      :doc:`rocSOLVER <rocsolver:index>`,3.26.2,3.26.0,3.26.0,3.26.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
+      :doc:`rocSPARSE <rocsparse:index>`,3.2.1,3.2.0,3.2.0,3.2.0,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
+      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
+      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.40.0,4.40.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
+      ,,,,,,,,,
+      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix-past-60:,,,,,,,,
+      :doc:`hipCUB <hipcub:index>`,3.2.1,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.3.0,1.3.0,1.3.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
+      :doc:`rocPRIM <rocprim:index>`,3.2.2,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`rocThrust <rocthrust:index>`,3.1.1,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      ,,,,,,,,,
+      SUPPORT LIBS,,,,,,,,,
+      `hipother <https://github.com/ROCm/hipother>`_,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.4,6.2.2,6.2.1,6.2.0,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
+      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.5.7,20240607.5.7,20240607.4.05,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
+      ,,,,,,,,,
+      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix-past-60:,,,,,,,,
+      :doc:`AMD SMI <amdsmi:index>`,24.6.3,24.6.3,24.6.3,24.6.2,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
+      :doc:`ROCm Data Center Tool <rdc:index>`,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
+      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.3.0,7.3.0,7.3.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
+      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.4,rocm-6.2.2,rocm-6.2.1,rocm-6.2.0,rocm-6.1.2,rocm-6.1.1,rocm-6.1.0,rocm-6.0.2,rocm-6.0.0
+      ,,,,,,,,,
+      PERFORMANCE TOOLS,,,,,,,,,
+      :doc:`Omniperf <omniperf:index>`,2.0.1,2.0.1,2.0.1,2.0.1,N/A,N/A,N/A,N/A,N/A
+      :doc:`Omnitrace <omnitrace:index>`,1.11.2,1.11.2,1.11.2,1.11.2,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
+      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60204,2.0.60202,2.0.60201,2.0.60200,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
+      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,0.4.0,0.4.0,0.4.0,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCTracer <roctracer:index>`,4.1.60204,4.1.60202,4.1.60201,4.1.60200,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
+      ,,,,,,,,,
+      DEVELOPMENT TOOLS,,,,,,,,,
+      :doc:`HIPIFY <hipify:index>`,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.13.0,0.13.0,0.13.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
+      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.76.0,0.76.0,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
+      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.2.0,14.2.0,14.2.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
+      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.4.0,0.4.0,0.3.0,0.3.0,0.3.0,N/A,N/A
+      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
+      ,,,,,,,,,
+      COMPILERS,.. _compilers-support-compatibility-matrix-past-60:,,,,,,,,
+      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,N/A,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
+      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.1.1,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`llvm-project <llvm-project:index>`,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      ,,,,,,,,,
+      RUNTIMES,.. _runtime-support-compatibility-matrix-past-60:,,,,,,,,
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      :doc:`HIP <hip:index>`,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
+      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.14.0,1.14.0,1.14.0,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0

docs/compatibility/compatibility-matrix.rst

@@ -10,17 +10,19 @@ Use this matrix to view the ROCm compatibility and system requirements across su
 
 You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-rocm-compatibility-matrix>`.
 
+Accelerators and GPUs listed in the following table support compute workloads (no display information or graphics). If you’re using ROCm with AMD Radeon or Radeon Pro GPUs for graphics workloads, see the `Use ROCm on Radeon GPU documentation <https://rocm.docs.amd.com/projects/radeon/en/latest/docs/compatibility.html>`_ to verify compatibility and system requirements.
+
 .. |br| raw:: html
 
    <br/>
 
 .. container:: format-big-table
 
-  .. csv-table:: 
-      :header: "ROCm Version", "6.2.1", "6.2.0", "6.1.0"
+  .. csv-table::
+      :header: "ROCm Version", "6.2.4", "6.2.2", "6.1.0"
       :stub-columns: 1
 
-      :ref:`Operating Systems & kernels <OS-kernel-versions>`,Ubuntu 24.04,Ubuntu 24.04,
+      :ref:`Operating systems & kernels <OS-kernel-versions>`,"Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04",
       ,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.4, 22.04.3"
       ,,,"Ubuntu 20.04.6, 20.04.5"
       ,"RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94]_, 9.3, 9.2"
@@ -37,13 +39,13 @@ You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-
       ,.. _gpu-support-compatibility-matrix:,,
       :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100
       ,gfx1030,gfx1030,gfx1030
-      ,gfx942 [#mi300_621]_,gfx942 [#mi300_620]_, gfx942 [#mi300_610]_
+      ,gfx942 [#mi300_624]_,gfx942 [#mi300_622]_, gfx942 [#mi300_610]_
       ,gfx90a,gfx90a,gfx90a
       ,gfx908,gfx908,gfx908
       ,,,
       FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix:,,
       :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15, 2.14, 2.13"
+      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1"
       :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26
       `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3
       ,,,
@@ -63,7 +65,7 @@ You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-
       :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.10.0,2.9.0
       :doc:`MIOpen <miopen:index>`,3.2.0,3.2.0,3.1.0
       :doc:`MIVisionX <mivisionx:index>`,3.0.0,3.0.0,2.5.0
-      :doc:`rocAL <rocal:index>`,2.0.0,1.0.0,1.0.0
+      :doc:`rocAL <rocal:index>`,2.0.0,2.0.0,1.0.0
       :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.5.0
       :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,0.1.0,N/A
       :doc:`RPP <rpp:index>`,1.8.0,1.8.0,1.5.0
@@ -75,49 +77,49 @@ You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-
       `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0
       :doc:`hipBLAS <hipblas:index>`,2.2.0,2.2.0,2.1.0
       :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.8.0,0.7.0
-      :doc:`hipFFT <hipfft:index>`,1.0.15,1.0.14,1.0.14
+      :doc:`hipFFT <hipfft:index>`,1.0.16,1.0.15,1.0.14
       :doc:`hipFORT <hipfort:index>`,0.4.0,0.4.0,0.4.0
-      :doc:`hipRAND <hiprand:index>`,2.11.0,2.11.0,2.10.16
+      :doc:`hipRAND <hiprand:index>`,2.11.1,2.11.0,2.10.16
       :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.2.0,2.1.0
       :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.1.1,3.0.1
       :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.1,0.1.0
-      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.2.0,3.1.1
-      :doc:`rocBLAS <rocblas:index>`,4.2.1,4.2.0,4.1.0
-      :doc:`rocFFT <rocfft:index>`,1.0.29,1.0.28,1.0.26
-      :doc:`rocRAND <rocrand:index>`,3.1.0,3.1.0,3.0.1
-      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.26.0,3.25.0
-      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.2.0,3.1.2
+      :doc:`rocALUTION <rocalution:index>`,3.2.1,3.2.0,3.1.1
+      :doc:`rocBLAS <rocblas:index>`,4.2.4,4.2.1,4.1.0
+      :doc:`rocFFT <rocfft:index>`,1.0.30,1.0.29,1.0.26
+      :doc:`rocRAND <rocrand:index>`,3.1.1,3.1.0,3.0.1
+      :doc:`rocSOLVER <rocsolver:index>`,3.26.2,3.26.0,3.25.0
+      :doc:`rocSPARSE <rocsparse:index>`,3.2.1,3.2.0,3.1.2
       :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.5.0,1.4.0
       `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.40.0
       ,,,
       PRIMITIVES,.. _primitivelibs-support-compatibility-matrix:,,
-      :doc:`hipCUB <hipcub:index>`,3.2.0,3.2.0,3.1.0
+      :doc:`hipCUB <hipcub:index>`,3.2.1,3.2.0,3.1.0
       :doc:`hipTensor <hiptensor:index>`,1.3.0,1.3.0,1.2.0
-      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.2.0,3.1.0
-      :doc:`rocThrust <rocthrust:index>`,3.1.0,3.0.1,3.0.1
+      :doc:`rocPRIM <rocprim:index>`,3.2.2,3.2.0,3.1.0
+      :doc:`rocThrust <rocthrust:index>`,3.1.1,3.1.0,3.0.1
       ,,,
       SUPPORT LIBS,,,
-      `hipother <https://github.com/ROCm/hipother>`_,6.2.41134,6.2.41133,6.1.40091
-      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.1,6.2.0,6.1.0
-      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.4.05,20240607.1.4246,20240125.3.30
+      `hipother <https://github.com/ROCm/hipother>`_,6.2.41134,6.2.41134,6.1.40091
+      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.4,6.2.2,6.1.0
+      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.5.7,20240607.5.7,20240125.3.30
       ,,,
       SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix:,,
-      :doc:`AMD SMI <amdsmi:index>`,24.6.3,24.6.2,24.4.1
-      :doc:`ROCm Data Center Tool <rdc:index>`,1.0.0,1.0.0,0.3.0
+      :doc:`AMD SMI <amdsmi:index>`,24.6.3,24.6.3,24.4.1
+      :doc:`ROCm Data Center Tool <rdc:index>`,0.3.0,0.3.0,0.3.0
       :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0
       :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.3.0,7.0.0
-      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.1,rocm-6.2.0,rocm-6.1.0
+      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.4,rocm-6.2.2,rocm-6.1.0
       ,,,
       PERFORMANCE TOOLS,,,
       :doc:`Omniperf <omniperf:index>`,2.0.1,2.0.1,N/A
       :doc:`Omnitrace <omnitrace:index>`,1.11.2,1.11.2,N/A
       :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0
-      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60201,2.0.60200,2.0.60100
+      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60204,2.0.60202,2.0.60100
       :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,0.4.0,N/A
-      :doc:`ROCTracer <roctracer:index>`,4.1.60201,4.1.60200,4.1.60100
+      :doc:`ROCTracer <roctracer:index>`,4.1.60204,4.1.60202,4.1.60100
       ,,,
       DEVELOPMENT TOOLS,,,
-      :doc:`HIPIFY <hipify:index>`,18.0.0.24332,18.0.0.24232,17.0.0.24103
+      :doc:`HIPIFY <hipify:index>`,18.0.0.24392,18.0.0.24355,17.0.0.24103
       :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.13.0,0.12.0
       :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.76.0,0.71.0
       :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.2.0,14.1.0
@@ -127,24 +129,24 @@ You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-
       COMPILERS,.. _compilers-support-compatibility-matrix:,,
       `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,0.5.0
       :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.0.0
-      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24332,18.0.0.24232,17.0.0.24103
-      :doc:`llvm-project <llvm-project:index>`,18.0.0.24332,18.0.0.24232,17.0.0.24103
-      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24332,18.0.0.24232,17.0.0.24103
+      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24392,18.0.0.24355,17.0.0.24103
+      :doc:`llvm-project <llvm-project:index>`,18.0.0.24392,18.0.0.24355,17.0.0.24103
+      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24392,18.0.0.24355,17.0.0.24103
       ,,,
       RUNTIMES,.. _runtime-support-compatibility-matrix:,,
-      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41134,6.2.41133,6.1.40091
-      :doc:`HIP <hip:index>`,6.2.41134,6.2.41133,6.1.40091
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41134,6.2.41134,6.1.40091
+      :doc:`HIP <hip:index>`,6.2.41134,6.2.41134,6.1.40091
       `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0
-      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.14.0,1.13.0,1.13.0
+      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.14.0,1.14.0,1.13.0
 
 
 .. rubric:: Footnotes
 
 .. [#red-hat94] RHEL 9.4 is supported only on AMD Instinct MI300A.
 .. [#oracle89] Oracle Linux is supported only on AMD Instinct MI300X.
-.. [#mi300_621] **For ROCm 6.2.1** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
-.. [#mi300_620] **For ROCm 6.2.0** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
-.. [#mi300_610] **For ROCm 6.1.2** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4 and Oracle Linux.
+.. [#mi300_624] **For ROCm 6.2.4** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
+.. [#mi300_622] **For ROCm 6.2.2** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
+.. [#mi300_610] **For ROCm 6.1.0** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4.
 .. [#kfd_support] ROCm provides forward and backward compatibility between the Kernel Fusion Driver (KFD) and its user space software for +/- 2 releases. These are the compatibility combinations that are currently supported.
 
 
@@ -160,7 +162,8 @@ Use this look up table to confirm which operating system and kernel versions are
    :widths: 40, 20, 40
    :stub-columns: 1
 
-   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 24.04, "6.8 GA"
+   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 24.04.1, "6.8 GA"
+   , 24.04, "6.8 GA"
    `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 22.04.05, "5.15 GA, 6.8 HWE"
    , 22.04.04, "5.15 GA, 6.5 HWE"
    , 22.04.03, "5.15 GA, 6.2 HWE"
@@ -206,7 +209,6 @@ Expand for full historical view of:
 
    .. csv-table::
       :file: compatibility-matrix-historical-6.0.csv
-      :widths: 20,10,10,10,10,10,10,10
       :header-rows: 1
       :stub-columns: 1
    
@@ -214,6 +216,8 @@ Expand for full historical view of:
 
    .. [#red-hat94-past-60] RHEL 9.4 is supported only on AMD Instinct MI300A.
    .. [#oracle89-past-60] Oracle Linux is supported only on AMD Instinct MI300X.
+   .. [#mi300_624-past-60] **For ROCm 6.2.4** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
+   .. [#mi300_622-past-60] **For ROCm 6.2.2** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
    .. [#mi300_621-past-60] **For ROCm 6.2.1** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
    .. [#mi300_620-past-60] **For ROCm 6.2.0** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
    .. [#mi300_612-past-60] **For ROCm 6.1.2** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4 and Oracle Linux.

docs/conceptual/ai-migraphx-optimization.md

@@ -1,333 +0,0 @@
-<head>
-  <meta charset="UTF-8">
-  <meta name="description" content="Inference optimization with MIGraphX">
-  <meta name="keywords" content="Inference optimization, MIGraphX, deep-learning, MIGraphX
-  installation, AMD, ROCm">
-</head>
-
-# Inference optimization with MIGraphX
-
-The following sections cover inferencing and introduces [MIGraphX](https://rocm.docs.amd.com/projects/AMDMIGraphX/en/latest/).
-
-## Inference
-
-The inference is where capabilities learned during deep-learning training are put to work. It refers to using a fully trained neural network to make conclusions (predictions) on unseen data that the model has never interacted with before. Deep-learning inferencing is achieved by feeding new data, such as new images, to the network, giving the Deep Neural Network a chance to classify the image.
-
-Taking our previous example of MNIST, the DNN can be fed new images of handwritten digit images, allowing the neural network to classify digits. A fully trained DNN should make accurate predictions about what an image represents, and inference cannot happen without training.
-
-## MIGraphX introduction
-
-MIGraphX is a graph compiler focused on accelerating the machine-learning inference that can target AMD GPUs and CPUs. MIGraphX accelerates the machine-learning models by leveraging several graph-level transformations and optimizations. These optimizations include:
-
-* Operator fusion
-* Arithmetic simplifications
-* Dead-code elimination
-* Common subexpression elimination (CSE)
-* Constant propagation
-
-After doing all these transformations, MIGraphX emits code for the AMD GPU by calling to MIOpen or rocBLAS or creating HIP kernels for a particular operator. MIGraphX can also target CPUs using DNNL or ZenDNN libraries.
-
-MIGraphX provides easy-to-use APIs in C++ and Python to import machine models in ONNX or TensorFlow. Users can compile, save, load, and run these models using the MIGraphX C++ and Python APIs. Internally, MIGraphX parses ONNX or TensorFlow models into internal graph representation where each operator in the model gets mapped to an operator within MIGraphX. Each of these operators defines various attributes such as:
-
-* Number of arguments
-* Type of arguments
-* Shape of arguments
-
-After optimization passes, all these operators get mapped to different kernels on GPUs or CPUs.
-
-After importing a model into MIGraphX, the model is represented as `migraphx::program`. `migraphx::program` is made up of `migraphx::module`. The program can consist of several modules, but it always has one main_module. Modules are made up of `migraphx::instruction_ref`. Instructions contain the `migraphx::op` and arguments to the operator.  
-
-## Installing MIGraphX
-
-There are three options to get started with MIGraphX installation. MIGraphX depends on ROCm libraries; assume that the machine has ROCm installed.
-
-### Option 1: installing binaries
-
-To install MIGraphX on Debian-based systems like Ubuntu, use the following command:
-
-```bash
-sudo apt update && sudo apt install -y migraphx
-```
-
-The header files and libraries are installed under `/opt/rocm-\<version\>`, where \<version\> is the ROCm version.
-
-### Option 2: building from source
-
-There are two ways to build the MIGraphX sources.
-
-* [Use the ROCm build tool](https://github.com/ROCm/AMDMIGraphX#use-the-rocm-build-tool-rbuild) - This approach uses `[rbuild](https://github.com/ROCm/rbuild)` to install the prerequisites and build the libraries with just one command.
-
-  or
-
-* [Use CMake](https://github.com/ROCm/AMDMIGraphX#use-cmake-to-build-migraphx) - This approach uses a script to install the prerequisites, then uses CMake to build the source.
-
-For detailed steps on building from source and installing dependencies, refer to the following `README` file:
-
-[https://github.com/ROCm/AMDMIGraphX#building-from-source](https://github.com/ROCm/AMDMIGraphX#building-from-source)
-
-### Option 3: use docker
-
-To use Docker, follow these steps:
-
-1. The easiest way to set up the development environment is to use Docker. To build Docker from scratch, first clone the MIGraphX repository by running:
-
-    ```bash
-    git clone --recursive https://github.com/ROCm/AMDMIGraphX
-    ```
-
-2. The repository contains a Dockerfile from which you can build a Docker image as:
-
-    ```bash
-    docker build -t migraphx .
-    ```
-
-3. Then to enter the development environment, use Docker run:
-
-    ```bash
-    docker run --device='/dev/kfd' --device='/dev/dri' -v=`pwd`:/code/AMDMIGraphX -w /code/AMDMIGraphX --group-add video -it migraphx
-    ```
-
-The Docker image contains all the prerequisites required for the installation, so users can go to the folder `/code/AMDMIGraphX` and follow the steps mentioned in [Option 2: Building from Source](#option-2-building-from-source).
-
-## MIGraphX example
-
-MIGraphX provides both C++ and Python APIs. The following sections show examples of both using the Inception v3 model. To walk through the examples, fetch the Inception v3 ONNX model by running the following:
-
-```py
-import torch
-import torchvision.models as models
-inception = models.inception_v3(pretrained=True)
-torch.onnx.export(inception,torch.randn(1,3,299,299), "inceptioni1.onnx")
-```
-
-This will create `inceptioni1.onnx`, which can be imported in MIGraphX using C++ or Python API.
-
-### MIGraphX Python API
-
-Follow these steps:
-
-1. To import the MIGraphX module in Python script, set `PYTHONPATH` to the MIGraphX libraries installation. If binaries are installed using steps mentioned in [Option 1: Installing Binaries](#option-1-installing-binaries), perform the following action:
-
-    ```bash
-    export PYTHONPATH=$PYTHONPATH:/opt/rocm/
-    ```
-
-2. The following script shows the usage of Python API to import the ONNX model, compile it, and run inference on it. Set `LD_LIBRARY_PATH` to `/opt/rocm/` if required.
-
-    ```py
-    # import migraphx and numpy
-    import migraphx
-    import numpy as np
-    # import and parse inception model
-    model = migraphx.parse_onnx("inceptioni1.onnx")
-    # compile model for the GPU target
-    model.compile(migraphx.get_target("gpu"))
-    # optionally print compiled model
-    model.print()
-    # create random input image
-    input_image = np.random.rand(1, 3, 299, 299).astype('float32')
-    # feed image to model, 'x.1` is the input param name
-    results = model.run({'x.1': input_image})
-    # get the results back
-    result_np = np.array(results[0])
-    # print the inferred class of the input image
-    print(np.argmax(result_np))
-    ```
-
-    Find additional examples of Python API in the `/examples` directory of the MIGraphX repository.
-
-## MIGraphX C++ API
-
-Follow these steps:
-
-1. The following is a minimalist example that shows the usage of MIGraphX C++ API to load ONNX file, compile it for the GPU, and run inference on it. To use MIGraphX C++ API, you only need to load the `migraphx.hpp` file. This example runs inference on the Inception v3 model.
-
-    ```c++
-    #include <vector>
-    #include <string>
-    #include <algorithm>
-    #include <ctime>
-    #include <random>
-    #include <migraphx/migraphx.hpp>
-
-    int main(int argc, char** argv)
-    {
-        migraphx::program prog;
-        migraphx::onnx_options onnx_opts;
-        // import and parse onnx file into migraphx::program
-        prog = parse_onnx("inceptioni1.onnx", onnx_opts);
-        // print imported model
-        prog.print();
-        migraphx::target targ = migraphx::target("gpu");
-        migraphx::compile_options comp_opts;
-        comp_opts.set_offload_copy();
-        // compile for the GPU
-        prog.compile(targ, comp_opts);
-        // print the compiled program
-        prog.print();
-        // randomly generate input image
-        // of shape (1, 3, 299, 299)
-        std::srand(unsigned(std::time(nullptr)));
-        std::vector<float> input_image(1*299*299*3);
-        std::generate(input_image.begin(), input_image.end(), std::rand);
-        // users need to provide data for the input
-        // parameters in order to run inference
-        // you can query into migraph program for the parameters
-        migraphx::program_parameters prog_params;
-        auto param_shapes = prog.get_parameter_shapes();
-        auto input        = param_shapes.names().front();
-        // create argument for the parameter
-        prog_params.add(input, migraphx::argument(param_shapes[input], input_image.data()));
-        // run inference
-        auto outputs = prog.eval(prog_params);
-        // read back the output
-        float* results = reinterpret_cast<float*>(outputs[0].data());
-        float* max     = std::max_element(results, results + 1000);
-        int answer = max - results;
-        std::cout << "answer: " << answer << std::endl;
-    }
-    ```
-
-2. To compile this program, you can use CMake and you only need to link the `migraphx::c` library to use MIGraphX's C++ API. The following is the `CMakeLists.txt` file that can build the earlier example:
-
-    ```cmake
-    cmake_minimum_required(VERSION 3.5)
-    project (CAI)
-
-    set (CMAKE_CXX_STANDARD 14)
-    set (EXAMPLE inception_inference)
-
-    list (APPEND CMAKE_PREFIX_PATH /opt/rocm/hip /opt/rocm)
-    find_package (migraphx)
-
-    message("source file: " ${EXAMPLE}.cpp " ---> bin: " ${EXAMPLE})
-    add_executable(${EXAMPLE} ${EXAMPLE}.cpp)
-
-    target_link_libraries(${EXAMPLE} migraphx::c)
-    ```
-
-3. To build the executable file, run the following from the directory containing the `inception_inference.cpp` file:
-
-    ```bash
-    mkdir build
-    cd build
-    cmake ..
-    make -j$(nproc)
-    ./inception_inference
-    ```
-
-:::{note}
-    Set `LD_LIBRARY_PATH` to `/opt/rocm/lib` if required during the build. Additional examples can be found in the MIGraphX repository under the `/examples/` directory.
-:::
-
-## Tuning MIGraphX
-
-MIGraphX uses MIOpen kernels to target AMD GPU. For the model compiled with MIGraphX, tune MIOpen to pick the best possible kernel implementation. The MIOpen tuning results in a significant performance boost. Tuning can be done by setting the environment variable `MIOPEN_FIND_ENFORCE=3`.
-
-:::{note}
-    The tuning process can take a long time to finish.
-:::
-
-**Example:** The average inference time of the inception model example shown previously over 100 iterations using untuned kernels is 0.01383ms. After tuning, it reduces to 0.00459ms, which is a 3x improvement. This result is from ROCm v4.5 on a MI100 GPU.
-
-:::{note}
-    The results may vary depending on the system configurations.
-:::
-
-For reference, the following code snippet shows inference runs for only the first 10 iterations for both tuned and untuned kernels:
-
-```console
-### UNTUNED ###
-iterator : 0
-Inference complete
-Inference time: 0.063ms
-iterator : 1
-Inference complete
-Inference time: 0.008ms
-iterator : 2
-Inference complete
-Inference time: 0.007ms
-iterator : 3
-Inference complete
-Inference time: 0.007ms
-iterator : 4
-Inference complete
-Inference time: 0.007ms
-iterator : 5
-Inference complete
-Inference time: 0.008ms
-iterator : 6
-Inference complete
-Inference time: 0.007ms
-iterator : 7
-Inference complete
-Inference time: 0.028ms
-iterator : 8
-Inference complete
-Inference time: 0.029ms
-iterator : 9
-Inference complete
-Inference time: 0.029ms
-
-### TUNED ###
-iterator : 0
-Inference complete
-Inference time: 0.063ms
-iterator : 1
-Inference complete
-Inference time: 0.004ms
-iterator : 2
-Inference complete
-Inference time: 0.004ms
-iterator : 3
-Inference complete
-Inference time: 0.004ms
-iterator : 4
-Inference complete
-Inference time: 0.004ms
-iterator : 5
-Inference complete
-Inference time: 0.004ms
-iterator : 6
-Inference complete
-Inference time: 0.004ms
-iterator : 7
-Inference complete
-Inference time: 0.004ms
-iterator : 8
-Inference complete
-Inference time: 0.004ms
-iterator : 9
-Inference complete
-Inference time: 0.004ms
-```
-
-### YModel
-
-The best inference performance through MIGraphX is conditioned upon having tuned kernel configurations stored in a `/home` local User Database (DB). If a user were to move their model to a different server or allow a different user to use it, they would have to run through the MIOpen tuning process again to populate the next User DB with the best kernel configurations and corresponding solvers.
-
-Tuning is time consuming, and if the users have not performed tuning, they would see discrepancies between expected or claimed inference performance and actual inference performance. This has led to repetitive and time-consuming tuning tasks for each user.
-
-MIGraphX introduces a feature, known as YModel, that stores the kernel config parameters found during tuning into a `.mxr` file. This ensures the same level of expected performance, even when a model is copied to a different user/system.
-
-The YModel feature is available starting from ROCm 5.4.1 and UIF 1.1.
-
-#### YModel example
-
-Through the `migraphx-driver` functionality, you can generate `.mxr` files with tuning information stored inside it by passing additional `--binary --output model.mxr` to `migraphx-driver` along with the rest of the necessary flags.
-
-For example, to generate `.mxr` file from the ONNX model, use the following:
-
-```bash
-./path/to/migraphx-driver compile --onnx resnet50.onnx --enable-offload-copy --binary --output resnet50.mxr
-```
-
-To run generated `.mxr` files through `migraphx-driver`, use the following:
-
-```bash
-./path/to/migraphx-driver run --migraphx resnet50.mxr --enable-offload-copy
-```
-
-Alternatively, you can use the MIGraphX C++ or Python API to generate `.mxr` files.
-
-![Generating an MXR file](../data/conceptual/image018.png "Generating an MXR file")

docs/conceptual/oversubscription.rst

@@ -0,0 +1,34 @@
+.. meta::
+   :description: Learn what causes oversubscription.
+   :keywords: warning, log, gpu, performance penalty, help
+
+*******************************************************************
+Oversubscription of hardware resources in AMD Instinct accelerators
+*******************************************************************
+
+When an AMD Instinct™ MI series accelerator enters an oversubscribed state, the ``amdgpu`` driver outputs the following
+message.
+
+``amdgpu: Runlist is getting oversubscribed. Expect reduced ROCm performance.``
+
+Oversubscription occurs when application demands exceed the available hardware resources. In an oversubscribed
+state, the hardware scheduler tries to manage resource usage in a round-robin fashion. However,
+this can result in reduced performance, as resources might be occupied by applications or queues not actively
+submitting work. The granularity of hardware resources occupied by an inactive queue can be in the order of
+milliseconds, during which the accelerator or GPU is effectively blocked and unable to process work submitted by other
+queues.
+
+What triggers oversubscription?
+===============================
+
+The system enters an oversubscribed state when one of the following conditions is met:
+
+* **Hardware queue limit exceeded**: The number of user-mode compute queues requested by applications exceeds the
+  hardware limit of 24 queues for current Instinct accelerators.
+
+* **Virtual memory context slots exceeded**: The number of user processes exceeds the number of available virtual memory
+  context slots, which is 11 for current Instinct accelerators.
+
+* **Multiple processes using cooperative workgroups**: More than one process attempts to use the cooperative workgroup
+  feature, leading to resource contention.
+

docs/conf.py

@@ -30,16 +30,15 @@ if os.environ.get("READTHEDOCS", "") == "True":
 project = "ROCm Documentation"
 author = "Advanced Micro Devices, Inc."
 copyright = "Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved."
-version = "6.2.1"
-release = "6.2.1"
+version = "6.2.4"
+release = "6.2.4"
 setting_all_article_info = True
 all_article_info_os = ["linux", "windows"]
 all_article_info_author = ""
 
 # pages with specific settings
 article_pages = [
-    {"file": "about/release-notes", "os": ["linux", "windows"], "date": "2024-09-20"},
-    {"file": "about/changelog", "os": ["linux", "windows"], "date": "2024-09-20"},
+    {"file": "about/release-notes", "os": ["linux", "windows"], "date": "2024-11-06"},
     {"file": "how-to/deep-learning-rocm", "os": ["linux"]},
     {"file": "how-to/rocm-for-ai/index", "os": ["linux"]},
     {"file": "how-to/rocm-for-ai/install", "os": ["linux"]},
@@ -82,6 +81,7 @@ article_pages = [
         "file": "how-to/llm-fine-tuning-optimization/profiling-and-debugging",
         "os": ["linux"],
     },
+    {"file": "how-to/performance-validation/mi300x/vllm-benchmark", "os": ["linux"]},
     {"file": "how-to/system-optimization/index", "os": ["linux"]},
     {"file": "how-to/system-optimization/mi300x", "os": ["linux"]},
     {"file": "how-to/system-optimization/mi200", "os": ["linux"]},

docs/contribute/building.md

@@ -7,17 +7,14 @@
 
 # Building documentation
 
-You can build our documentation via GitHub (in a pull request) or locally (using the command line or
-Visual Studio (VS) Code.
-
 ## GitHub
 
-If you open a pull request on the `develop` branch of a ROCm repository and scroll to the bottom of
-the page, there is a summary panel. Next to the line
-`docs/readthedocs.com:advanced-micro-devices-demo`, there is a `Details` link. If you click this, it takes
-you to the Read the Docs build for your pull request.
+If you open a pull request and scroll down to the summary panel,
+there is a commit status section. Next to the line
+`docs/readthedocs.com:advanced-micro-devices-demo`, there is a `Details` link.
+If you click this, it takes you to the Read the Docs build for your pull request.
 
-![Screenshot of the GitHub documentation build link](../data/contribute/github-docs-build.png)
+![GitHub PR commit status](../data/contribute/commit-status.png)
 
 If you don't see this line, click `Show all checks` to get an itemized view.
 

docs/contribute/contributing.md

@@ -4,113 +4,68 @@
   <meta name="keywords" content="ROCm, contributing, contribute, maintainer, contributor">
 </head>
 
-# Contribute to ROCm documentation
+# Contributing to the ROCm documentation
 
-All ROCm projects are GitHub-based, so if you want to contribute, you can do so by:
+The ROCm documentation, like all of ROCm, is open source and available on GitHub. You can contribute to the ROCm documentation by forking the appropriate repository, making your changes, and opening a pull request.
 
-* [Submitting a pull request in the appropriate GitHub repository](#submit-a-pull-request)
-* [Creating an issue in the appropriate GitHub repository](#create-an-issue)
-* [Suggesting a new feature](#suggest-a-new-feature)
+To provide feedback on the ROCm documentation, including submitting an issue or suggesting a feature, see [Providing feedback about the ROCm documentation](./feedback.md).
 
-```{important}
-By creating a pull request (PR), you agree to allow your contribution to be licensed under the terms of the
-LICENSE.txt file in the corresponding repository. Different repositories may use different licenses.
-```
+## The ROCm repositories
 
-## Submit a pull request
+The repositories for ROCm and all ROCm components are available on GitHub.
 
-To make edits to our documentation via PR, follow these steps:
+| Module | Documentation location |
+| --- | --- |
+| ROCm framework | [https://github.com/ROCm/ROCm/tree/develop/docs](https://github.com/ROCm/ROCm/tree/develop/docs) |
+| ROCm installation for Linux | [https://github.com/ROCm/rocm-install-on-linux/tree/develop/docs](https://github.com/ROCm/rocm-install-on-linux/tree/develop/docs) |
+| ROCm HIP SDK installation for Windows |  [https://github.com/ROCm/rocm-install-on-windows/tree/develop/docs](https://github.com/ROCm/rocm-install-on-windows/tree/develop/docs) |
 
-1. Identify the repository and the file you want to update. For example, to update this page, you would
-  need to modify content located in this file:
-  `https://github.com/ROCm/ROCm/blob/develop/docs/contribute/contributing.md`
+Individual components have their own repositories with their own documentation in their own `docs` folders.
 
-2. (optional, but recommended) Fork the repository.
+The sub-folders within the `docs` folders across ROCm are typically structured as follows:
 
-3. Clone the repository locally and (optionally) add your fork. Select the green 'Code' button and copy
-   the URL (e.g., `git@github.com:ROCm/ROCm.git`).
+| Sub-folder name | Documentation type |
+|-------|----------|
+| `install` | Installation instructions, build instructions, and prerequisites |
+| `conceptual` | Important concepts |
+| `how-to` | How to implement specific use cases |
+| `tutorials` | Tutorials |
+| `reference` | API references and other reference resources |
 
-   * From your terminal, run:
+## Editing and adding to the documentation
 
-      ```bash
-      git clone git@github.com:ROCm/ROCm.git
-      ```
+The ROCm documentation is written in [reStructuredText (rst)](https://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html) and [Github-flavoured Markdown](https://github.github.com/gfm/), and follows the [Google developer documentation style guide](https://developers.google.com/style/highlights). reStructuredText is preferred when adding content to the documentation.
 
-   * Optionally add your fork to this local copy of the repository by running:
+To edit or add to the documentation:
 
-      ```bash
-      git add remote <name-of-my-fork> <git@github.com:my-username/ROCm.git>
-      ```
+1. Fork the repository you want to add to or edit.
+2. Clone your fork locally.
+3. Create a new local branch cut from the `develop` branch of the repository.
+4. Make your changes to the documentation.
 
-      To get the URL of your fork, go to your GitHub profile, select the fork and click the green 'Code'
-      button (the same process you followed to get the main GitHub repository URL).
-
-4. Change directory into your local copy of the repository, and run ``git pull`` (or ``git pull origin develop``) to ensure your local copy has the most recent content.
-
-5. Create and checkout a new branch using the following command:
+5. Optionally, build the documentation locally before creating a pull request by running the following commands from within the `docs` folder:
 
     ```bash
-    git checkout -b <branch_name>
-    ```
-
-6. Change directory into the `./docs` folder and make any documentation changes locally using your preferred code editor. Follow the guidelines listed on the
-   [documentation structure](./doc-structure.md) page.
-
-   ```{note}
-   Spell checking is performed for pull requests by {doc}`ROCm Docs Core<rocm-docs-core:index>`. To ensure your PR passes spell checking you might need at add new words or acronyms to the `.wordlist.txt` file as described in {doc}`Spell Check<rocm-docs-core:user_guide/spellcheck>`. 
-   ```
-
-7. Optionally run a local test build of the documentation to ensure the content builds and looks as expected. In your terminal, run the following commands from within the `./docs` folder of your cloned repository:
-
-     ```bash
      pip3 install -r sphinx/requirements.txt  # You only need to run this command once
      python3 -m sphinx -T -E -b html -d _build/doctrees -D language=en . _build/html
      ```
 
-    The build output files are located in the `docs/_build` folder. To preview your build, open the index file
-    (`docs/_build/html/index.html`) file. For more information, see [Building documentation](building.md). To learn
-    more about our build tools, see [Documentation toolchain](toolchain.md).
+    The output files will be located in the `docs/_build` folder. Open `docs/_build/html/index.html` to view the documentation.
 
-8. Commit your changes and push them to GitHub by running:
+    For more information on ROCm build tools, see [Documentation toolchain](toolchain.md).
+6. Push your changes. A GitHub link will be returned in the output of the `git push` command. Open this link in a browser to create the pull request.
 
-    ```bash
-    git add <path-to-my-modified-file> # To add all modified files, you can use: git add .
-    git commit -m "my-updates"
-    git push <name-of-my-fork>
-    ```
+    The documentation is built as part of the checks on pull request, along with spell checking and linting. Scroll to the bottom of your pull request to view all the checks.
 
-    After pushing, you will get a GitHub link in the terminal output. Copy this link and paste it into a
-    browser to create your PR.
+    Verify that the linking and spell checking have passed, and that the documentation was built successfully. New words or acronyms can be added to the [wordlist file](https://github.com/ROCm/rocm-docs-core/blob/develop/.wordlist.txt) as needed.
 
-## Create an issue
+    The Read The Docs build of your pull request can be accessed by clicking on the Details link next to the Read The Docs build check. Verify that your changes are in the build and look as expected.
 
-1. To create a new GitHub issue, select the 'Issues' tab in the appropriate repository
-  (e.g., https://github.com/ROCm/ROCm/issues).
-2. Use the search bar to make sure the issue doesn't already exist.
-3. If your issue is not already listed, select the green 'New issue' button to the right of the page. Select
-  the type of issue and fill in the resulting template.
+    Your pull request will be reviewed by a member of the ROCm documentation team.
 
-### General issue guidelines
+See the [GitHub documentation](https://docs.github.com/en) for information on how to fork and clone a repository, and how to create and push a local branch.
 
-* Use your best judgement for issue creation. If your issue is already listed, upvote the issue and
-  comment or post to provide additional details, such as how you reproduced this issue.
-* If you're not sure if your issue is the same, err on the side of caution and file your issue.
-  You can add a comment to include the issue number (and link) for the similar issue. If we evaluate
-  your issue as being the same as the existing issue, we'll close the duplicate.
-* If your issue doesn't exist, use the issue template to file a new issue.
-  * When filing an issue, be sure to provide as much information as possible, including script output so
-    we can collect information about your configuration. This helps reduce the time required to
-    reproduce your issue.
-  * Check your issue regularly, as we may require additional information to successfully reproduce the
-    issue.
-
-## Suggest a new feature
-
-Use the [GitHub Discussion forum](https://github.com/ROCm/ROCm/discussions)
-(Ideas category) to propose new features. Our maintainers are happy to provide direction and
-feedback on feature development.
-
-## Future development workflow
-
-The current ROCm development workflow is GitHub-based. If, in the future, we change this platform,
-the tools and links may change. In this instance, we will update contribution guidelines accordingly.
+```{important}
+By creating a pull request (PR), you agree to allow your contribution to be licensed under the terms of the
+LICENSE.txt file in the corresponding repository. Different repositories can use different licenses.
+```

docs/contribute/doc-structure.md

@@ -1,219 +0,0 @@
-# Documentation structure
-
-Our documentation follows the Pitchfork folder structure. Most documentation files are stored in the
-`/docs` folder. Some special files (such as release, contributing, and changelog) are stored in the root
-(`/`) folder.
-
-All images are stored in the `/docs/data` folder. An image's file path mirrors that of the documentation
-file where it is used.
-
-Our naming structure uses kebab case; for example, `my-file-name.rst`.
-
-## Supported formats and syntax
-
-Our documentation includes both Markdown and RST files. We are gradually transitioning existing
-Markdown to RST in order to more effectively meet our documentation needs. When contributing,
-RST is preferred; if you must use Markdown, use GitHub-flavored Markdown.
-
-We use [Sphinx Design](https://sphinx-design.readthedocs.io/en/latest/index.html) syntax and compile
-our API references using [Doxygen](https://www.doxygen.nl/).
-
-The following table shows some common documentation components and the syntax convention we
-use for each:
-
-<table>
-<tr>
-<th>Component</th>
-<th>RST syntax</th>
-</tr>
-<tr>
-<td>Code blocks</td>
-<td>
-
-```rst
-
-.. code-block:: language-name
-
-  My code block.
-
-
-```
-
-</td>
-</tr>
-<tr>
-<td>Cross-referencing internal files</td>
-<td>
-
-```rst
-
-:doc:`Title <../path/to/file/filename>`
-
-```
-
-</td>
-</tr>
-<tr>
-<td>External links</td>
-<td>
-
-```rst
-
-`link name  <URL>`_
-
-```
-
-</td>
-</tr>
-<tr>
-<tr>
-<td>Headings</td>
-<td>
-
-```rst
-
-******************
-Chapter title (H1)
-******************
-
-Section title (H2)
-===============
-
-Subsection title (H3)
----------------------
-
-Sub-subsection title (H4)
-^^^^^^^^^^^^^^^^^^^^
-
-
-```
-
-</td>
-</tr>
-<tr>
-<td>Images</td>
-<td>
-
-```rst
-
-.. image:: image1.png
-
-```
-
-</td>
-</tr>
-<tr>
-<td>Internal links</td>
-<td>
-
-```rst
-
-1. Add a tag to the section you want to reference:
-
-.. _my-section-tag: section-1
-
-Section 1
-==========
-
-2. Link to your tag:
-
-As shown in :ref:`section-1`.
-
-```
-
-</td>
-</tr>
-<tr>
-<tr>
-<td>Lists</td>
-<td>
-
-```rst
-
-# Ordered (numbered) list item
-
-* Unordered (bulleted) list item
-
-```
-
-</td>
-</tr>
-<tr>
-<tr>
-<td>Math (block)</td>
-<td>
-
-```rst
-
-.. math::
-
-  A = \begin{pmatrix}
-          0.0 & 1.0 & 1.0 & 3.0 \\
-          4.0 & 5.0 & 6.0 & 7.0 \\
-        \end{pmatrix}
-
-```
-
-</td>
-</tr>
-<tr>
-<td>Math (inline)</td>
-<td>
-
-```rst
-
-:math:`2 \times 2 `
-
-```
-
-</td>
-</tr>
-<tr>
-<td>Notes</td>
-<td>
-
-```rst
-
-.. note::
-
-  My note here.
-
-```
-
-</td>
-</tr>
-<tr>
-<td>Tables</td>
-<td>
-
-```rst
-
-.. csv-table::  Optional title here
-  :widths: 30, 70  #optional column widths
-  :header: "entry1 header", "entry2 header"
-
-   "entry1", "entry2"
-
-```
-
-</td>
-</tr>
-</table>
-
-## Language and style
-
-We use the
-[Google developer documentation style guide](https://developers.google.com/style/highlights) to
-guide our content.
-
-Font size and type, page layout, white space control, and other formatting
-details are controlled via
-[rocm-docs-core](https://github.com/ROCm/rocm-docs-core). If you want to notify us
-of any formatting issues, create a pull request in our
-[rocm-docs-core](https://github.com/ROCm/rocm-docs-core) GitHub repository.
-
-## Building our documentation
-
-<!--  % TODO: Fix the link to be able to work at every files  -->
-To learn how to build our documentation, refer to
-[Building documentation](./building.md).

docs/contribute/feedback.md

@@ -4,12 +4,24 @@
   <meta name="keywords" content="documentation, pull request, GitHub, AMD, ROCm">
 </head>
 
-# Providing feedback
+# Providing feedback about the ROCm documentation
 
-Your feedback is welcome. You can provide feedback either through GitHub Discussions or GitHub Issues.
+Feedback about the ROCm documentation is welcome. You can provide feedback about the ROCm documentation either through GitHub Discussions or GitHub Issues.
 
-Use [GitHub Discussions](https://github.com/ROCm/ROCm/discussions) to ask questions, view announcements, and communicate with other members of the community.
+## Participating in discussions through GitHub Discussions
 
-Use [GitHub Issues](https://github.com/ROCm/ROCm/issues) to submit issues you find with ROCm or with the ROCm documentation.
+You can ask questions, view announcements, suggest new features, and communicate with other members of the community through [GitHub Discussions](https://github.com/ROCm/ROCm/discussions).
 
-For information about contributing to the ROCm repository and creating a pull request (PR), see [Contributing](./contributing.md).
+## Submitting issues through GitHub Issues
+
+You can submit issues through [GitHub Issues](https://github.com/ROCm/ROCm/issues).
+
+When creating a new issue, follow the following guidelines:
+
+1. Always do a search to see if the same issue already exists. If the issue already exists, upvote it, and comment or post to provide any additional details you might have.
+2. If you find an issue that is similar to your issue, log your issue, then add a comment that includes a link to the similar issue, as well as its issue number.
+3. Always provide as much information as possible. This helps reduce the time required to reproduce the issue.
+
+After creating your issue, make sure to check it regularly for any requests for additional information.
+
+For information about contributing content to the ROCm documentation, see [Contributing to the ROCm documentation](./contributing.md).

docs/contribute/toolchain.md

@@ -6,60 +6,47 @@
 
 # ROCm documentation toolchain
 
-Our documentation relies on several open source toolchains and sites.
+The ROCm documentation relies on several open source toolchains and sites.
 
-## `rocm-docs-core`
+## rocm-docs-core
 
 [rocm-docs-core](https://github.com/ROCm/rocm-docs-core) is an AMD-maintained
-project that applies customization for our documentation. This project is the tool most ROCm
-repositories use as part of the documentation build. It is also available as a
-[pip package on PyPI](https://pypi.org/project/rocm-docs-core/).
+project that applies customizations for the ROCm documentation. This project is the tool most ROCm repositories use as part of their documentation build pipeline. It is available as a [pip package on PyPI](https://pypi.org/project/rocm-docs-core/).
 
 See the user and developer guides for rocm-docs-core at
 {doc}`rocm-docs-core documentation<rocm-docs-core:index>`.
 
 ## Sphinx
 
-[Sphinx](https://www.sphinx-doc.org/en/master/) is a documentation generator originally used for
-Python. It is now widely used in the open source community.
+[Sphinx](https://www.sphinx-doc.org/en/master/) is a documentation generator originally used for Python. It is now widely used in the open source community.
 
 ### Sphinx External ToC
 
-[Sphinx External ToC](https://sphinx-external-toc.readthedocs.io/en/latest/intro.html) is a Sphinx
-extension used for ROCm documentation navigation. This tool generates a navigation menu on the left
+[Sphinx External ToC](https://sphinx-external-toc.readthedocs.io/en/latest/intro.html) is a Sphinx extension used for ROCm documentation navigation. This tool generates a navigation menu on the left
 based on a YAML file (`_toc.yml.in`) that contains the table of contents.
 
 ### Sphinx-book-theme
 
-[Sphinx-book-theme](https://sphinx-book-theme.readthedocs.io/en/latest/) is a Sphinx theme that
-defines the base appearance for ROCm documentation. ROCm documentation applies some
-customization, such as a custom header and footer on top of the Sphinx Book Theme.
+[Sphinx-book-theme](https://sphinx-book-theme.readthedocs.io/en/latest/) is a Sphinx theme that defines the base appearance for ROCm documentation. ROCm documentation applies some customization, such as a custom header and footer, on top of the Sphinx Book Theme.
 
 ### Sphinx Design
 
-[Sphinx design](https://sphinx-design.readthedocs.io/en/latest/index.html) is a Sphinx extension that
-adds design functionality. ROCm documentation uses Sphinx Design for grids, cards, and synchronized
-tabs.
+[Sphinx design](https://sphinx-design.readthedocs.io/en/latest/index.html) is a Sphinx extension that adds design functionality. ROCm documentation uses Sphinx Design for grids, cards, and synchronized tabs.
 
 ## Doxygen
 
-[Doxygen](https://www.doxygen.nl/) is a documentation generator that extracts information from inline
-code. ROCm projects typically use Doxygen for public API documentation (unless the upstream project
-uses a different tool).
+[Doxygen](https://www.doxygen.nl/) is a documentation generator that extracts information from in-code comments. It is used for API documentation.
 
 ## Breathe
 
-[Breathe](https://www.breathe-doc.org/) is a Sphinx plugin to integrate Doxygen content.
+[Breathe](https://www.breathe-doc.org/) is a Sphinx plugin for integrating Doxygen content.
 
 ## MyST
 
-[Markedly Structured Text (MyST)](https://myst-tools.org/docs/spec) is an extended flavor of
-Markdown ([CommonMark](https://commonmark.org/)) influenced by reStructuredText (RST) and
-Sphinx. It's integrated into ROCm documentation by the Sphinx extension
-[`myst-parser`](https://myst-parser.readthedocs.io/en/latest/).
-A MyST syntax cheat sheet is available on the [Jupyter reference](https://jupyterbook.org/en/stable/reference/cheatsheet.html) site.
+[Markedly Structured Text (MyST)](https://myst-tools.org/docs/spec) is an extended flavor of Markdown ([CommonMark](https://commonmark.org/)) influenced by reStructuredText (rst) and Sphinx. It is integrated into the ROCm documentation with the [`myst-parser`](https://myst-parser.readthedocs.io/en/latest/) Sphinx extension.
+
+See the [MyST syntax cheat sheet](https://jupyterbook.org/en/stable/reference/cheatsheet.html) at the Jupyter Book site.
 
 ## Read the Docs
 
-[Read the Docs](https://docs.readthedocs.io/en/stable/) is the service that builds and hosts the HTML
-documentation generated using Sphinx to our end users.
+[Read the Docs](https://docs.readthedocs.io/en/stable/) is the service that builds and hosts the HTML version of the ROCm documentation.

docs/how-to/llm-fine-tuning-optimization/llm-inference-frameworks.rst

@@ -16,7 +16,7 @@ This section discusses how to implement `vLLM <https://docs.vllm.ai/en/latest>`_
 vLLM inference
 ==============
 
-vLLM is renowned for its paged attention algorithm that can reduce memory consumption and increase throughput thanks to
+vLLM is renowned for its PagedAttention algorithm that can reduce memory consumption and increase throughput thanks to
 its paging scheme. Instead of allocating GPU high-bandwidth memory (HBM) for the maximum output token lengths of the
 models, the paged attention of vLLM allocates GPU HBM dynamically for its actual decoding lengths. This paged attention
 is also effective when multiple requests share the same key and value contents for a large value of beam search or
@@ -139,9 +139,7 @@ Refer to :ref:`mi300x-vllm-optimization` for performance optimization tips.
 
 ROCm provides a prebuilt optimized Docker image for validating the performance of LLM inference with vLLM 
 on the MI300X accelerator. The Docker image includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
-format. For more information, see the guide to 
-`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
-on the ROCm GitHub repository.
+format. For more information, see :doc:`/how-to/performance-validation/mi300x/vllm-benchmark`.
 
 .. _fine-tuning-llms-tgi:
 

docs/how-to/llm-fine-tuning-optimization/model-quantization.rst

@@ -181,7 +181,7 @@ Installing bitsandbytes
       # Clone the github repo
       git clone --recurse https://github.com/ROCm/bitsandbytes.git
       cd bitsandbytes
-      git checkout rocm_enabled
+      git checkout rocm_enabled_multi_backend
 
       # Install dependencies 
       pip install -r requirements-dev.txt

docs/how-to/llm-fine-tuning-optimization/single-gpu-fine-tuning-and-inference.rst

@@ -91,7 +91,7 @@ Setting up the base implementation environment
       # Use -DBNB_ROCM_ARCH to target a specific GPU architecture.
       git clone --recurse https://github.com/ROCm/bitsandbytes.git
       cd bitsandbytes
-      git checkout rocm_enabled
+      git checkout rocm_enabled_multi_backend
       pip install -r requirements-dev.txt
       cmake -DBNB_ROCM_ARCH="gfx942" -DCOMPUTE_BACKEND=hip -S .
       python setup.py install

docs/how-to/performance-validation/mi300x/vllm-benchmark.rst

@@ -0,0 +1,407 @@
+.. meta::
+   :description: Learn how to validate LLM inference performance on MI300X accelerators using AMD MAD and the unified
+                 ROCm Docker image.
+   :keywords: model, MAD, automation, dashboarding, validate
+
+***********************************************************
+LLM inference performance validation on AMD Instinct MI300X
+***********************************************************
+
+.. _vllm-benchmark-unified-docker:
+
+The `ROCm vLLM Docker <https://hub.docker.com/r/rocm/vllm/tags>`_ image offers
+a prebuilt, optimized environment designed for validating large language model
+(LLM) inference performance on the AMD Instinct™ MI300X accelerator. This
+ROCm vLLM Docker image integrates vLLM and PyTorch tailored specifically for the
+MI300X accelerator and includes the following components:
+
+* `ROCm 6.2.1 <https://github.com/ROCm/ROCm>`_
+
+* `vLLM 0.6.4 <https://docs.vllm.ai/en/latest>`_
+
+* `PyTorch 2.5.0 <https://github.com/pytorch/pytorch>`_
+
+* Tuning files (in CSV format)
+
+With this Docker image, you can quickly validate the expected inference
+performance numbers on the MI300X accelerator. This topic also provides tips on
+optimizing performance with popular AI models.
+
+.. hlist::
+   :columns: 6
+
+   * Llama 3.1 8B
+
+   * Llama 3.1 70B
+
+   * Llama 3.1 405B
+
+   * Llama 2 7B
+
+   * Llama 2 70B
+
+   * Mixtral 8x7B
+
+   * Mixtral 8x22B
+
+   * Mixtral 7B
+
+   * Qwen2 7B
+
+   * Qwen2 72B
+
+   * JAIS 13B
+
+   * JAIS 30B
+
+.. _vllm-benchmark-vllm:
+
+.. note::
+
+   vLLM is a toolkit and library for LLM inference and serving. AMD implements
+   high-performance custom kernels and modules in vLLM to enhance performance.
+   See :ref:`fine-tuning-llms-vllm` and :ref:`mi300x-vllm-optimization` for
+   more information.
+
+Getting started
+===============
+
+Use the following procedures to reproduce the benchmark results on an
+MI300X accelerator with the prebuilt vLLM Docker image.
+
+.. _vllm-benchmark-get-started:
+
+1. Disable NUMA auto-balancing.
+
+   To optimize performance, disable automatic NUMA balancing. Otherwise, the GPU
+   might hang until the periodic balancing is finalized. For more information,
+   see :ref:`AMD Instinct MI300X system optimization <mi300x-disable-numa>`.
+
+   .. code-block:: shell
+
+      # disable automatic NUMA balancing
+      sh -c 'echo 0 > /proc/sys/kernel/numa_balancing'
+      # check if NUMA balancing is disabled (returns 0 if disabled)
+      cat /proc/sys/kernel/numa_balancing
+      0
+
+2. Download the :ref:`ROCm vLLM Docker image <vllm-benchmark-unified-docker>`.
+
+   Use the following command to pull the Docker image from Docker Hub.
+
+   .. code-block:: shell
+
+      docker pull rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4
+
+Once setup is complete, you can choose between two options to reproduce the
+benchmark results:
+
+-  :ref:`MAD-integrated benchmarking <vllm-benchmark-mad>`
+
+-  :ref:`Standalone benchmarking <vllm-benchmark-standalone>`
+
+.. _vllm-benchmark-mad:
+
+MAD-integrated benchmarking
+===========================
+
+Clone the ROCm Model Automation and Dashboarding (`<https://github.com/ROCm/MAD>`__) repository to a local
+directory and install the required packages on the host machine.
+
+.. code-block:: shell
+
+   git clone https://github.com/ROCm/MAD
+   cd MAD
+   pip install -r requirements.txt
+
+Use this command to run a performance benchmark test of the Llama 3.1 8B model
+on one GPU with ``float16`` data type in the host machine.
+
+.. code-block:: shell
+
+   export MAD_SECRETS_HFTOKEN="your personal Hugging Face token to access gated models"
+   python3 tools/run_models.py --tags pyt_vllm_llama-3.1-8b --keep-model-dir --live-output --timeout 28800
+
+ROCm MAD launches a Docker container with the name
+``container_ci-pyt_vllm_llama-3.1-8b``. The latency and throughput reports of the
+model are collected in the following path: ``~/MAD/reports_float16/``.
+
+Although the following models are preconfigured to collect latency and
+throughput performance data, you can also change the benchmarking parameters.
+Refer to the :ref:`Standalone benchmarking <vllm-benchmark-standalone>` section.
+
+Available models
+----------------
+
+.. hlist::
+   :columns: 3
+
+   * ``pyt_vllm_llama-3.1-8b``
+
+   * ``pyt_vllm_llama-3.1-70b``
+
+   * ``pyt_vllm_llama-3.1-405b``
+
+   * ``pyt_vllm_llama-2-7b``
+
+   * ``pyt_vllm_llama-2-70b``
+
+   * ``pyt_vllm_mixtral-8x7b``
+
+   * ``pyt_vllm_mixtral-8x22b``
+
+   * ``pyt_vllm_mistral-7b``
+
+   * ``pyt_vllm_qwen2-7b``
+
+   * ``pyt_vllm_qwen2-72b``
+
+   * ``pyt_vllm_jais-13b``
+
+   * ``pyt_vllm_jais-30b``
+
+   * ``pyt_vllm_llama-3.1-8b_fp8``
+
+   * ``pyt_vllm_llama-3.1-70b_fp8``
+
+   * ``pyt_vllm_llama-3.1-405b_fp8``
+
+   * ``pyt_vllm_mixtral-8x7b_fp8``
+
+   * ``pyt_vllm_mixtral-8x22b_fp8``
+
+.. _vllm-benchmark-standalone:
+
+Standalone benchmarking
+=======================
+
+You can run the vLLM benchmark tool independently by starting the
+:ref:`Docker container <vllm-benchmark-get-started>` as shown in the following
+snippet.
+
+.. code-block::
+
+   docker pull rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4
+   docker run -it --device=/dev/kfd --device=/dev/dri --group-add video --shm-size 128G --security-opt seccomp=unconfined --security-opt apparmor=unconfined --cap-add=SYS_PTRACE -v $(pwd):/workspace --env HUGGINGFACE_HUB_CACHE=/workspace --name vllm_v0.6.4 rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4
+
+In the Docker container, clone the ROCm MAD repository and navigate to the
+benchmark scripts directory at ``~/MAD/scripts/vllm``.
+
+.. code-block::
+
+   git clone https://github.com/ROCm/MAD
+   cd MAD/scripts/vllm
+
+Command
+-------
+
+To start the benchmark, use the following command with the appropriate options.
+See :ref:`Options <vllm-benchmark-standalone-options>` for the list of
+options and their descriptions.
+
+.. code-block:: shell
+
+   ./vllm_benchmark_report.sh -s $test_option -m $model_repo -g $num_gpu -d $datatype
+
+See the :ref:`examples <vllm-benchmark-run-benchmark>` for more information.
+
+.. note::
+
+   The input sequence length, output sequence length, and tensor parallel (TP) are
+   already configured. You don't need to specify them with this script.
+
+.. note::
+
+   If you encounter the following error, pass your access-authorized Hugging
+   Face token to the gated models.
+
+   .. code-block:: shell
+
+      OSError: You are trying to access a gated repo.
+
+      # pass your HF_TOKEN
+      export HF_TOKEN=$your_personal_hf_token
+
+.. _vllm-benchmark-standalone-options:
+
+Options
+-------
+
+.. list-table::
+   :header-rows: 1
+   :align: center
+
+   * - Name
+     - Options
+     - Description
+
+   * - ``$test_option``
+     - latency
+     - Measure decoding token latency
+
+   * -
+     - throughput
+     - Measure token generation throughput
+
+   * -
+     - all
+     - Measure both throughput and latency
+
+   * - ``$model_repo``
+     - ``meta-llama/Meta-Llama-3.1-8B-Instruct``
+     - Llama 3.1 8B
+
+   * - (``float16``)
+     - ``meta-llama/Meta-Llama-3.1-70B-Instruct``
+     - Llama 3.1 70B
+
+   * -
+     - ``meta-llama/Meta-Llama-3.1-405B-Instruct``
+     - Llama 3.1 405B
+
+   * -
+     - ``meta-llama/Llama-2-7b-chat-hf``
+     - Llama 2 7B
+
+   * -
+     - ``meta-llama/Llama-2-70b-chat-hf``
+     - Llama 2 70B
+
+   * -
+     - ``mistralai/Mixtral-8x7B-Instruct-v0.1``
+     - Mixtral 8x7B
+
+   * -
+     - ``mistralai/Mixtral-8x22B-Instruct-v0.1``
+     - Mixtral 8x22B
+
+   * -
+     - ``mistralai/Mistral-7B-Instruct-v0.3``
+     - Mixtral 7B
+
+   * -
+     - ``Qwen/Qwen2-7B-Instruct``
+     - Qwen2 7B
+
+   * -
+     - ``Qwen/Qwen2-72B-Instruct``
+     - Qwen2 72B
+
+   * -
+     - ``core42/jais-13b-chat``
+     - JAIS 13B
+
+   * -
+     - ``core42/jais-30b-chat-v3``
+     - JAIS 30B
+
+   * - ``$model_repo``
+     - ``amd/Meta-Llama-3.1-8B-Instruct-FP8-KV``
+     - Llama 3.1 8B
+
+   * - (``float8``)
+     - ``amd/Meta-Llama-3.1-70B-Instruct-FP8-KV``
+     - Llama 3.1 70B
+
+   * -
+     - ``amd/Meta-Llama-3.1-405B-Instruct-FP8-KV``
+     - Llama 3.1 405B
+
+   * -
+     - ``amd/Mixtral-8x7B-Instruct-v0.1-FP8-KV``
+     - Mixtral 8x7B
+
+   * -
+     - ``amd/Mixtral-8x22B-Instruct-v0.1-FP8-KV``
+     - Mixtral 8x22B
+
+   * - ``$num_gpu``
+     - 1 or 8
+     - Number of GPUs
+
+   * - ``$datatype``
+     - ``float16`` or ``float8``
+     - Data type
+
+.. _vllm-benchmark-run-benchmark:
+
+Running the benchmark on the MI300X accelerator
+-----------------------------------------------
+
+Here are some examples of running the benchmark with various options.
+See :ref:`Options <vllm-benchmark-standalone-options>` for the list of
+options and their descriptions.
+
+Example 1: latency benchmark
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ 
+Use this command to benchmark the latency of the Llama 3.1 8B model on one GPU with the ``float16`` and ``float8`` data types.
+
+.. code-block::
+
+   ./vllm_benchmark_report.sh -s latency -m meta-llama/Meta-Llama-3.1-8B-Instruct -g 1 -d float16
+   ./vllm_benchmark_report.sh -s latency -m amd/Meta-Llama-3.1-8B-Instruct-FP8-KV -g 1 -d float8
+
+Find the latency reports at:
+
+- ``./reports_float16/summary/Meta-Llama-3.1-8B-Instruct_latency_report.csv``
+
+- ``./reports_float8/summary/Meta-Llama-3.1-8B-Instruct-FP8-KV_latency_report.csv``
+
+Example 2: throughput benchmark
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Use this command to benchmark the throughput of the Llama 3.1 8B model on one GPU with the ``float16`` and ``float8`` data types.
+
+.. code-block:: shell
+
+   ./vllm_benchmark_report.sh -s throughput -m meta-llama/Meta-Llama-3.1-8B-Instruct -g 1 -d float16
+   ./vllm_benchmark_report.sh -s throughput -m amd/Meta-Llama-3.1-8B-Instruct-FP8-KV -g 1 -d float8
+
+Find the throughput reports at:
+
+- ``./reports_float16/summary/Meta-Llama-3.1-8B-Instruct_throughput_report.csv``
+
+- ``./reports_float8/summary/Meta-Llama-3.1-8B-Instruct-FP8-KV_throughput_report.csv``
+
+.. raw:: html
+
+   <style>
+   mjx-container[jax="CHTML"][display="true"] {
+       text-align: left;
+       margin: 0;
+   }
+   </style>
+
+.. note::
+
+   Throughput is calculated as:
+
+   - .. math:: throughput\_tot = requests \times (\mathsf{\text{input lengths}} + \mathsf{\text{output lengths}}) / elapsed\_time
+
+   - .. math:: throughput\_gen = requests \times \mathsf{\text{output lengths}} / elapsed\_time
+
+Further reading
+===============
+
+- For application performance optimization strategies for HPC and AI workloads,
+  including inference with vLLM, see :doc:`/how-to/tuning-guides/mi300x/workload`.
+
+- To learn more about the options for latency and throughput benchmark scripts,
+  see `<https://github.com/ROCm/vllm/tree/main/benchmarks>`_.
+
+- To learn more about system settings and management practices to configure your system for
+  MI300X accelerators, see :doc:`/how-to/system-optimization/mi300x`.
+
+- To learn how to run LLM models from Hugging Face or your own model, see
+  :doc:`Using ROCm for AI </how-to/rocm-for-ai/index>`.
+
+- To learn how to optimize inference on LLMs, see
+  :doc:`Fine-tuning LLMs and inference optimization </how-to/llm-fine-tuning-optimization/index>`.
+
+- For a list of other ready-made Docker images for ROCm, see the
+  :doc:`Docker image support matrix <rocm-install-on-linux:reference/docker-image-support-matrix>`.
+
+- To compare with the previous version of the ROCm vLLM Docker image for performance validation, refer to
+  `LLM inference performance validation on AMD Instinct MI300X (ROCm 6.2.0) <https://rocm.docs.amd.com/en/docs-6.2.0/how-to/performance-validation/mi300x/vllm-benchmark.html>`_.
+

docs/how-to/system-optimization/mi100.md

@@ -342,8 +342,8 @@ If SMT is enabled by setting "CCD/Core/Thread Enablement > SMT Control" to
 (logical) cores of the system:
 
 * In the server BIOS, set IOMMU to "Enabled".
-* When configuring the Grub boot loader, add the following arguments for the
-  Linux kernel: `amd_iommu=on iommu=pt`
+* When configuring the Grub boot loader, add the following argument for the
+  Linux kernel: `iommu=pt`
 * Update Grub to use the modified configuration:
 
   ```shell
@@ -355,7 +355,7 @@ If SMT is enabled by setting "CCD/Core/Thread Enablement > SMT Control" to
 
   ```none
   [...]
-  [   0.000000] Kernel command line: [...] amd_iommu=on iommu=pt
+  [   0.000000] Kernel command line: [...] iommu=pt
      [...]
   ```
 

docs/how-to/system-optimization/mi200.md

@@ -327,8 +327,8 @@ If SMT is enabled by setting "CCD/Core/Thread Enablement > SMT Control" to
 (logical) cores of the system:
 
 * In the server BIOS, set IOMMU to "Enabled".
-* When configuring the Grub boot loader, add the following arguments for the
-  Linux kernel: `amd_iommu=on iommu=pt`
+* When configuring the Grub boot loader, add the following argument for the
+  Linux kernel: `iommu=pt`
 * Update Grub to use the modified configuration:
 
   ```shell
@@ -340,7 +340,7 @@ If SMT is enabled by setting "CCD/Core/Thread Enablement > SMT Control" to
 
   ```none
   [...]
-  [   0.000000] Kernel command line: [...] amd_iommu=on iommu=pt
+  [   0.000000] Kernel command line: [...] iommu=pt
      [...]
   ```
 

docs/how-to/system-optimization/mi300x.rst

@@ -299,7 +299,7 @@ For a system that has AMD host CPUs add this to ``GRUB_CMDLINE_LINUX``:
 
 .. code-block:: text
 
-   amd_iommu=on iommu=pt
+   iommu=pt
 
 Otherwise, if the system has Intel host CPUs add this instead to
 ``GRUB_CMDLINE_LINUX``:
@@ -500,7 +500,7 @@ If SMT is enabled by setting ``CCD/Core/Thread Enablement > SMT Control`` to
 
 #. In the server BIOS, set IOMMU to ``Enabled``.
 
-#. When configuring the GRUB boot loader, add the following arguments for the Linux kernel: ``amd_iommu=on iommu=pt``.
+#. When configuring the GRUB boot loader, add the following argument for the Linux kernel: ``iommu=pt``.
 
 #. Update GRUB.
 
@@ -515,7 +515,7 @@ If SMT is enabled by setting ``CCD/Core/Thread Enablement > SMT Control`` to
 .. code-block:: shell
 
    [...]
-   [   0.000000] Kernel command line: [...] amd_iommu=on iommu=pt
+   [   0.000000] Kernel command line: [...] iommu=pt
    [...]
 
 Once the system is properly configured, ROCm software can be

docs/how-to/system-optimization/w6000-v620.md

@@ -111,7 +111,7 @@ sudo virsh net-start default /*to enable Virtual network by default
 Enable input-output memory management unit (IOMMU) in GRUB settings by adding the following line to `/etc/default/grub`:
 
 ```none
-GRUB_CMDLINE_LINUX_DEFAULT="quiet splash amd_iommu=on" for AMD CPU
+GRUB_CMDLINE_LINUX_DEFAULT="quiet splash" for AMD CPU
 ```
 
 Update grub and reboot

docs/how-to/tuning-guides/mi300x/index.rst

@@ -8,6 +8,8 @@ accelerators. They include detailed instructions on system settings and
 application tuning suggestions to help you fully leverage the capabilities of
 these accelerators, thereby achieving optimal performance.
 
+* :doc:`/how-to/performance-validation/mi300x/vllm-benchmark`
+
 * :doc:`/how-to/tuning-guides/mi300x/system`
 
 * :doc:`/how-to/tuning-guides/mi300x/workload`

docs/how-to/tuning-guides/mi300x/workload.rst

@@ -152,9 +152,7 @@ address any new bottlenecks that may emerge.
 
 ROCm provides a prebuilt optimized Docker image that has everything required to implement
 the tips in this section. It includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
-format. For more information, see the guide to 
-`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
-on the ROCm GitHub repository.
+format. For more information, see :doc:`/how-to/performance-validation/mi300x/vllm-benchmark`.
 
 .. _mi300x-profiling-tools:
 
@@ -378,11 +376,10 @@ Refer to `vLLM documentation <https://docs.vllm.ai/en/latest/models/performance.
 for additional performance tips. :ref:`fine-tuning-llms-vllm` describes vLLM
 usage with ROCm.
 
-ROCm provides a prebuilt optimized Docker image for validating the performance of LLM inference with vLLM 
-on the MI300X accelerator. The Docker image includes ROCm, vLLM, PyTorch, and tuning files in the CSV 
-format. For more information, see the guide to 
-`LLM inference performance validation with vLLM on the AMD Instinct™ MI300X accelerator <https://github.com/ROCm/MAD/blob/develop/benchmark/vllm/README.md>`_ 
-on the ROCm GitHub repository.
+ROCm provides a prebuilt optimized Docker image for validating the performance
+of LLM inference with vLLM on the MI300X accelerator. The Docker image includes
+ROCm, vLLM, PyTorch, and tuning files in the CSV format. For more information,
+see :doc:`/how-to/performance-validation/mi300x/vllm-benchmark`.
 
 Maximize throughput
 -------------------