Update RELEASE.md

Add Optimized entry and removed Resolved Issue for HIP

.azuredevops/ci-builds/aomp-mainline.yml

@@ -0,0 +1,42 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+resources:
+  repositories:
+  - repository: aomp_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp
+    ref: amd-mainline-open
+  - repository: aomp-extras_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/aomp-extras
+    ref: amd-mainline-open
+  - repository: flang_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/flang
+    ref: amd-mainline-open
+  - repository: llvm-project_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/llvm-project
+    ref: amd-mainline-open
+  pipelines:
+  - pipeline: rocr-runtime_pipeline
+    source: \ROCR-Runtime
+    trigger:
+      branches:
+        include:
+        - amd-master
+# this job will only be triggered after successful build sequence of llvm-project and ROCR-Runtime
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/aomp.yml
+    parameters:
+      checkoutRepo: aomp_repo

.azuredevops/ci-builds/aomp-staging.yml

@@ -27,7 +27,10 @@ resources:
   pipelines:
   - pipeline: rocr-runtime_pipeline
     source: \ROCR-Runtime
-    trigger: true
+    trigger:
+      branches:
+        include:
+        - amd-staging
 # this job will only be triggered after successful build sequence of llvm-project and ROCR-Runtime
 
 trigger: none

.azuredevops/components/AMDMIGraphX.yml

@@ -8,30 +8,31 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - git
     - cmake
-    - ninja-build
+    - git
+    - libdnnl-dev
     - libdrm-dev
-    - libnuma-dev
-    - python3-pip
-    - python3-venv
-    - libtbb-dev
-    - nlohmann-json3-dev
     - libmsgpack-dev
-    - libsqlite3-dev
+    - libnuma-dev
     - libprotobuf-dev
+    - libsqlite3-dev
+    - libtbb-dev
+    - ninja-build
+    - nlohmann-json3-dev
     - protobuf-compiler
+    - python3-pip
     - python3-pybind11
+    - python3-venv
 - name: pipModules
   type: object
   default:
     - https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
-    - onnx==1.14.1
-    - numpy==1.21.6
-    - typing==3.7.4
-    - pytest==6.0.1
-    - packaging==23.0
-    - protobuf==3.20.2
+    - onnx>=1.14.1
+    - numpy>=1.21.6
+    - typing>=3.7.4
+    - pytest>=6.0.1
+    - packaging>=23.0
+    - protobuf>=3.20.2
 - name: rocmDependencies
   type: object
   default:
@@ -50,6 +51,26 @@ parameters:
     - rocminfo
     - rocMLIR
     - rocprofiler-register
+    - roctracer
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - aomp-extras
+    - clr
+    - composable_kernel
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - MIOpen
+    - rocm-cmake
+    - ROCR-Runtime
+    - rocBLAS
+    - rocminfo
+    - rocMLIR
+    - rocprofiler-register
+    - roctracer
 
 jobs:
 - job: AMDMIGraphX
@@ -63,8 +84,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -95,17 +114,87 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
-        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
-        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
         -DCMAKE_BUILD_TYPE=Release
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
         -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
-        -DMIGRAPHX_USE_COMPOSABLEKERNEL=OFF
         -DBUILD_TESTING=ON
         -GNinja
-# REFERENCE: https://github.com/ROCm/composable_kernel/issues/782
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: AMDMIGraphX_testing
+  dependsOn: AMDMIGraphX
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: MIGRAPHX_TRACE_BENCHMARKING
+    value: 1
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+# half version should be fixed to 5.6.0
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencySource: fixed
+      fixedComponentName: half
+      fixedPipelineIdentifier: ${{ variables.HALF560_PIPELINE_ID }}
+      skipLibraryLinking: true
+      skipLlvmSymlink: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: CMake@1
+    displayName: MIGraphXTest CMake Flags
+    inputs:
+      cmakeArgs: >-
+        -DCMAKE_BUILD_TYPE=Release
+        -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm/llvm;$(Agent.BuildDirectory)/rocm
+        -DHALF_INCLUDE_DIR=$(Agent.BuildDirectory)/rocm/include
+        -DBUILD_TESTING=ON
+        -DMIGRAPHX_ENABLE_C_API_TEST=ON
+        ..
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: AMDMIGraphX
+      testExecutable: make
+      testParameters: -j$(nproc) check
+      testPublishResults: false

.azuredevops/components/HIP.yml

@@ -82,6 +82,9 @@ jobs:
         -DHIPCC_BIN_DIR=$(Agent.BuildDirectory)/rocm/bin
         -DCLR_BUILD_HIP=ON
         -DCLR_BUILD_OCL=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      artifactName: amd
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: amd

.azuredevops/components/HIPIFY.yml

@@ -97,6 +97,7 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Pipeline.Workspace)/llvm;/usr/local/cuda/targets/x86_64-linux/lib
         -DLLVM_EXTERNAL_LIT=$(Pipeline.Workspace)/llvm-project/llvm/build/bin/llvm-lit
       multithreadFlag: -- -j32
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:

.azuredevops/components/MIOpen.yml

@@ -8,26 +8,29 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - software-properties-common
-    - python3-pip
     - cmake
-    - ninja-build
-    - libsqlite3-dev
-    - libbz2-dev
-    - nlohmann-json3-dev
-    - libgtest-dev
+    - jq
     - libdrm-dev
+    - libsqlite3-dev
+    - libstdc++-12-dev
+    - ninja-build
+    - python3-pip
+    - python3-venv
+    - software-properties-common
+    - zip
+- name: pipModules
+  type: object
+  default:
+    - cget
 - name: rocmDependencies
   type: object
   default:
-    - rocMLIR
+    - half
     - rocRAND
     - rocBLAS
     - hipBLAS
     - hipBLASLt
     - hipBLAS-common
-    - half
-    - composable_kernel
     - rocm-cmake
     - llvm-project
     - ROCR-Runtime
@@ -35,6 +38,22 @@ parameters:
     - clr
     - rocminfo
     - roctracer
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - half
+    - hipBLAS
+    - hipBLAS-common
+    - hipBLASLt
+    - llvm-project
+    - rocBLAS
+    - rocm-cmake
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler-register
+    - rocRAND
+    - roctracer
 
 jobs:
 - job: MIOpen
@@ -48,28 +67,18 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
-  # The default boost library from apt is 1.74, which does not satisfy MIOpen's build requirement (1.79+)
-  # Upgrade boost from apt by following https://launchpad.net/~mhier/+archive/ubuntu/libboost-latest
-  - task: Bash@3
-    displayName: 'update boost version'
-    inputs:
-      targetType: inline
-      script: sudo add-apt-repository ppa:mhier/libboost-latest -y
-  - task: Bash@3
-    displayName: 'install boost'
-    inputs:
-      targetType: inline
-      script: sudo apt-get --yes install libboost1.83-dev libboost-system1.83-dev libboost-filesystem1.83-dev
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
@@ -80,17 +89,112 @@ jobs:
       # manual build case: triggered by ROCm/ROCm repo
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Build and install other dependencies
+    inputs:
+      targetType: inline
+      workingDirectory: $(Build.SourcesDirectory)
+      script: |
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        sed -i '/composable_kernel/d' requirements.txt
+        mkdir -p $(Agent.BuildDirectory)/miopen-deps
+        cmake -P install_deps.cmake --prefix $(Agent.BuildDirectory)/miopen-deps
+        sudo rm -rf /opt/rocm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DMIOPEN_BACKEND=HIP
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/miopen-deps
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DMIOPEN_ENABLE_AI_KERNEL_TUNING=OFF
         -DMIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK=OFF
         -DCMAKE_BUILD_TYPE=Release
         -DBUILD_TESTING=ON
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: MIOpen_testing
+  timeoutInMinutes: 90
+  dependsOn: MIOpen
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Build and install other dependencies
+    inputs:
+      targetType: inline
+      workingDirectory: $(Build.SourcesDirectory)
+      script: |
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        sed -i '/composable_kernel/d' requirements.txt
+        mkdir -p $(Agent.BuildDirectory)/miopen-deps
+        cmake -P install_deps.cmake --prefix $(Agent.BuildDirectory)/miopen-deps
+        sudo rm -rf /opt/rocm
+  - task: CMake@1
+    displayName: 'MIOpen Test CMake Flags'
+    inputs:
+      cmakeArgs: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Build.SourcesDirectory)/bin;$(Agent.BuildDirectory)/miopen-deps
+        -DCMAKE_INSTALL_PREFIX=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DMIOPEN_BACKEND=HIP
+        -DMIOPEN_TEST_FLAGS=" --disable-verification-cache"
+        -DCMAKE_BUILD_TYPE=release
+        -DBUILD_DEV=OFF
+        -DMIOPEN_USE_MLIR=ON
+        -DMIOPEN_GPU_SYNC=OFF
+        ..
+  - task: Bash@3
+    displayName: 'MIOpen Test Build'
+    inputs:
+      targetType: inline
+      script: |
+        cmake --build . --target tests -- -j$(nproc)
+      workingDirectory: $(Build.SourcesDirectory)/build
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: MIOpen
+      testParameters: '-VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml --exclude-regex test_rnn_seq_api'

.azuredevops/components/MIVisionX.yml

@@ -10,39 +10,30 @@ parameters:
   default:
     - cmake
     - ninja-build
-    - git
     - wget
     - unzip
+    - libstdc++-12-dev
     - pkg-config
-    - protobuf-compiler
-    - libprotoc-dev
     - ffmpeg
     - libavcodec-dev
     - libavformat-dev
     - libavutil-dev
     - libswscale-dev
-    - build-essential
-    - libgtk2.0-dev
-    - libavcodec-dev
-    - libavformat-dev
-    - libswscale-dev
-    - libtbb2
-    - libtbb-dev
-    - libjpeg-dev
-    - libpng-dev
-    - libtiff-dev
-    - libdc1394-dev
     - libgmp-dev
+    - libomp-dev
     - libopencv-dev
+    - protobuf-compiler
+    - libprotoc-dev
 - name: pipModules
   type: object
   default:
-    - future==0.18.2
+    - future==1.0.0
     - pytz==2022.1
-    - numpy==1.21
+    - numpy==1.23
     - google==3.0.0
     - protobuf==3.12.4
     - onnx==1.12.0
+    - nnef==1.0.7
 - name: rocmDependencies
   type: object
   default:
@@ -86,8 +77,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -114,13 +103,16 @@ jobs:
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DROCM_DEP_ROCMCORE=ON
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: MIVisionX_testing
   dependsOn: MIVisionX
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -162,6 +154,7 @@ jobs:
       mkdir mivisionx-tests
       cd mivisionx-tests
       cmake /opt/rocm/share/mivisionx/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: MIVisionX

.azuredevops/components/ROCR-Runtime.yml

@@ -8,15 +8,21 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - libelf-dev
     - g++
     - libdrm-dev
+    - libelf-dev
     - libnuma-dev
 - name: rocmDependencies
   type: object
   default:
     - llvm-project
     - rocprofiler-register
+- name: rocmTestDependencies
+  type: object
+  default:
+    - llvm-project
+    - rocm_smi_lib
+    - rocprofiler-register
 
 jobs:
 - job: ROCR_Runtime
@@ -49,4 +55,96 @@ jobs:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_SHARED_LIBS=ON
+        -DCMAKE_BUILD_TYPE=Release
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: ROCR_Runtime_testing
+  dependsOn: ROCR_Runtime
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - task: Bash@3
+    displayName: Install libhwloc5
+    inputs:
+      targetType: 'inline'
+      script: |
+        wget http://ftp.us.debian.org/debian/pool/main/h/hwloc/libhwloc5_1.11.12-3_amd64.deb
+        wget http://ftp.us.debian.org/debian/pool/main/h/hwloc/libhwloc-dev_1.11.12-3_amd64.deb
+        sudo apt install -y --allow-downgrades ./libhwloc5_1.11.12-3_amd64.deb ./libhwloc-dev_1.11.12-3_amd64.deb
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
+  - task: Bash@3
+    displayName: Build kfdtest
+    continueOnError: true
+    inputs:
+      targetType: 'inline'
+      workingDirectory: $(Build.SourcesDirectory)/libhsakmt/tests/kfdtest
+      script: |
+        mkdir build && cd build
+        cmake -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm ..
+        make
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: kfdtest
+      testExecutable: BIN_DIR=$(Build.SourcesDirectory)/libhsakmt/tests/kfdtest/build ./run_kfdtest.sh
+      testParameters: '-p core --gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: $(Build.SourcesDirectory)/libhsakmt/tests/kfdtest/scripts
+  - task: Bash@3
+    displayName: Build rocrtst
+    continueOnError: true
+    inputs:
+      targetType: 'inline'
+      workingDirectory: $(Build.SourcesDirectory)/rocrtst/suites/test_common
+      script: |
+        sudo rm -rf $(Agent.BuildDirectory)/external/llvm-project
+        mkdir -p $(Agent.BuildDirectory)/external/llvm-project/clang/lib
+        sudo ln -s $(Agent.BuildDirectory)/rocm/llvm/lib/clang/20/include $(Agent.BuildDirectory)/external/llvm-project/clang/lib/Headers
+        mkdir build && cd build
+        cmake .. \
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm \
+          -DTARGET_DEVICES=$(JOB_GPU_TARGET) \
+          -DROCM_DIR=$(Agent.BuildDirectory)/rocm \
+          -DLLVM_DIR=$(Agent.BuildDirectory)/rocm/llvm/bin \
+          -DOPENCL_DIR=$(Agent.BuildDirectory)/rocm/llvm/bin
+        make
+        make rocrtst_kernels
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocrtst
+      testExecutable: ./rocrtst64
+      testParameters: '--gtest_filter="-rocrtstNeg.Memory_Negative_Tests:rocrtstFunc.Memory_Max_Mem" --gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testDir: $(Build.SourcesDirectory)/rocrtst/suites/test_common/build/$(JOB_GPU_TARGET)

.azuredevops/components/ROCT-Thunk-Interface.yml

@@ -29,4 +29,5 @@ jobs:
     parameters:
       checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/ROCdbgapi.yml

@@ -50,4 +50,5 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/ROCgdb.yml

@@ -32,6 +32,7 @@ parameters:
 
 jobs:
 - job: ROCgdb
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -91,6 +92,7 @@ jobs:
         --with-rocm-dbgapi=$(Agent.BuildDirectory)/rocm
         LDFLAGS="-Wl,--enable-new-dtags,-rpath=$(Agent.BuildDirectory)/rocm/lib"
       makeCallPrefix: LD_RUN_PATH='${ORIGIN}/../lib'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - task: Bash@3
     displayName: Setup test environment
@@ -100,6 +102,7 @@ jobs:
         sudo rm -rf /opt/rocm
         sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
         echo "##vso[task.prependpath]/opt/rocm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - task: Bash@3
     displayName: check-gdb
     continueOnError: true

.azuredevops/components/ROCmValidationSuite.yml

@@ -13,6 +13,7 @@ parameters:
     - libyaml-cpp-dev
     - libpci-dev
     - libpci3
+    - libgst-dev
     - libgtest-dev
     - git
 - name: rocmDependencies
@@ -40,9 +41,11 @@ parameters:
     - llvm-project
     - rocBLAS
     - rocm_smi_lib
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
     - rocRAND
+    - roctracer
 
 jobs:
 - job: ROCmValidationSuite
@@ -63,8 +66,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -91,13 +92,16 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCPACK_PACKAGING_INSTALL_PREFIX=$(Build.BinariesDirectory)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: ROCmValidationSuite_testing
   dependsOn: ROCmValidationSuite
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -131,6 +135,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: ROCmValidationSuite

.azuredevops/components/Tensile.yml

@@ -0,0 +1,177 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - python3-pip
+    - cmake
+    - libmsgpack-dev
+    - libboost-program-options-dev
+- name: pipModules
+  type: object
+  default:
+    - tox
+    - pytest
+- name: rocmDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - rocm_smi_lib
+    - rocprofiler-register
+    - ROCR-Runtime
+
+jobs:
+- job: Tensile
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+  - task: Bash@3
+    displayName: Create wheel file
+    inputs:
+      targetType: inline
+      script: python3 setup.py bdist_wheel
+      workingDirectory: $(Build.SourcesDirectory)
+    env:
+      ROCM_PATH: $(Agent.BuildDirectory)/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.SourcesDirectory)/dist
+      contentsString: '*.whl'
+      targetDir: $(Build.ArtifactStagingDirectory)
+      clean: false
+  - task: PublishPipelineArtifact@1
+    displayName: 'wheel file Publish'
+    retryCountOnTaskFailure: 3
+    inputs:
+      targetPath: $(Build.ArtifactStagingDirectory)
+
+- job: Tensile_testing
+  timeoutInMinutes: 90
+  dependsOn: Tensile
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Wheel Files'
+    inputs:
+      itemPattern: '**/*.whl'
+      targetPath: $(Agent.BuildDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: pip install
+    inputs:
+      targetType: inline
+      script: find -name *.whl -exec pip install {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Setup test environment
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: Add Python site-packages binaries to path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.prependpath]$USER_BASE/bin"
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - task: Bash@3
+    displayName: tox test
+    inputs:
+      targetType: inline
+      script: tox run -v -e ci -- -m pre_checkin
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Remove Python site-packages binaries from path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$USER_BASE/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"

.azuredevops/components/amdsmi.yml

@@ -31,11 +31,12 @@ jobs:
     parameters:
       extraBuildFlags: >-
         -DBUILD_TESTS=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
 
 - job: amdsmi_testing
   dependsOn: amdsmi
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -53,6 +54,9 @@ jobs:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: amdsmi

.azuredevops/components/aomp-extras.yml

@@ -54,4 +54,5 @@ jobs:
         -DAOMP_VERSION_STRING=$(LATEST_RELEASE_TAG)
         -GNinja
       installDir: $(Build.BinariesDirectory)/llvm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/aomp.yml

@@ -109,11 +109,12 @@ jobs:
 # for the compilation and installation to go through.
   - script: |
       sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-      mkdir -p $(Build.BinariesDirectory)/bin
-      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang $(Build.BinariesDirectory)/bin/clang
-      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang++ $(Build.BinariesDirectory)/bin/clang++
-      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/llvm-config $(Build.BinariesDirectory)/bin/llvm-config
+      mkdir -p $(Build.BinariesDirectory)/lib/llvm/bin
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang $(Build.BinariesDirectory)/lib/llvm/bin/clang
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/clang++ $(Build.BinariesDirectory)/lib/llvm/bin/clang++
+      ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/llvm-config $(Build.BinariesDirectory)/lib/llvm/bin/llvm-config
       ln -s $(Agent.BuildDirectory)/rocm/llvm $(Build.BinariesDirectory)/llvm
+      ls -1R $(Build.BinariesDirectory)
     displayName: Extra build environment setup
 # We follow the sequence described in the aomp repo instructions
 # https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL.md
@@ -126,7 +127,7 @@ jobs:
 # method leads to a giant build log compared to separate logs per script call.
 #
 # Components compiled and the order for non-standalone build found at
-# https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L135-L143
+# https://github.com/ROCm/aomp/blob/aomp-dev/bin/build_aomp.sh#L135-L142
   - task: Bash@3
     displayName: Build Prereq
     inputs:
@@ -176,7 +177,6 @@ jobs:
       AOMP_USE_NINJA: 1
       ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
@@ -196,12 +196,11 @@ jobs:
       AOMP_USE_NINJA: 1
       ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
       INSTALL_OPENMP: $(Build.BinariesDirectory)
-  - script: ln -s $(Build.BinariesDirectory)/include/omp.h $(Build.SourcesDirectory)/llvm-project/llvm/include/omp.h
+  - script: ln -s $(Build.BinariesDirectory)/lib/llvm/include/omp.h $(Build.SourcesDirectory)/llvm-project/llvm/include/omp.h
     displayName: Link omp header
   - task: Bash@3
     displayName: Build offload
@@ -237,15 +236,14 @@ jobs:
       AOMP_USE_NINJA: 1
       ALTAOMP: $(Agent.BuildDirectory)/rocm/llvm
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
       INSTALL_OPENMP: $(Build.BinariesDirectory)
   - task: Bash@3
-    displayName: Build flang-legacy
+    displayName: Build llvm-classic
     inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-legacy.sh
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_llvm-classic.sh
     env:
       AOMP_REPOS: $(Build.SourcesDirectory)
       AOMP_PROJECT_REPO_NAME: llvm-project
@@ -254,14 +252,13 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
   - task: Bash@3
-    displayName: Install flang-legacy
+    displayName: Install llvm-classic
     inputs:
-      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-legacy.sh
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_llvm-classic.sh
       arguments: install
     env:
       AOMP_REPOS: $(Build.SourcesDirectory)
@@ -275,6 +272,37 @@ jobs:
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Build flang-classic
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-classic.sh
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
+  - task: Bash@3
+    displayName: Install flang-classic
+    inputs:
+      filePath: $(Build.SourcesDirectory)/aomp/bin/build_flang-classic.sh
+      arguments: install
+    env:
+      AOMP_REPOS: $(Build.SourcesDirectory)
+      AOMP_PROJECT_REPO_NAME: llvm-project
+      AOMP_STANDALONE_BUILD: 0
+      AOMP_BUILD_SANITIZER: 0
+      AOMP_BUILD_DEBUG: 0
+      AOMP_USE_NINJA: 1
+      INSTALL_PREFIX: $(Build.BinariesDirectory)
+      LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
+      AOMP: $(Build.BinariesDirectory)
+      AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Build pgmath
     inputs:
@@ -287,11 +315,9 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Install pgmath
     inputs:
@@ -305,11 +331,9 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Build flang
     inputs:
@@ -322,11 +346,9 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Install flang
     inputs:
@@ -340,11 +362,9 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Build flang_runtime
     inputs:
@@ -357,11 +377,9 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
   - task: Bash@3
     displayName: Install flang_runtime
     inputs:
@@ -375,24 +393,36 @@ jobs:
       AOMP_BUILD_DEBUG: 0
       AOMP_USE_NINJA: 1
       INSTALL_PREFIX: $(Build.BinariesDirectory)
-      CMAKE_INSTALL_PREFIX: $(Build.BinariesDirectory)
       LLVM_PROJECT_ROOT: $(Build.SourcesDirectory)/llvm-project
       AOMP: $(Build.BinariesDirectory)
       AOMP_INSTALL_DIR: $(Build.BinariesDirectory)
-      INSTALL_FLANG: $(Build.BinariesDirectory)
 # Clean up build environment before publish artifact
   - script: |
-      rm $(Build.BinariesDirectory)/bin/clang
-      rm $(Build.BinariesDirectory)/bin/clang++
-      rm $(Build.BinariesDirectory)/bin/llvm-config
+      rm $(Build.BinariesDirectory)/lib/llvm/bin/clang
+      rm $(Build.BinariesDirectory)/lib/llvm/bin/clang++
+      rm $(Build.BinariesDirectory)/lib/llvm/bin/llvm-config
+      rm $(Build.BinariesDirectory)/lib/llvm/bin/flang
       rm $(Build.BinariesDirectory)/llvm
     displayName: Remove temporary symbolic links
+# aomp scripts changed where files get installed in scripts, copy to expected location
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.BinariesDirectory)/lib/llvm
+      targetDir: $(Build.ArtifactStagingDirectory)
+# Remove temporary directory used to deal with expected paths of scripts
+  - script: |
+      rm -rf $(Build.BinariesDirectory)/lib/llvm
+    displayName: Remove temporary directories
 # Copy the files to artifact staging temporarily to clean up binaries directory
 # and then copy files back to llvm subdirectory in the cleaned up binaries directory
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
       sourceDir: $(Build.BinariesDirectory)
       targetDir: $(Build.ArtifactStagingDirectory)
+      clean: false
+  - script: |
+      ln -s $(Build.ArtifactStagingDirectory)/bin/flang-classic $(Build.ArtifactStagingDirectory)/bin/flang
+    displayName: Recreate flang symlink
   - task: DeleteFiles@1
     displayName: 'Cleanup Binaries Directory'
     inputs:
@@ -409,11 +439,12 @@ jobs:
       SourceFolder: $(Build.ArtifactStagingDirectory)
       Contents: '/**/*'
       RemoveDotFiles: true
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
 
 - job: aomp_testing
   dependsOn: aomp
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -472,6 +503,7 @@ jobs:
       Contents: FileCheck
       TargetFolder: $(Agent.BuildDirectory)/rocm/share/openmp-extras/tests/bin
       retryCount: 3
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - task: Bash@3
     displayName: Test AOMP
     continueOnError: true

.azuredevops/components/composable_kernel.yml

@@ -48,8 +48,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -90,16 +88,21 @@ jobs:
         -DCMAKE_C_COMPILER_LAUNCHER=ccache
         -DCMAKE_HIP_FLAGS="-Wno-missing-include-dirs"
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCK_BUILD_JIT_LIB=ON
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
         -DCMAKE_BUILD_TYPE=Release
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: composable_kernel_testing
   dependsOn: composable_kernel
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -135,6 +138,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - task: Bash@3
     displayName: Iterate through test scripts
     inputs:

.azuredevops/components/copyHIP.yml

@@ -25,8 +25,9 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-download.yml
     parameters:
       componentName: HIP
-      pipelineId: $(hip-pipeline-id)
+      pipelineId: $(HIP_PIPELINE_ID)
   - template:  ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
       sourceDir: $(Agent.BuildDirectory)/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/half.yml

@@ -52,6 +52,7 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   # only run test03 because test11 has too many test cases, taking way too long
   - task: Bash@3

.azuredevops/components/hip-tests.yml

@@ -52,8 +52,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -84,14 +82,17 @@ jobs:
         -DHIP_PATH=$(Agent.BuildDirectory)/rocm
         -DOFFLOAD_ARCH_STR="--offload-arch=$(JOB_GPU_TARGET)"
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hip_tests_testing
-  timeoutInMinutes: 180
+  timeoutInMinutes: 240
   dependsOn: hip_tests
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -130,15 +131,17 @@ jobs:
     inputs:
       targetType: inline
       script: |
+        sudo rm -rf /opt/rocm
         sudo mkdir -p /opt/rocm/bin
         sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rocm_agent_enumerator /opt/rocm/bin/rocm_agent_enumerator
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hip_tests
       testDir: $(Agent.BuildDirectory)/rocm/share/hip
-      reloadAMDGPU: true
   - task: Bash@3
     displayName: Clean up symlink
+    condition: always()
     inputs:
       targetType: inline
       script: sudo rm -rf /opt/rocm

.azuredevops/components/hipBLAS-common.yml

@@ -29,7 +29,7 @@ jobs:
   - name: ROCM_PATH
     value: $(Agent.BuildDirectory)/rocm
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -59,4 +59,5 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/hipBLAS.yml

@@ -16,6 +16,10 @@ parameters:
     - libgtest-dev
     - wget
     - python3-pip
+- name: pipModules
+  type: object
+  default:
+    - pyyaml
 - name: rocmDependencies
   type: object
   default:
@@ -59,12 +63,11 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
@@ -92,13 +95,16 @@ jobs:
         -DBUILD_CLIENTS_SAMPLES=OFF
         -DCPACK_SET_DESTDIR=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipBLAS_testing
   dependsOn: hipBLAS
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -114,6 +120,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
     parameters:
@@ -132,6 +139,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipBLAS

.azuredevops/components/hipBLASLt.yml

@@ -22,6 +22,7 @@ parameters:
   type: object
   default:
     - joblib
+    - packaging
 - name: rocmDependencies
   type: object
   default:
@@ -74,8 +75,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -143,6 +142,9 @@ jobs:
         -DCMAKE_PREFIX_PATH="$(Agent.BuildDirectory)/rocm"
         -DBUILD_CLIENTS_TESTS=ON
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
@@ -150,7 +152,7 @@ jobs:
 
 - job: hipBLASLt_testing
   dependsOn: hipBLASLt
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -166,6 +168,7 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
     parameters:
@@ -184,6 +187,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipBLASLt

.azuredevops/components/hipCUB.yml

@@ -43,8 +43,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -72,13 +70,16 @@ jobs:
         -DBUILD_TEST=ON
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipCUB_testing
   dependsOn: hipCUB
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -112,6 +113,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipCUB

.azuredevops/components/hipFFT.yml

@@ -55,8 +55,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -90,13 +88,16 @@ jobs:
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipFFT_testing
   dependsOn: hipFFT
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -130,6 +131,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipFFT

.azuredevops/components/hipRAND.yml

@@ -45,8 +45,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -76,13 +74,16 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipRAND_testing
   dependsOn: hipRAND
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -116,6 +117,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipRAND

.azuredevops/components/hipSOLVER.yml

@@ -42,6 +42,7 @@ parameters:
     - ROCR-Runtime
     - rocSOLVER
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: hipSOLVER
@@ -56,8 +57,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -96,13 +95,16 @@ jobs:
         -DBUILD_CLIENTS_TESTS=ON
         -DUSE_CUDA=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipSOLVER_testing
   dependsOn: hipSOLVER
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -136,6 +138,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipSOLVER

.azuredevops/components/hipSPARSE.yml

@@ -37,6 +37,7 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: hipSPARSE
@@ -51,8 +52,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -80,6 +79,10 @@ jobs:
         -DBUILD_CLIENTS_TESTS=ON
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      artifactName: hipSPARSE
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: hipSPARSE
@@ -96,7 +99,7 @@ jobs:
 
 - job: hipSPARSE_testing
   dependsOn: hipSPARSE
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -130,6 +133,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipSPARSE

.azuredevops/components/hipSPARSELt.yml

@@ -117,13 +117,16 @@ jobs:
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_CLIENTS_TESTS=ON
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipSPARSELt_testing
   dependsOn: hipSPARSELt
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -151,6 +154,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipSPARSELt

.azuredevops/components/hipTensor.yml

@@ -42,8 +42,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -72,6 +70,9 @@ jobs:
         -DHIPTENSOR_BUILD_TESTS=ON
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
       multithreadFlag: -- -j32
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
@@ -79,7 +80,7 @@ jobs:
 - job: hipTensor_testing
   timeoutInMinutes: 90
   dependsOn: hipTensor
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -113,8 +114,9 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: hipTensor
       testDir: '$(Agent.BuildDirectory)/rocm/bin/hiptensor'
-      testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
+      testParameters: '-E ".*-extended" -VV --output-on-failure --force-new-ctest-process --output-junit test_output.xml'

.azuredevops/components/hipfort.yml

@@ -51,8 +51,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -88,13 +86,16 @@ jobs:
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DBUILD_TESTING=ON
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: hipfort_testing
   dependsOn: hipfort
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -139,6 +140,7 @@ jobs:
         sudo rm -rf /opt/rocm
         sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
       workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - task: Bash@3
     displayName: 'Test hipfort'
     inputs:

.azuredevops/components/llvm-project.yml

@@ -27,6 +27,8 @@ jobs:
   - template: /.azuredevops/variables-global.yml
   - name: HIP_DEVICE_LIB_PATH
     value: '$(Build.BinariesDirectory)/amdgcn/bitcode'
+  - name: HIP_PATH
+    value: '$(Agent.BuildDirectory)/rocm'
   pool: ${{ variables.MEDIUM_BUILD_POOL }}
   workspace:
     clean: all
@@ -124,6 +126,7 @@ jobs:
       componentName: comgr
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH="$(Build.SourcesDirectory)/llvm/build;$(Build.SourcesDirectory)/amd/device-libs/build"
+        -DCOMGR_DISABLE_SPIRV=1
         -DCMAKE_BUILD_TYPE=Release
       cmakeBuildDir: 'amd/comgr/build'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
@@ -139,4 +142,5 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DHIPCC_BACKWARD_COMPATIBILITY=OFF
       cmakeBuildDir: 'amd/hipcc/build'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/omniperf.yml

@@ -58,8 +58,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -92,7 +90,7 @@ jobs:
 
 - job: omniperf_testing
   dependsOn: omniperf
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -150,6 +148,7 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DENABLE_TESTS=ON
         -DINSTALL_TESTS=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: omniperf

.azuredevops/components/omnitrace.yml

@@ -0,0 +1,145 @@
+# largely referenced from: https://github.com/ROCm/omnitrace/blob/main/.github/workflows/ubuntu-jammy.yml
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - autoconf
+    - autotools-dev
+    - bison
+    - build-essential
+    - bzip2
+    - clang
+    - cmake
+    - environment-modules
+    - g++-12
+    - libdrm-dev
+    - libfabric-dev
+    - libiberty-dev
+    - libpapi-dev
+    - libpfm4-dev
+    - libtool
+    - libopenmpi-dev
+    - m4
+    - openmpi-bin
+    - software-properties-common
+    - python3-pip
+    - texinfo
+    - zlib1g-dev
+- name: pipModules
+  type: object
+  default:
+    - numpy
+    - perfetto
+    - dataclasses
+- name: rocmDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - llvm-project
+    - rccl
+    - rocm-core
+    - rocm_smi_lib
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler
+    - rocprofiler-register
+    - roctracer
+
+jobs:
+- job: omnitrace
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+# build flags reference: https://rocm.docs.amd.com/projects/omnitrace/en/latest/install/install.html
+      extraBuildFlags: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DOMNITRACE_BUILD_TESTING=ON
+        -DOMNITRACE_BUILD_DYNINST=ON
+        -DOMNITRACE_BUILD_LIBUNWIND=ON
+        -DDYNINST_BUILD_TBB=ON
+        -DDYNINST_BUILD_ELFUTILS=ON
+        -DDYNINST_BUILD_LIBIBERTY=ON
+        -DDYNINST_BUILD_BOOST=ON
+        -DOMNITRACE_USE_PAPI=ON
+        -DOMNITRACE_USE_MPI=ON
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j32
+  - task: Bash@3
+    displayName: Set up omnitrace env
+    inputs:
+      targetType: inline
+      script: source share/omnitrace/setup-env.sh
+      workingDirectory: build
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: omnitrace
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    condition: always()
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    condition: always()
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rccl.yml

@@ -60,8 +60,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -95,6 +93,9 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/share/rocm/cmake/
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
@@ -102,7 +103,7 @@ jobs:
 - job: rccl_testing
   timeoutInMinutes: 120
   dependsOn: rccl
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -136,6 +137,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rccl

.azuredevops/components/rdc.yml

@@ -19,15 +19,19 @@ parameters:
     - libtool
     - pkg-config
     - libdrm-dev
+    - libnuma-dev
     - libyaml-cpp-dev
 - name: rocmDependencies
   type: object
   default:
     - amdsmi
     - clr
+    - hipBLAS-common
+    - hipBLASLt
     - llvm-project
     - rocBLAS
     - rocm-cmake
+    - rocm-core
     - rocminfo
     - rocm_smi_lib
     - ROCmValidationSuite
@@ -60,8 +64,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -107,13 +109,17 @@ jobs:
         -DBUILD_RVS=ON
         -DBUILD_PROFILER=ON
         -DBUILD_TESTS=ON
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rdc_testing
   dependsOn: rdc
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -131,6 +137,8 @@ jobs:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
     parameters:
       ${{ if eq(parameters.checkoutRef, '') }}:
@@ -156,6 +164,7 @@ jobs:
         sudo ln -s $(Agent.BuildDirectory)/rocm/bin/rdcd /usr/sbin/rdcd
         echo $(Agent.BuildDirectory)/rocm/lib/rdc/grpc/lib | sudo tee /etc/ld.so.conf.d/grpc.conf
         sudo ldconfig -v
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - task: Bash@3
     displayName: Test rdc
     inputs:

.azuredevops/components/rocAL.yml

@@ -53,6 +53,7 @@ parameters:
     - half
     - llvm-project
     - MIVisionX
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
     - rpp
@@ -70,8 +71,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - task: Bash@3
     displayName: 'Register libjpeg-turbo packages'
@@ -151,13 +150,16 @@ jobs:
         -DCMAKE_INSTALL_PREFIX_PYTHON=$Python3_STDARCH
         -DCMAKE_BUILD_TYPE=Release
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocAL_testing
   dependsOn: rocAL
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -220,13 +222,14 @@ jobs:
         mkdir rocAL-tests
         cd rocAL-tests
         cmake $(Agent.BuildDirectory)/rocm/share/rocal/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocAL
       testDir: rocAL-tests
-      reloadAMDGPU: true
   - task: Bash@3
     displayName: Clean up libjpeg-turbo
+    condition: always()
     inputs:
       targetType: inline
       script: |

.azuredevops/components/rocALUTION.yml

@@ -43,6 +43,7 @@ parameters:
     - ROCR-Runtime
     - rocRAND
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: rocALUTION
@@ -59,8 +60,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -91,13 +90,16 @@ jobs:
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocALUTION_testing
   dependsOn: rocALUTION
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -131,6 +133,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocALUTION

.azuredevops/components/rocBLAS.yml

@@ -40,6 +40,7 @@ parameters:
     - aomp-extras
     - hipBLAS-common
     - hipBLASLt
+    - roctracer
 - name: rocmTestDependencies
   type: object
   default:
@@ -52,6 +53,7 @@ parameters:
     - rocprofiler-register
     - rocm_smi_lib
     - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: rocBLAS
@@ -73,8 +75,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -114,13 +114,16 @@ jobs:
         -DBUILD_CLIENTS_SAMPLES=OFF
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocBLAS_testing
   dependsOn: rocBLAS
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -155,6 +158,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocBLAS

.azuredevops/components/rocDecode.yml

@@ -17,7 +17,7 @@ parameters:
     - libavformat-dev
     - libavutil-dev
     - libstdc++-12-dev
-    - libva-dev
+    - libva-amdgpu-dev
     - mesa-amdgpu-va-drivers
     - libdrm-dev
 - name: rocmDependencies
@@ -35,6 +35,7 @@ parameters:
   default:
     - clr
     - llvm-project
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
 
@@ -84,15 +85,18 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DCMAKE_BUILD_TYPE=Release
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
 
 - job: rocDecode_testing
   dependsOn: rocDecode
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+  pool:
+    name: $(JOB_TEST_POOL)
+    demands: firstRenderDeviceAccess
   workspace:
     clean: all
   strategy:
@@ -101,6 +105,18 @@ jobs:
         JOB_GPU_TARGET: gfx942
         JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
+  # Since mesa-amdgpu-multimedia-devel is not directly available from apt, register it
+  - task: Bash@3
+    displayName: 'Register ROCm packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/${{ variables.KEYRING_VERSION }}/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/amdgpu.list
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${{ variables.KEYRING_VERSION }} jammy main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
+        echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+        sudo apt update
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
@@ -128,8 +144,10 @@ jobs:
       mkdir rocDecode-tests
       cd rocDecode-tests
       cmake /opt/rocm/share/rocdecode/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocDecode
       testDir: 'rocDecode-tests'
   - script: sudo rm /opt/rocm
+    condition: always()

.azuredevops/components/rocFFT.yml

@@ -55,8 +55,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -89,13 +87,16 @@ jobs:
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocFFT_testing
   dependsOn: rocFFT
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -129,6 +130,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocFFT

.azuredevops/components/rocJPEG.yml

@@ -0,0 +1,148 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - libdrm-dev
+    - libstdc++-12-dev
+    - libva-amdgpu-dev
+    - mesa-amdgpu-va-drivers
+    - ninja-build
+    - pkg-config
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocm-core
+    - rocprofiler-register
+    - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
+
+jobs:
+- job: rocJPEG
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  steps:
+  # Since mesa-amdgpu-multimedia-devel is not directly available from apt, register it
+  - task: Bash@3
+    displayName: 'Register ROCm packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/${{ variables.KEYRING_VERSION }}/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/amdgpu.list
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${{ variables.KEYRING_VERSION }} jammy main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
+        echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+        sudo apt update
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, 'develop') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, 'develop') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+
+- job: rocJPEG_testing
+  dependsOn: rocJPEG
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    name: $(JOB_TEST_POOL)
+    demands: firstRenderDeviceAccess
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  # Since mesa-amdgpu-multimedia-devel is not directly available from apt, register it
+  - task: Bash@3
+    displayName: 'Register ROCm packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/${{ variables.KEYRING_VERSION }}/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/amdgpu.list
+        echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${{ variables.KEYRING_VERSION }} jammy main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
+        echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+        sudo apt update
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, 'develop') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, 'develop') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, 'develop') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, 'develop') }}:
+        dependencySource: tag-builds
+  # anything in /opt may be persistent across runs
+  # so we need to remove the symlink if it already exists
+  - script: |
+      sudo rm -rf /opt/rocm
+      sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      mkdir rocJPEG-tests
+      cd rocJPEG-tests
+      cmake $(Agent.BuildDirectory)/rocm/share/rocjpeg/test
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocJPEG
+      testDir: 'rocJPEG-tests'
+  - script: sudo rm /opt/rocm
+    condition: always()

.azuredevops/components/rocMLIR.yml

@@ -13,6 +13,7 @@ parameters:
     - git
     - python3-pip
     - libdrm-dev
+    - libstdc++-12-dev
 - name: pipModules
   type: object
   default:
@@ -28,7 +29,7 @@ parameters:
     - ROCR-Runtime
 
 jobs:
-- job: rocMLIR_library
+- job: rocMLIR
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -62,20 +63,24 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DBUILD_FAT_LIBROCKCOMPILER=1
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
 
 # compiling and running test on the test system together
 - job: rocMLIR_testing
+  dependsOn: rocMLIR
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: ${{ variables.GFX942_TEST_POOL }}
+  pool: $(JOB_TEST_POOL)
   workspace:
     clean: all
   strategy:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -105,7 +110,9 @@ jobs:
     displayName: ROCm symbolic link
     inputs:
       targetType: inline
-      script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       installEnabled: false
@@ -117,6 +124,7 @@ jobs:
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DROCM_TEST_CHIPSET=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocMLIR

.azuredevops/components/rocPRIM.yml

@@ -42,8 +42,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -71,13 +69,16 @@ jobs:
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DBUILD_TEST=ON
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocPRIM_testing
   dependsOn: rocPRIM
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -111,6 +112,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocPRIM

.azuredevops/components/rocPyDecode.yml

@@ -8,25 +8,22 @@ parameters:
 - name: aptPackages
   type: object
   default:
-    - python3-pip
     - cmake
-    - ninja-build
-    - pkg-config
     - ffmpeg
     - libavcodec-dev
     - libavformat-dev
     - libavutil-dev
-    - libva-dev
-    - libdrm-dev
-    - pybind11-dev
-    - python3-pybind11
     - libdlpack-dev
+    - libdrm-dev
+    - libva-dev
+    - ninja-build
+    - pkg-config
+    - python3-pip
 - name: pipModules
   type: object
   default:
-    - -i
-    - https://test.pypi.org/simple
-    - hip-python
+    - numpy
+    - pybind11
 - name: rocmDependencies
   type: object
   default:
@@ -52,13 +49,16 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
       pipModules: ${{ parameters.pipModules }}
+  - task: Bash@3
+    displayName: 'pip install hip-python'
+    inputs:
+      targetType: inline
+      script: pip install -i https://test.pypi.org/simple hip-python
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
@@ -73,26 +73,67 @@ jobs:
       # manual build case: triggered by ROCm/ROCm repo
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - task: Bash@3
+    displayName: 'ROCm symbolic link'
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: 'Save Python Package Paths'
+    inputs:
+      targetType: inline
+      script: |
+        echo "##vso[task.setvariable variable=PYTHON_USER_SITE;]$(python3 -m site --user-site)"
+        echo "##vso[task.setvariable variable=PYTHON_DIST_PACKAGES;]$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
+        echo "##vso[task.setvariable variable=PYBIND11_PATH;]$(python3 -c 'import pybind11; print(pybind11.get_cmake_dir())')"
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
-        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(PYTHON_USER_SITE)/pybind11;$(PYTHON_DIST_PACKAGES)/pybind11;$(PYBIND11_PATH)
         -DCMAKE_BUILD_TYPE=Release
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DCMAKE_INSTALL_PREFIX_PYTHON=$(Build.BinariesDirectory)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+      publish: false
+  - task: Bash@3
+    displayName: Create wheel file
+    inputs:
+      targetType: inline
+      script: |
+        export ROCM_PATH=$(Agent.BuildDirectory)/rocm
+        export HIP_INCLUDE_DIRS=$(Agent.BuildDirectory)/rocm/include/hip
+        python3 setup.py bdist_wheel
+      workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+    parameters:
+      sourceDir: $(Build.SourcesDirectory)/dist
+      contentsString: '*.whl'
+      targetDir: $(Build.ArtifactStagingDirectory)
+      clean: false
+  - task: PublishPipelineArtifact@1
+    displayName: 'wheel file Publish'
+    retryCountOnTaskFailure: 3
+    inputs:
+      targetPath: $(Build.ArtifactStagingDirectory)
 
 - job: rocPyDecode_testing
   dependsOn: rocPyDecode
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: $(JOB_TEST_POOL)
+  pool:
+    name: $(JOB_TEST_POOL)
+    demands: firstRenderDeviceAccess
   workspace:
     clean: all
   strategy:
@@ -101,11 +142,35 @@ jobs:
         JOB_GPU_TARGET: gfx942
         JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
   steps:
+  - task: Bash@3
+    displayName: Ensure pybind11-dev is not installed
+    inputs:
+      targetType: inline
+      script: |
+        if dpkg -l | grep -q pybind11-dev; then
+          echo "Removing pybind11-dev..."
+          sudo apt remove -y pybind11-dev
+        else
+          echo "pybind11-dev is not installed."
+        fi
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - task: Bash@3
+    displayName: 'pip install hip-python'
+    inputs:
+      targetType: inline
+      script: pip install -i https://test.pypi.org/simple hip-python
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Wheel Files'
+    inputs:
+      itemPattern: '**/*.whl'
+      targetPath: $(Agent.BuildDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
     parameters:
       ${{ if eq(parameters.checkoutRef, '') }}:
@@ -116,31 +181,49 @@ jobs:
     parameters:
       dependencyList: ${{ parameters.rocmDependencies }}
       gpuTarget: $(JOB_GPU_TARGET)
+      setupHIPLibrarySymlinks: true
       ${{ if eq(parameters.checkoutRef, '') }}:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
-    parameters:
-      checkoutRepo: ${{ parameters.checkoutRepo }}
   - task: Bash@3
-    displayName: Setup test environment
+    displayName: pip install
+    inputs:
+      targetType: inline
+      script: |
+        pip uninstall -y rocPyDecode
+        find -name *.whl -exec pip install {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Setup search paths
     inputs:
       targetType: inline
       script: |
         sudo rm -rf /opt/rocm
         sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
-        cd $(Build.SourcesDirectory)
-        sudo pip install .
-        cmake -DAMDGPU_TARGETS=$(JOB_GPU_TARGET) .
+        echo "##vso[task.setvariable variable=PYTHON_USER_SITE;]$(python3 -m site --user-site)"
+        echo "##vso[task.setvariable variable=PYTHON_DIST_PACKAGES;]$(python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"
+        echo "##vso[task.setvariable variable=PYBIND11_PATH;]$(python3 -c 'import pybind11; print(pybind11.get_cmake_dir())')"
+  - task: CMake@1
+    displayName: 'rocPyDecode Test CMake Flags'
+    inputs:
+      cmakeArgs: >-
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(PYTHON_USER_SITE)/pybind11;$(PYTHON_DIST_PACKAGES)/pybind11;$(PYBIND11_PATH)
+        -DCMAKE_BUILD_TYPE=Release
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        ..
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocPyDecode
-      testDir: $(Build.SourcesDirectory)
+      testDir: $(Build.SourcesDirectory)/build
 # sudo required for pip install but screws up permissions for next pipeline run
   - task: Bash@3
     displayName: Clean up test environment
+    condition: always()
     inputs:
       targetType: inline
       script: |
-        sudo rm -rf $(Build.SourcesDirectory)/*
+        pip uninstall -y rocPyDecode
+        pip uninstall -y hip-python

.azuredevops/components/rocRAND.yml

@@ -45,8 +45,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -73,13 +71,16 @@ jobs:
         -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocRAND_testing
   dependsOn: rocRAND
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -113,6 +114,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocRAND

.azuredevops/components/rocSOLVER.yml

@@ -44,6 +44,7 @@ parameters:
     - rocprofiler-register
     - ROCR-Runtime
     - rocSPARSE
+    - roctracer
 
 jobs:
 - job: rocSOLVER
@@ -57,8 +58,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -106,13 +105,16 @@ jobs:
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DBUILD_CLIENTS_SAMPLES=OFF
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocSOLVER_testing
   dependsOn: rocSOLVER
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -146,6 +148,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocSOLVER

.azuredevops/components/rocSPARSE.yml

@@ -40,6 +40,7 @@ parameters:
     - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
+    - roctracer
 
 jobs:
 - job: rocSPARSE
@@ -55,8 +56,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -89,6 +88,10 @@ jobs:
         -DBUILD_CLIENTS_BENCHMARKS=OFF
         -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip;$(Agent.BuildDirectory)/rocm/hip/cmake
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      artifactName: rocSPARSE
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       artifactName: rocSPARSE
@@ -106,7 +109,7 @@ jobs:
 - job: rocSPARSE_testing
   timeoutInMinutes: 90
   dependsOn: rocSPARSE
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -140,6 +143,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocSPARSE

.azuredevops/components/rocThrust.yml

@@ -47,8 +47,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -76,13 +74,16 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -DBUILD_TEST=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocThrust_testing
   dependsOn: rocThrust
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -116,6 +117,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocThrust

.azuredevops/components/rocWMMA.yml

@@ -56,8 +56,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -85,8 +83,12 @@ jobs:
         -DROCWMMA_BUILD_TESTS=ON
         -DROCWMMA_BUILD_SAMPLES=OFF
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON
         -GNinja
 # gfx1030 not supported in documentation
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
@@ -94,7 +96,7 @@ jobs:
 - job: rocWMMA_testing
   timeoutInMinutes: 90
   dependsOn: rocWMMA
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -128,6 +130,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocWMMA

.azuredevops/components/rocm-cmake.yml

@@ -47,4 +47,5 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocm-cmake
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/rocm-core.yml

@@ -30,4 +30,5 @@ jobs:
         -DCPACK_DEBIAN_PACKAGE_RELEASE="local.9999~99.99"
         -DCPACK_RPM_PACKAGE_RELEASE="local.9999"
         -DROCM_VERSION="$(next-release)"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/rocm-examples.yml

@@ -10,6 +10,7 @@ parameters:
   default:
     - cmake
     - libglfw3-dev
+    - libtbb-dev
     - python3-pip
 - name: rocmDependencies
   type: object
@@ -60,6 +61,7 @@ parameters:
     - rocSOLVER
     - rocSPARSE
     - rocThrust
+    - roctracer
 
 jobs:
 - job: rocm_examples
@@ -73,8 +75,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -110,13 +110,16 @@ jobs:
       script: |
         mkdir -p $(Build.BinariesDirectory)/examples
         mv $(Build.BinariesDirectory)/bin/* $(Build.BinariesDirectory)/examples
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocm_examples_testing
   dependsOn: rocm_examples
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -135,9 +138,9 @@ jobs:
     parameters:
       aptPackages: ${{ parameters.aptPackages }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
     parameters:
-      gpuTarget: $(JOB_GPU_TARGET)
+      checkoutRepo: ${{ parameters.checkoutRepo }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
     parameters:
       ${{ if eq(parameters.checkoutRef, '') }}:
@@ -152,20 +155,18 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
-  - task: Bash@3
-    displayName: Unload and reload AMDGPU
-    inputs:
-      targetType: inline
-      script: |
-        sudo modprobe -r amdgpu
-        sudo modprobe amdgpu
-  - task: Bash@3
-    displayName: Iterate through examples
-    inputs:
-      targetType: inline
-      script: |
-        for file in *; do
-          echo Now running: $file
-          ./$file | tee -a $(TEST_LOG_FILE)
-        done
-      workingDirectory: $(Agent.BuildDirectory)/rocm/examples
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      # https://github.com/ROCm/HIP/issues/2203
+      extraBuildFlags: >-
+        -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_ROOT=$(Agent.BuildDirectory)/rocm
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
+        -DCMAKE_EXE_LINKER_FLAGS=-fgpu-rdc
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocm-examples
+      testDir: $(Build.SourcesDirectory)/build

.azuredevops/components/rocm_bandwidth_test.yml

@@ -27,6 +27,7 @@ parameters:
 - name: rocmTestDependencies
   type: object
   default:
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
 
@@ -68,11 +69,12 @@ jobs:
         -DCMAKE_MODULE_PATH=$(Build.SourcesDirectory)/cmake_modules
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/rocm/include;$(Agent.BuildDirectory)/rocm/include/hsa
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
 
 - job: rocm_bandwidth_test_testing
   dependsOn: rocm_bandwidth_test
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -99,6 +101,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocm_bandwidth_test

.azuredevops/components/rocm_smi_lib.yml

@@ -25,11 +25,12 @@ jobs:
       extraBuildFlags: >-
         -DBUILD_TESTS=ON
         -DROCM_DEP_ROCMCORE=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
 
 - job: rocm_smi_lib_testing
   dependsOn: rocm_smi_lib
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -44,6 +45,9 @@ jobs:
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocm_smi_lib

.azuredevops/components/rocminfo.yml

@@ -45,11 +45,12 @@ jobs:
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCRTST_BLD_TYPE=release
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
 
 - job: rocminfo_testing
   dependsOn: rocminfo
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -72,6 +73,9 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+    parameters:
+      runRocminfo: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocminfo

.azuredevops/components/rocprofiler-compute.yml

@@ -0,0 +1,183 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - locales
+    - python3-pip
+- name: pipModules
+  type: object
+  default:
+    - astunparse==1.6.2
+    - colorlover
+    - dash>=1.12.0
+    - matplotlib
+    - numpy>=1.17.5
+    - pandas>=1.4.3
+    - pymongo
+    - pyyaml
+    - tabulate
+    - tqdm
+    - dash-svg
+    - dash-bootstrap-components
+    - kaleido
+    - setuptools
+    - plotille
+    - mock
+    - pytest
+    - pytest-cov
+    - pytest-xdist
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - rocm_smi_lib
+    - ROCR-Runtime
+    - rocprofiler
+    - rocprofiler-register
+    - roctracer
+
+jobs:
+- job: rocprofiler_compute
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rocprofiler_compute_testing
+  timeoutInMinutes: 120
+  dependsOn: rocprofiler_compute
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: PYTHON_VERSION
+    value: 3.10
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: Bash@3
+    displayName: Add en_US.UTF-8 locale
+    inputs:
+      targetType: inline
+      script: |
+        sudo locale-gen en_US.UTF-8
+        sudo update-locale
+        locale -a
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DCMAKE_HIP_ARCHITECTURES=$(JOB_GPU_TARGET)
+        -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+        -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DCMAKE_BUILD_TYPE=Release
+        -DENABLE_TESTS=ON
+        -DINSTALL_TESTS=ON
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofiler-compute
+      testDir: $(Build.BinariesDirectory)/libexec/rocprofiler-compute
+      testExecutable: ROCPROFCOMPUTE_ARCH_OVERRIDE="MI300X" ROCM_PATH=$(Agent.BuildDirectory)/rocm ctest
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    condition: always()
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    condition: always()
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"

.azuredevops/components/rocprofiler-register.yml

@@ -34,4 +34,5 @@ jobs:
     parameters:
       componentName: rocprofiler-register
       testDir: 'tests/build'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/components/rocprofiler-sdk.yml

@@ -0,0 +1,97 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - cmake
+    - python3-pip
+    - libdrm-dev
+    - libdw-dev
+    - libelf-dev
+- name: pipModules
+  type: object
+  default:
+    - black
+    - clang-format
+    - clang-tidy
+    - cmake
+    - cmake-format
+    - dataclasses
+    - numpy
+    - otf2
+    - pandas
+    - perfetto
+    - pycobertura
+    - pytest
+    - pyyaml
+- name: rocmDependencies
+  type: object
+  default:
+    - clr
+    - llvm-project
+    - rccl
+    - rocm-cmake
+    - rocm-core
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler-register
+    - roctracer
+    - aomp
+
+jobs:
+- job: rocprofilersdk
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+      extraBuildFlags: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCPROFILER_BUILD_TESTS=ON
+        -DROCPROFILER_BUILD_SAMPLES=OFF
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j2
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocprofiler-systems.yml

@@ -0,0 +1,154 @@
+# largely referenced from: https://github.com/ROCm/omnitrace/blob/main/.github/workflows/ubuntu-jammy.yml
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - autoconf
+    - autotools-dev
+    - bison
+    - build-essential
+    - bzip2
+    - clang
+    - cmake
+    - environment-modules
+    - g++-12
+    - libdrm-dev
+    - libfabric-dev
+    - libiberty-dev
+    - libpapi-dev
+    - libpfm4-dev
+    - libtool
+    - libopenmpi-dev
+    - m4
+    - openmpi-bin
+    - software-properties-common
+    - python3-pip
+    - texinfo
+    - zlib1g-dev
+- name: pipModules
+  type: object
+  default:
+    - numpy
+    - perfetto
+    - dataclasses
+- name: rocmDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - llvm-project
+    - rccl
+    - rocm-core
+    - rocm_smi_lib
+    - rocminfo
+    - ROCR-Runtime
+    - rocprofiler
+    - rocprofiler-register
+    - roctracer
+    - rocprofiler-sdk
+
+jobs:
+- job: rocprofiler_systems
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool: ${{ variables.MEDIUM_BUILD_POOL }}
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      # CI case: download latest default branch build
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      # manual build case: triggered by ROCm/ROCm repo
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - task: Bash@3
+    displayName: ROCm symbolic link
+    inputs:
+      targetType: inline
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
+    parameters:
+# build flags reference: https://rocm.docs.amd.com/projects/omnitrace/en/latest/install/install.html
+      extraBuildFlags: >-
+        -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCM_PATH=$(Agent.BuildDirectory)/rocm
+        -DROCPROFSYS_BUILD_TESTING=ON
+        -DROCPROFSYS_BUILD_DYNINST=ON
+        -DROCPROFSYS_BUILD_LIBUNWIND=ON
+        -DROCPROFSYS_DISABLE_EXAMPLES="openmp-target"
+        -DDYNINST_BUILD_TBB=ON
+        -DDYNINST_BUILD_ELFUTILS=ON
+        -DDYNINST_BUILD_LIBIBERTY=ON
+        -DDYNINST_BUILD_BOOST=ON
+        -DROCPROFSYS_USE_PAPI=ON
+        -DROCPROFSYS_USE_MPI=ON
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
+      multithreadFlag: -- -j32
+  - task: Bash@3
+    displayName: Set up rocprofiler-systems env
+    inputs:
+      targetType: inline
+      script: source share/rocprofiler-systems/setup-env.sh
+      workingDirectory: build
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rocprofiler-systems
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    condition: always()
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm compilers from PATH
+    condition: always()
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)

.azuredevops/components/rocprofiler.yml

@@ -57,8 +57,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -94,13 +92,16 @@ jobs:
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
       multithreadFlag: -- -j32
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: rocprofiler_testing
   dependsOn: rocprofiler
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -141,6 +142,7 @@ jobs:
       script: |
         sudo rm -rf /opt/rocm
         sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocprofilerV1

.azuredevops/components/rocr_debug_agent.yml

@@ -68,11 +68,12 @@ jobs:
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DROCM_PATH=$(Agent.BuildDirectory)/rocm
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
 
 - job: rocr_debug_agent_testing
   dependsOn: rocr_debug_agent
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -114,6 +115,7 @@ jobs:
       cmakeBuildDir: '$(Agent.BuildDirectory)/rocm/src/rocm-debug-agent-test'
       cmakeSourceDir: '.'
       installEnabled: false
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocr_debug_agent

.azuredevops/components/roctracer.yml

@@ -30,6 +30,7 @@ parameters:
   default:
     - clr
     - llvm-project
+    - rocminfo
     - rocprofiler-register
     - ROCR-Runtime
 
@@ -48,8 +49,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -77,14 +76,18 @@ jobs:
         -DCMAKE_MODULE_PATH=$(Agent.BuildDirectory)/rocm/lib/cmake/hip
         -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
         -DGPU_TARGETS=$(JOB_GPU_TARGET)
+        -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
 
 - job: roctracer_testing
   dependsOn: roctracer
-  condition: succeeded()
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
@@ -118,6 +121,7 @@ jobs:
         dependencySource: staging
       ${{ elseif ne(parameters.checkoutRef, '') }}:
         dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: roctracer

.azuredevops/components/rpp.yml

@@ -9,8 +9,18 @@ parameters:
   type: object
   default:
     - cmake
+    - libopencv-dev
+    - libsndfile1-dev
+    - libstdc++-12-dev
+    - imagemagick
     - ninja-build
     - clang
+- name: pipModules
+  type: object
+  default:
+    - openpyxl
+    - pandas
+    - sphinx
 - name: rocmDependencies
   type: object
   default:
@@ -20,6 +30,18 @@ parameters:
     - llvm-project
     - rocminfo
     - ROCR-Runtime
+- name: rocmTestDependencies
+  type: object
+  default:
+    - aomp
+    - clr
+    - half
+    - hipTensor
+    - llvm-project
+    - rocm-cmake
+    - rocminfo
+    - rocprofiler-register
+    - ROCR-Runtime
 
 jobs:
 - job: rpp
@@ -34,8 +56,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
     parameters:
@@ -65,6 +85,100 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DAMDGPU_TARGETS=$(JOB_GPU_TARGET)
         -GNinja
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     parameters:
       gpuTarget: $(JOB_GPU_TARGET)
+
+- job: rpp_testing
+  dependsOn: rpp
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: LD_LIBRARY_PATH
+    value: $(Agent.BuildDirectory)/rocm/lib;$(Agent.BuildDirectory)/rocm/llvm/lib
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+    parameters:
+      gpuTarget: $(JOB_GPU_TARGET)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      ${{ if eq(parameters.checkoutRef, '') }}:
+        dependencySource: staging
+      ${{ elseif ne(parameters.checkoutRef, '') }}:
+        dependencySource: tag-builds
+  # Dependencies from: https://github.com/ROCm/rpp/blob/develop/utilities/test_suite/README.md
+  - task: Bash@3
+    displayName: Build and install Turbo JPEG
+    inputs:
+      targetType: 'inline'
+      script: |
+        sudo apt-get install nasm
+        sudo apt-get install wget
+        git clone -b 3.0.2 https://github.com/libjpeg-turbo/libjpeg-turbo.git 
+        cd libjpeg-turbo
+        mkdir build
+        cd build
+        cmake -DCMAKE_INSTALL_PREFIX=/usr \
+              -DCMAKE_BUILD_TYPE=RELEASE  \
+              -DENABLE_STATIC=FALSE       \
+              -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib  \
+              -DWITH_JPEG8=TRUE           \
+              ..
+        make -j$nproc
+        sudo make install
+  - task: Bash@3
+    displayName: Build and install Nifti
+    inputs:
+      targetType: 'inline'
+      script: |
+        git clone https://github.com/NIFTI-Imaging/nifti_clib.git
+        cd nifti_clib
+        mkdir build
+        cd build
+        cmake ..
+        sudo make -j$nproc install
+  - task: Bash@3
+    displayName: Build rpp tests
+    inputs:
+      targetType: 'inline'
+      script: |
+        sudo rm -rf /opt/rocm
+        sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
+        mkdir rpp-tests
+        cd rpp-tests
+        cmake /opt/rocm/share/rpp/test \
+          -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ \
+          -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
+    parameters:
+      componentName: rpp
+      testExecutable: 'export PATH=$(Agent.BuildDirectory)/rocm/llvm/bin:$PATH; CC=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang CMAKE_VERBOSE_MAKEFILE=ON VERBOSE=1 ctest'
+      testDir: 'rpp-tests'
+  - script: sudo rm /opt/rocm
+    condition: always()

.azuredevops/dependencies/boost.yml

@@ -0,0 +1,65 @@
+parameters:
+- name: checkoutRepo
+  type: string
+  default: 'self'
+- name: checkoutRef
+  type: string
+  default: ''
+- name: boostVersion
+  type: string
+  default: ''
+- name: aptPackages
+  type: object
+  default:
+    - git
+- name: rocmDependencies
+  type: object
+  default:
+    - llvm-project
+
+jobs:
+- job: boost
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: Bash@3
+    displayName: 'git clone boost'
+    inputs:
+      targetType: inline
+      script: git clone -b ${{ parameters.boostVersion }} https://github.com/boostorg/boost --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmDependencies }}
+      dependencySource: staging
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add ROCm compilers to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+  - task: Bash@3
+    displayName: 'Build Boost with clang'
+    inputs:
+      targetType: inline
+      script: |
+        export CC=$(Agent.BuildDirectory)/rocm/llvm/bin/clang
+        export CXX=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
+        ./bootstrap.sh --with-toolset=clang --prefix=$(Build.BinariesDirectory)
+        ./b2 --toolset=clang threading=multi link=shared --prefix=$(Build.BinariesDirectory) cxxflags="-std=c++20"
+        ./b2 install
+      workingDirectory: $(Build.SourcesDirectory)/boost
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml

.azuredevops/dependencies/grpc.yml

@@ -20,7 +20,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/dependencies/gtest.yml

@@ -20,7 +20,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/dependencies/lapack.yml

@@ -21,7 +21,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/nightly/pytorch.yml

@@ -11,7 +11,6 @@ parameters:
     - ca-certificates
     - bc
     - bridge-utils
-    - cmake
     - devscripts
     - dkms
     - doxygen
@@ -62,11 +61,14 @@ parameters:
     - ffmpeg
     - libopenblas-dev
     - liblapack-dev
+    - libswscale-dev
+    - libavformat-dev
 - name: pipModules
   type: object
   default:
+    - cmake
     - astunparse
-    - expecttest!=0.2.0
+    - expecttest>=0.2.1
     - hypothesis
     - numpy
     - psutil
@@ -83,11 +85,15 @@ parameters:
     - lintrunner
     - ninja
     - packaging
-    - optree>=0.12.0
+    - optree>=0.13.0
+    - click>=8.0.3
   # list for vision
     - auditwheel
     - future
-# list from https://github.com/pytorch/builder/blob/main/manywheel/build_rocm.sh
+    - pytest
+    - pytest-azurepipelines
+    - pillow
+# list from https://github.com/pytorch/pytorch/blob/main/.ci/manywheel/build_rocm.sh
 - name: rocmDependencies
   type: object
   default:
@@ -117,6 +123,29 @@ parameters:
     - hipCUB
     - rocThrust
     - hipBLAS-common
+    - composable_kernel
+- name: rocmTestDependencies
+  type: object
+  default:
+    - rocminfo
+# Reference on what tests to run for torchvision found in private repo:
+# https://github.com/ROCm/rocAutomation/blob/jenkins-pipelines/pytorch/pytorch_ci/test_pytorch_test1.sh#L54
+# Will iterate through this list using pytest
+- name: torchTestList
+  type: object
+  default:
+    - nn
+    - torch
+#    - cuda seg faults and might need cuda installed on test system
+#    - ops takes too long
+    - unary_ufuncs
+    - binary_ufuncs
+    - autograd
+#    - inductor/torchinductor takes too long
+# set to false to disable torchvision build and test
+- name: includeVision
+  type: boolean
+  default: false
 
 trigger: none
 pr: none
@@ -136,13 +165,10 @@ jobs:
       amd-staging-gfx942:
         ROCM_BRANCH: amd-staging
         JOB_GPU_TARGET: gfx942
-      amd-staging-gfx90a:
-        ROCM_BRANCH: amd-staging
-        JOB_GPU_TARGET: gfx90a
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-# various flags/parameters expected by bash scripts in pytorch builder repo
+# various flags/parameters expected by bash scripts in pytorch repo's .ci directory
   - name: ROCM_VERSION
     value: 6.3.0
   - name: ROCM_PATH
@@ -163,7 +189,7 @@ jobs:
   workspace:
     clean: all
   steps:
-# copy environment setup from https://github.com/pytorch/builder/blob/main/manywheel/Dockerfile
+# copy environment setup from https://github.com/pytorch/pytorch/blob/main/.ci/docker/manywheel/Dockerfile
 # but instead of centos, use ubuntu environment
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
   - task: Bash@3
@@ -200,18 +226,27 @@ jobs:
       targetType: inline
       script: sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
   - checkout: self
-  - task: Bash@3
-    displayName: git clone pytorch builder
-    inputs:
-      targetType: inline
-      script: git clone https://github.com/pytorch/builder.git --depth=1 --recurse-submodules
-      workingDirectory: $(Build.SourcesDirectory)
   - task: Bash@3
     displayName: git clone upstream pytorch
     inputs:
       targetType: inline
       script: git clone https://github.com/pytorch/pytorch.git --depth=1 --recurse-submodules
       workingDirectory: $(Build.SourcesDirectory)
+# builder clone still needed due to run_tests.sh at end of build_common.sh call
+  - task: Bash@3
+    displayName: git clone pytorch builder
+    inputs:
+      targetType: inline
+      script: |
+        git clone https://github.com/pytorch/builder.git --depth=1 --recurse-submodules
+        sudo ln -s $(Build.SourcesDirectory)/builder /builder
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Temporarily Patch CK Submodule
+    inputs:
+      targetType: inline
+      script: git pull origin develop
+      workingDirectory: $(Build.SourcesDirectory)/pytorch/third_party/composable_kernel
   - task: Bash@3
     displayName: Install patchelf
     inputs:
@@ -264,58 +299,252 @@ jobs:
         PYTORCH_BUILD_VERSION=$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)
         PYTORCH_BUILD_NUMBER=$(date -u +%Y%m%d)
         SKIP_ALL_TESTS=1
-        bash ./manywheel/build_rocm.sh
-      workingDirectory: $(Build.SourcesDirectory)/builder
+        bash ./.ci/manywheel/build_rocm.sh
+      workingDirectory: $(Build.SourcesDirectory)/pytorch
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
     parameters:
       sourceDir: /remote/wheelhouserocm$(ROCM_VERSION)
       contentsString: '*.whl'
 # common helper source for pytorch vision and audio
-  - task: Bash@3
-    displayName: git clone pytorch test-infra
-    inputs:
-      targetType: inline
-      script: git clone https://github.com/pytorch/test-infra.git --depth=1 --recurse-submodules
-      workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: install package helper
-    inputs:
-      targetType: inline
-      script: python3 -m pip install test-infra/tools/pkg-helpers
-      workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: pytorch pkg helpers
-    inputs:
-      targetType: inline
-      script: CU_VERSION=${CU_VERSION} CHANNEL=${CHANNEL} python -m pytorch_pkg_helpers
-# get torch vision source and build
-  - task: Bash@3
-    displayName: git clone pytorch vision
-    inputs:
-      targetType: inline
-      script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
-      workingDirectory: $(Build.SourcesDirectory)
-  - task: Bash@3
-    displayName: Build vision
-    inputs:
-      targetType: inline
-      script: >-
-        BUILD_VERSION=$(ROCM_BRANCH).$(JOB_GPU_TARGET)-$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)$(date -u +%Y%m%d)
-        python3 setup.py bdist_wheel
-      workingDirectory: $(Build.SourcesDirectory)/vision
-  - task: Bash@3
-    displayName: Relocate vision
-    inputs:
-      targetType: inline
-      script: python3 packaging/wheel/relocate.py
-      workingDirectory: $(Build.SourcesDirectory)/vision
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
-    parameters:
-      sourceDir: $(Build.SourcesDirectory)/vision/dist
-      contentsString: '*.whl'
-      clean: false
+  - ${{ if eq(parameters.includeVision, true) }}:
+    - task: Bash@3
+      displayName: git clone pytorch test-infra
+      inputs:
+        targetType: inline
+        script: git clone https://github.com/pytorch/test-infra.git --depth=1 --recurse-submodules
+        workingDirectory: $(Build.SourcesDirectory)
+    - task: Bash@3
+      displayName: install package helper
+      inputs:
+        targetType: inline
+        script: python3 -m pip install test-infra/tools/pkg-helpers
+        workingDirectory: $(Build.SourcesDirectory)
+    - task: Bash@3
+      displayName: pytorch pkg helpers
+      inputs:
+        targetType: inline
+        script: CU_VERSION=${CU_VERSION} CHANNEL=${CHANNEL} python -m pytorch_pkg_helpers
+  # get torch vision source and build
+    - task: Bash@3
+      displayName: git clone pytorch vision
+      inputs:
+        targetType: inline
+        script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
+        workingDirectory: $(Build.SourcesDirectory)
+    - task: Bash@3
+      displayName: Build vision
+      inputs:
+        targetType: inline
+        script: >-
+          TORCH_PACKAGE_NAME=torch.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
+          TORCHVISION_PACKAGE_NAME=torchvision.$(ROCM_BRANCH).$(JOB_GPU_TARGET)
+          PYTORCH_VERSION=$(cat $(Build.SourcesDirectory)/pytorch/version.txt | cut -da -f1)post$(date -u +%Y%m%d)
+          BUILD_VERSION=$(cat $(Build.SourcesDirectory)/vision/version.txt | cut -da -f1)post$(date -u +%Y%m%d)
+          python3 setup.py bdist_wheel
+        workingDirectory: $(Build.SourcesDirectory)/vision
+    - task: Bash@3
+      displayName: Relocate vision
+      inputs:
+        targetType: inline
+        script: python3 packaging/wheel/relocate.py
+        workingDirectory: $(Build.SourcesDirectory)/vision
+    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
+      parameters:
+        sourceDir: $(Build.SourcesDirectory)/vision/dist
+        contentsString: '*.whl'
+        clean: false
   - task: PublishPipelineArtifact@1
     displayName: 'wheel file Publish'
     retryCountOnTaskFailure: 3
     inputs:
       targetPath: $(Build.BinariesDirectory)
+
+- job: pytorch_testing
+  dependsOn: pytorch
+  condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName'])))
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: PYTORCH_TEST_WITH_ROCM
+    value: 1
+  pool: $(JOB_TEST_POOL)
+  workspace:
+    clean: all
+  strategy:
+    matrix:
+      gfx942:
+        JOB_GPU_TARGET: gfx942
+        JOB_TEST_POOL: ${{ variables.GFX942_TEST_POOL }}
+  steps:
+  - task: Bash@3
+    displayName: 'Register libjpeg-turbo packages'
+    inputs:
+      targetType: inline
+      script: |
+        sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+        wget -q -O- https://packagecloud.io/dcommander/libjpeg-turbo/gpgkey | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/libjpeg-turbo.gpg > /dev/null
+        echo "deb [signed-by=/etc/apt/trusted.gpg.d/libjpeg-turbo.gpg] https://packagecloud.io/dcommander/libjpeg-turbo/any/ any main" | sudo tee /etc/apt/sources.list.d/libjpeg-turbo.list
+        sudo apt update
+        apt-cache show libjpeg-turbo-official | grep Version
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+      pipModules: ${{ parameters.pipModules }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Wheel Files'
+    inputs:
+      itemPattern: '**/*$(JOB_GPU_TARGET)*.whl'
+      targetPath: $(Agent.BuildDirectory)
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+    parameters:
+      dependencySource: staging
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      dependencyList: ${{ parameters.rocmTestDependencies }}
+      gpuTarget: $(JOB_GPU_TARGET)
+      dependencySource: staging
+      skipLlvmSymlink: true
+# get sources to run test scripts
+  - task: Bash@3
+    displayName: git clone upstream pytorch
+    inputs:
+      targetType: inline
+      script: git clone https://github.com/pytorch/pytorch.git --depth=1 --recurse-submodules
+      workingDirectory: $(Build.SourcesDirectory)
+  - ${{ if eq(parameters.includeVision, true) }}:
+    - task: Bash@3
+      displayName: git clone pytorch vision
+      inputs:
+        targetType: inline
+        script: git clone https://github.com/pytorch/vision.git --depth=1 --recurse-submodules
+        workingDirectory: $(Build.SourcesDirectory)
+  - task: Bash@3
+    displayName: Install Wheel Files
+    inputs:
+      targetType: inline
+      script: find . -name "*.whl" -exec pip install --no-index --find-links=. --no-dependencies -v {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Show Updated pip List
+    inputs:
+      targetType: inline
+      script: pip list -v
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Add ROCm binaries to PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+  - task: Bash@3
+    displayName: Add Python site-packages binaries to path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.prependpath]$USER_BASE/bin"
+  - task: Bash@3
+    displayName: Add torch libs to ldconfig
+    inputs:
+      targetType: inline
+      script: |
+        echo $(python3 -m site --user-site)/torch/lib | sudo tee /etc/ld.so.conf.d/torch.conf
+        sudo ldconfig -v
+        ldconfig -p
+# https://pytorch.org/get-started/locally/#linux-verification
+# https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/3rd-party/pytorch-install.html#testing-the-pytorch-installation
+  - task: Bash@3
+    displayName: Simple Import Torch Tests
+    inputs:
+      targetType: inline
+      script: |
+        python3 -c 'import torch' 2> /dev/null && echo 'Success' || echo 'Failure'
+        python3 -c 'import torch; print(torch.cuda.is_available())'
+        python3 -c 'import torch; x = torch.rand(5, 3); print(x)'
+# Test artifact build script has too many if statements for different environments
+# Based off the snippet of interest for this environment, with some adjustments
+# https://github.com/pytorch/pytorch/blob/main/.ci/pytorch/build.sh#L335-L375
+# Removing in-line comments since it does not fit with the yaml markup
+  - task: Bash@3
+    displayName: Build Pytorch Test Artifacts
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: |
+        CUSTOM_TEST_ARTIFACT_BUILD_DIR="build/custom_test_artifacts"
+        CUSTOM_TEST_USE_ROCM=ON
+        CUSTOM_TEST_MODULE_PATH="${PWD}/cmake/public"
+        mkdir -pv "${CUSTOM_TEST_ARTIFACT_BUILD_DIR}"
+
+        CUSTOM_OP_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-op-build"
+        CUSTOM_OP_TEST="${PWD}/test/custom_operator"
+        python --version
+        SITE_PACKAGES="$(python -c 'import site; print(";".join([x for x in site.getsitepackages()] + [x + "/torch" for x in site.getsitepackages()]))')"
+
+        mkdir -p "$CUSTOM_OP_BUILD"
+        pushd "$CUSTOM_OP_BUILD"
+        cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
+              -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
+        make VERBOSE=1
+        popd
+
+        JIT_HOOK_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/jit-hook-build"
+        JIT_HOOK_TEST="$PWD/test/jit_hooks"
+        python --version
+        SITE_PACKAGES="$(python -c 'import site; print(";".join([x for x in site.getsitepackages()] + [x + "/torch" for x in site.getsitepackages()]))')"
+        mkdir -p "$JIT_HOOK_BUILD"
+        pushd "$JIT_HOOK_BUILD"
+        cmake "$JIT_HOOK_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
+              -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
+        make VERBOSE=1
+        popd
+
+        CUSTOM_BACKEND_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-backend-build"
+        CUSTOM_BACKEND_TEST="${PWD}/test/custom_backend"
+        python --version
+        mkdir -p "$CUSTOM_BACKEND_BUILD"
+        pushd "$CUSTOM_BACKEND_BUILD"
+        cmake "$CUSTOM_BACKEND_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
+              -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
+        make VERBOSE=1
+        popd
+      workingDirectory: $(Build.SourcesDirectory)/pytorch
+  - ${{ each torchTest in parameters.torchTestList }}:
+    - task: Bash@3
+      displayName: Test ${{ torchTest }}
+      continueOnError: true
+      inputs:
+        targetType: inline
+        workingDirectory: $(Build.SourcesDirectory)/pytorch
+        ${{ if contains(torchTest, '/') }}:
+          script: pytest test/${{ split(torchTest, '/')[0] }}/test_${{ split(torchTest, '/')[1] }}.py
+        ${{ else }}:
+          script: pytest test/test_${{ torchTest }}.py
+# Reference on what tests to run for torchvision found in private repo:
+# https://github.com/ROCm/rocAutomation/blob/jenkins-pipelines/pytorch/pytorch_ci/test_torchvision.sh#L51
+  - ${{ if eq(parameters.includeVision, true) }}:
+    - task: Bash@3
+      displayName: Test vision/transforms
+      continueOnError: true
+      inputs:
+        targetType: inline
+        script: pytest test/test_transforms.py
+        workingDirectory: $(Build.SourcesDirectory)/vision
+  - task: Bash@3
+    displayName: Uninstall Wheel Files
+    inputs:
+      targetType: inline
+      script: find . -name "*.whl" -exec pip uninstall -y {} \;
+      workingDirectory: $(Agent.BuildDirectory)
+  - task: Bash@3
+    displayName: Remove Python site-packages binaries from path
+    inputs:
+      targetType: inline
+      script: |
+        USER_BASE=$(python3 -m site --user-base)
+        echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$USER_BASE/bin;;' -e 's;^/;;' -e 's;/$;;')"
+  - task: Bash@3
+    displayName: Remove ROCm binaries from PATH
+    inputs:
+      targetType: inline
+      script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"

.azuredevops/nightly/rocm-nightly.yml

@@ -36,6 +36,7 @@ parameters:
     - rocDecode
     - rocFFT
     - ROCgdb
+    - rocJPEG
     - rocm-cmake
     - rocm-core
     - rocm-examples
@@ -45,7 +46,10 @@ parameters:
     - rocm_bandwidth_test
     - rocm_smi_lib
     - rocPRIM
+    - rocprofiler-compute
     - rocprofiler-register
+    - rocprofiler-sdk
+    - rocprofiler-systems
     - rocprofiler
     - rocPyDecode
     - ROCR-Runtime
@@ -80,8 +84,6 @@ jobs:
     matrix:
       gfx942:
         JOB_GPU_TARGET: gfx942
-      gfx90a:
-        JOB_GPU_TARGET: gfx90a
   steps:
   - task: DeleteFiles@1
     displayName: 'Cleanup checkout space'
@@ -101,11 +103,13 @@ jobs:
       dependencyList: ${{ parameters.rocmDependencies }}
       dependencySource: staging
       skipLibraryLinking: true
+      skipLlvmSymlink: true
       gpuTarget: $(JOB_GPU_TARGET)
   - script: df -h
     displayName: System disk space after ROCm
   - script: du -sh $(Agent.BuildDirectory)/rocm
     displayName: Uncompressed ROCm size
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - task: ArchiveFiles@2
     displayName: Compress rocm-nightly
     inputs:
@@ -117,7 +121,7 @@ jobs:
   - script: du -sh $(Build.ArtifactStagingDirectory)
     displayName: Compressed ROCm size
   - task: PublishPipelineArtifact@1
-    displayName: 'Public ROCm Nightly Artifact'
+    displayName: 'Publish ROCm Nightly Artifact'
     retryCountOnTaskFailure: 3
     inputs:
       targetPath: '$(Build.ArtifactStagingDirectory)'

.azuredevops/tag-builds/Tensile.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/Tensile
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/Tensile.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/boost.yml

@@ -0,0 +1,23 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: boostVersion
+  type: string
+  default: 'master'
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_DEPENDENCIES_PATH }}/boost.yml
+    parameters:
+      boostVersion: ${{ parameters.boostVersion }}

.azuredevops/tag-builds/omnitrace.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/omnitrace
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/omnitrace.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocJPEG.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocJPEG
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocJPEG.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocprofiler-compute.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocprofiler-compute
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocprofiler-compute.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocprofiler-sdk.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocprofiler-sdk
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocprofiler-sdk.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/tag-builds/rocprofiler-systems.yml

@@ -0,0 +1,29 @@
+variables:
+- group: common
+- template: /.azuredevops/variables-global.yml
+
+parameters:
+- name: checkoutRef
+  type: string
+  default: refs/tags/$(LATEST_RELEASE_TAG)
+
+resources:
+  repositories:
+  - repository: pipelines_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/ROCm
+  - repository: release_repo
+    type: github
+    endpoint: ROCm
+    name: ROCm/rocprofiler-systems
+    ref: ${{ parameters.checkoutRef }}
+
+trigger: none
+pr: none
+
+jobs:
+  - template: ${{ variables.CI_COMPONENT_PATH }}/rocprofiler-systems.yml
+    parameters:
+      checkoutRepo: release_repo
+      checkoutRef: ${{ parameters.checkoutRef }}

.azuredevops/templates/steps/artifact-download.yml

@@ -9,6 +9,10 @@ parameters:
 - name: useDefaultBranch
   type: boolean
   default: true
+# useMainlineBranch only processed if useDefaultBranch is false
+- name: useMainlineBranch
+  type: boolean
+  default: false
 - name: latestFromBranch
   type: boolean
   default: true
@@ -22,13 +26,14 @@ parameters:
   type: object
   default:
     AMDMIGraphX: develop
-    amdsmi: develop
+    amdsmi: amd-staging
     aomp-extras: aomp-dev
     aomp: aomp-dev
     clr: amd-staging
     composable_kernel: develop
     half: rocm
     HIP: amd-staging
+    hip-tests: amd-staging
     hipBLAS: develop
     hipBLASLt: develop
     hipBLAS-common: develop
@@ -44,8 +49,10 @@ parameters:
     llvm-project: amd-staging
     MIOpen: develop
     MIVisionX: develop
+    omniperf: amd-staging
+    omnitrace: amd-staging
     rccl: develop
-    rdc: develop
+    rdc: amd-staging
     rocAL: develop
     rocALUTION: develop
     rocBLAS: develop
@@ -53,17 +60,22 @@ parameters:
     rocDecode: develop
     rocFFT: develop
     ROCgdb: amd-staging
+    rocJPEG: develop
     rocm-cmake: develop
-    rocm-core: master
+    rocm-core: amd-staging
     rocm-examples: develop
     rocminfo: amd-staging
     rocMLIR: develop
     ROCmValidationSuite: master
     rocm_bandwidth_test: master
-    rocm_smi_lib: develop
+    rocm_smi_lib: amd-staging
     rocPRIM: develop
-    rocprofiler-register: amd-mainline
     rocprofiler: amd-staging
+    rocprofiler-compute: amd-staging
+    rocprofiler-register: amd-staging
+    rocprofiler-sdk: amd-staging
+    rocprofiler-systems: amd-staging
+    rocPyDecode: develop
     ROCR-Runtime: amd-staging
     rocRAND: develop
     rocr_debug_agent: amd-staging
@@ -73,24 +85,85 @@ parameters:
     roctracer: amd-staging
     rocWMMA: develop
     rpp: develop
-- name: allowPartiallySucceededBuilds
+- name: mainlineBranchList
   type: object
   default:
-    - amdsmi
-    - aomp
-    - HIPIFY
-    - rccl
-    - rdc
-    - rocm-cmake
-    - rocm_smi_lib
-    - rocFFT
-    - MIVisionX
+    AMDMIGraphX: mainline
+    amdsmi: amd-mainline
+    aomp-extras: amd-mainline-open
+    aomp: amd-mainline-open
+    clr: amd-mainline
+    composable_kernel: mainline
+    half: rocm
+    HIP: amd-mainline
+    hip-tests: amd-mainline
+    hipBLAS: mainline
+    hipBLASLt: mainline
+    hipBLAS-common: mainline
+    hipCUB: mainline
+    hipFFT: mainline
+    hipfort: mainline
+    HIPIFY: amd-mainline
+    hipRAND: mainline
+    hipSOLVER: mainline
+    hipSPARSE: mainline
+    hipSPARSELt: mainline
+    hipTensor: mainline
+    llvm-project: amd-mainline-open
+    MIOpen: mainline
+    MIVisionX: mainline
+    omniperf: amd-mainline
+    omnitrace: amd-mainline
+    rccl: mainline
+    rdc: amd-mainline
+    rocAL: master # needs the yaml file
+    rocALUTION: mainline
+    rocBLAS: mainline
+    ROCdbgapi : amd-mainline
+    rocDecode: mainline
+    rocFFT: mainline
+    ROCgdb: amd-mainline-rocgdb-15
+    rocJPEG: mainline
+    rocm-cmake: mainline
+    rocm-core: amd-master
+    rocm-examples: develop # no mainline
+    rocminfo: amd-master
+    rocMLIR: mainline # needs the yaml file
+    ROCmValidationSuite: mainline
+    rocm_bandwidth_test: master
+    rocm_smi_lib: amd-mainline
+    rocPRIM: mainline
+    rocprofiler: amd-master
+    rocprofiler-compute: amd-mainline
+    rocprofiler-register: amd-mainline
+    rocprofiler-sdk: amd-mainline
+    rocprofiler-systems: amd-mainline
+    rocPyDecode: mainline
+    ROCR-Runtime: amd-master
+    rocRAND: mainline
+    rocr_debug_agent: amd-mainline
+    rocSOLVER: mainline
+    rocSPARSE: mainline
+    rocThrust: mainline
+    roctracer: amd-master
+    rocWMMA: mainline
+    rpp: mainline
 # BELOW REQUIRED IF useDefaultBranch false
 - name: branchName
   type: string
   default: '$(Build.SourceBranchName)' # for tagged builds
 
 steps:
+- task: Bash@3
+  displayName: Set allowPartiallySucceededBuilds
+  inputs:
+    targetType: inline
+    script: |
+      if [[ ",$ALLOWED_PARTIAL_SUCCEED_BUILDS," == *",${{ parameters.componentName }},"* ]]; then
+        echo "##vso[task.setvariable variable=allowPartiallySucceededBuilds;]true"
+      else
+        echo "##vso[task.setvariable variable=allowPartiallySucceededBuilds;]false"
+      fi
 - task: DownloadPipelineArtifact@2
   displayName: Download ${{ parameters.componentName }}
   inputs:
@@ -104,10 +177,11 @@ steps:
         buildVersionToDownload: latestFromBranch # default is 'latest'
     ${{ if eq(parameters.useDefaultBranch, true) }}:
       branchName: refs/heads/${{ parameters.defaultBranchList[parameters.componentName] }}
+    ${{ elseif eq(parameters.useMainlineBranch, true) }}:
+      branchName: refs/heads/${{ parameters.mainlineBranchList[parameters.componentName] }}
     ${{ else }}:
       branchName: ${{ parameters.branchName }}
-    ${{ if containsValue(parameters.allowPartiallySucceededBuilds, parameters.componentName) }}:
-      allowPartiallySucceededBuilds: true
+    allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
     targetPath: '$(Pipeline.Workspace)/d'
 - task: ExtractFiles@1
   displayName: Extract ${{ parameters.componentName }}

.azuredevops/templates/steps/dependencies-aqlprofile.yml

@@ -8,20 +8,22 @@ parameters:
 - name: repositoryUrl
   type: object
   default: 
-    staging: https://repo.radeon.com/rocm/apt/6.2/pool/main/h/hsa-amd-aqlprofile
-    tag-builds: https://repo.radeon.com/rocm/apt/6.2/pool/main/h/hsa-amd-aqlprofile
-- name: packageName
-  type: object
-  default:
-    staging: hsa-amd-aqlprofile_1.0.0.60200.60200-66~22.04_amd64.deb
-    tag-builds: hsa-amd-aqlprofile_1.0.0.60200.60200-66~22.04_amd64.deb
+    staging: https://repo.radeon.com/rocm/apt/latest/pool/main/h/hsa-amd-aqlprofile/ # end slash is important for curl!
+    tag-builds: https://repo.radeon.com/rocm/apt/$(TAGGED_RELEASE)/pool/main/h/hsa-amd-aqlprofile/
 
 steps:
+- task: Bash@3
+  displayName: Get aqlprofile package name
+  inputs:
+    targetType: inline
+    script: |
+      export packageName=$(curl -s ${{ parameters.repositoryUrl[parameters.dependencySource] }} | grep -oP "href=\"\K[^\"]*$(lsb_release -rs)[^\"]*\.deb")
+      echo "##vso[task.setvariable variable=packageName;isreadonly=true]$packageName"
 - task: Bash@3
   displayName: 'Download aqlprofile'
   inputs:
     targetType: inline
-    script: wget -nv ${{ parameters.repositoryUrl[parameters.dependencySource] }}/${{ parameters.packageName[parameters.dependencySource] }}
+    script: wget -nv ${{ parameters.repositoryUrl[parameters.dependencySource] }}$(packageName)
     workingDirectory: '$(Pipeline.Workspace)'
 - task: Bash@3
   displayName: 'Extract aqlprofile'
@@ -29,7 +31,7 @@ steps:
     targetType: inline
     script: |
       mkdir hsa-amd-aqlprofile
-      dpkg-deb -R ${{ parameters.packageName[parameters.dependencySource] }} hsa-amd-aqlprofile
+      dpkg-deb -R $(packageName) hsa-amd-aqlprofile
     workingDirectory: '$(Pipeline.Workspace)'
 - task: Bash@3
   displayName: 'Copy aqlprofile files'
@@ -43,5 +45,5 @@ steps:
   displayName: 'Clean up aqlprofile'
   inputs:
     targetType: inline
-    script: rm -rf hsa-amd-aqlprofile ${{ parameters.packageName[parameters.dependencySource] }}
+    script: rm -rf hsa-amd-aqlprofile $(packageName)
     workingDirectory: '$(Pipeline.Workspace)'

.azuredevops/templates/steps/dependencies-boost.yml

@@ -0,0 +1,35 @@
+steps:
+- task: DownloadPipelineArtifact@2
+  displayName: Download Boost
+  inputs:
+    buildType: specific
+    project: ROCm-CI
+    definition: $(BOOST_DEPENDENCY_PIPELINE_ID)
+    targetPath: $(Pipeline.Workspace)/d
+- task: ExtractFiles@1
+  displayName: Extract Boost
+  inputs:
+    archiveFilePatterns: '$(Pipeline.Workspace)/d/**/*.tar.gz'
+    destinationFolder: $(Agent.BuildDirectory)/boost
+    cleanDestinationFolder: true
+    overwriteExistingFiles: true
+- task: DeleteFiles@1
+  displayName: Cleanup Compressed Boost
+  inputs:
+    SourceFolder: $(Pipeline.Workspace)/d
+    Contents: '**/*.tar.gz'
+    RemoveDotFiles: true
+- task: Bash@3
+  displayName: 'List Boost files'
+  inputs:
+    targetType: inline
+    script: ls -1R $(Agent.BuildDirectory)/boost
+- task: Bash@3
+  displayName: 'Link Boost shared libraries'
+  inputs:
+    targetType: inline
+    script: |
+      echo $(Agent.BuildDirectory)/boost/lib | sudo tee /etc/ld.so.conf.d/boost.conf
+      sudo cat /etc/ld.so.conf.d/boost.conf
+      sudo ldconfig -v
+      ldconfig -p

.azuredevops/templates/steps/dependencies-other.yml

@@ -8,6 +8,12 @@ parameters:
   default: []
 
 steps:
+# firefox takes time to upgrade and is not needed for CI workloads, hold version
+- task: Bash@3
+  displayName: 'sudo apt-mark hold firefox'
+  inputs:
+    targetType: inline
+    script: sudo apt-mark hold firefox
 - task: Bash@3
   displayName: 'sudo apt-get update'
   inputs:

.azuredevops/templates/steps/dependencies-rocm.yml

@@ -9,6 +9,7 @@ parameters:
   default: staging
   values:
     - staging
+    - mainline
     - tag-builds
     - fixed
 - name: extractToMnt
@@ -37,6 +38,7 @@ parameters:
     composable_kernel: $(COMPOSABLE_KERNEL_PIPELINE_ID)
     half: $(HALF_PIPELINE_ID)
     HIP: $(HIP_PIPELINE_ID)
+    hip-tests: $(HIP_TESTS_PIPELINE_ID)
     hipBLAS: $(HIPBLAS_PIPELINE_ID)
     hipBLAS-common: $(HIPBLAS_COMMON_PIPELINE_ID)
     hipBLASLt: $(HIPBLASLT_PIPELINE_ID)
@@ -53,6 +55,7 @@ parameters:
     MIOpen: $(MIOpen_PIPELINE_ID)
     MIVisionX: $(MIVISIONX_PIPELINE_ID)
     omniperf: $(OMNIPERF_PIPELINE_ID)
+    omnitrace: $(OMNITRACE_PIPELINE_ID)
     rccl: $(RCCL_PIPELINE_ID)
     rdc: $(RDC_PIPELINE_ID)
     rocAL: $(ROCAL_PIPELINE_ID)
@@ -62,6 +65,7 @@ parameters:
     rocDecode: $(ROCDECODE_PIPELINE_ID)
     rocFFT: $(ROCFFT_PIPELINE_ID)
     ROCgdb: $(ROCGDB_PIPELINE_ID)
+    rocJPEG: $(ROCJPEG_PIPELINE_ID)
     rocm-cmake: $(ROCM_CMAKE_PIPELINE_ID)
     rocm-core: $(ROCM_CORE_PIPELINE_ID)
     rocm-examples: $(ROCM_EXAMPLES_PIPELINE_ID)
@@ -71,8 +75,12 @@ parameters:
     rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_PIPELINE_ID)
     rocm_smi_lib: $(ROCM_SMI_LIB_PIPELINE_ID)
     rocPRIM: $(ROCPRIM_PIPELINE_ID)
+    rocprofiler-compute: $(ROCPROFILER_COMPUTE_PIPELINE_ID)
     rocprofiler-register: $(ROCPROFILER_REGISTER_PIPELINE_ID)
+    rocprofiler-sdk: $(ROCPROFILER_SDK_PIPELINE_ID)
+    rocprofiler-systems: $(ROCPROFILER_SYSTEMS_PIPELINE_ID)
     rocprofiler: $(ROCPROFILER_PIPELINE_ID)
+    rocPyDecode: $(ROCPYDECODE_PIPELINE_ID)
     ROCR-Runtime: $(ROCR_RUNTIME_PIPELINE_ID)
     rocRAND: $(ROCRAND_PIPELINE_ID)
     rocr_debug_agent: $(ROCR_DEBUG_AGENT_PIPELINE_ID)
@@ -94,6 +102,7 @@ parameters:
     composable_kernel: $(COMPOSABLE_KERNEL_TAGGED_PIPELINE_ID)
     half: $(HALF_TAGGED_PIPELINE_ID)
     HIP: $(HIP_TAGGED_PIPELINE_ID)
+    hip-tests: $(HIP_TESTS_TAGGED_PIPELINE_ID)
     hipBLAS: $(HIPBLAS_TAGGED_PIPELINE_ID)
     hipBLAS-common: $(HIPBLAS_COMMON_TAGGED_PIPELINE_ID)
     hipBLASLt: $(HIPBLASLT_TAGGED_PIPELINE_ID)
@@ -110,6 +119,7 @@ parameters:
     MIOpen: $(MIOpen_TAGGED_PIPELINE_ID)
     MIVisionX: $(MIVISIONX_TAGGED_PIPELINE_ID)
     omniperf: $(OMNIPERF_TAGGED_PIPELINE_ID)
+    omnitrace: $(OMNITRACE_TAGGED_PIPELINE_ID)
     rccl: $(RCCL_TAGGED_PIPELINE_ID)
     rdc: $(RDC_TAGGED_PIPELINE_ID)
     rocAL: $(ROCAL_TAGGED_PIPELINE_ID)
@@ -119,6 +129,7 @@ parameters:
     rocDecode: $(ROCDECODE_TAGGED_PIPELINE_ID)
     rocFFT: $(ROCFFT_TAGGED_PIPELINE_ID)
     ROCgdb: $(ROCGDB_TAGGED_PIPELINE_ID)
+    rocJPEG: $(ROCJPEG_TAGGED_PIPELINE_ID)
     rocm-cmake: $(ROCM_CMAKE_TAGGED_PIPELINE_ID)
     rocm-core: $(ROCM_CORE_TAGGED_PIPELINE_ID)
     rocm-examples: $(ROCM_EXAMPLES_TAGGED_PIPELINE_ID)
@@ -128,8 +139,12 @@ parameters:
     rocm_bandwidth_test: $(ROCM_BANDWIDTH_TEST_TAGGED_PIPELINE_ID)
     rocm_smi_lib: $(ROCM_SMI_LIB_TAGGED_PIPELINE_ID)
     rocPRIM: $(ROCPRIM_TAGGED_PIPELINE_ID)
+    rocprofiler-compute: $(ROCPROFILER_COMPUTE_TAGGED_PIPELINE_ID)
     rocprofiler-register: $(ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID)
+    rocprofiler-sdk: $(ROCPROFILER_SDK_TAGGED_PIPELINE_ID)
+    rocprofiler-systems: $(ROCPROFILER_SYSTEMS_PIPELINE_ID)
     rocprofiler: $(ROCPROFILER_TAGGED_PIPELINE_ID)
+    rocPyDecode: $(ROCPYDECODE_TAGGED_PIPELINE_ID)
     ROCR-Runtime: $(ROCR_RUNTIME_TAGGED_PIPELINE_ID)
     rocRAND: $(ROCRAND_TAGGED_PIPELINE_ID)
     rocr_debug_agent: $(ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID)
@@ -150,6 +165,11 @@ parameters:
 - name: skipLlvmSymlink
   type: boolean
   default: false
+# set to true if dlopen calls for HIP libraries are causing failures
+# because they do not follow shared library symlink convention
+- name: setupHIPLibrarySymlinks
+  type: boolean
+  default: false
 # some ROCm components can specify GPU target and this will affect downloads
 - name: gpuTarget
   type: string
@@ -168,13 +188,18 @@ parameters:
     - hipRAND
     - hipSPARSELt
     - hipTensor
+    - omnitrace
     - rccl
     - rocALUTION
     - rocBLAS
     - rocFFT
     - rocm-examples
     - rocPRIM
+    - rocprofiler-compute
+    - rocprofiler-sdk
+    - rocprofiler-systems
     - rocprofiler
+    - rocPyDecode
     - rocRAND
     - rocSOLVER
     - rocSPARSE
@@ -208,6 +233,10 @@ steps:
         ${{ if eq(parameters.dependencySource, 'staging') }}:
           pipelineId: ${{ parameters.stagingPipelineIdentifiers[ split(dependency, ':')[0] ] }}
           latestFromBranch: ${{ parameters.latestFromBranch }}
+        ${{ elseif eq(parameters.dependencySource, 'mainline') }}:
+          pipelineId: ${{ parameters.stagingPipelineIdentifiers[ split(dependency, ':')[0] ] }}
+          useMainlineBranch: true
+          latestFromBranch: ${{ parameters.latestFromBranch }}
         ${{ elseif eq(parameters.dependencySource, 'tag-builds') }}:
           pipelineId: ${{ parameters.taggedPipelineIdentifiers[ split(dependency, ':')[0] ] }}
           latestFromBranch: false
@@ -224,6 +253,10 @@ steps:
         ${{ if eq(parameters.dependencySource, 'staging') }}:
           pipelineId: ${{ parameters.stagingPipelineIdentifiers[dependency] }}
           latestFromBranch: ${{ parameters.latestFromBranch }}
+        ${{ elseif eq(parameters.dependencySource, 'mainline') }}:
+          pipelineId: ${{ parameters.stagingPipelineIdentifiers[dependency] }}
+          useMainlineBranch: true
+          latestFromBranch: ${{ parameters.latestFromBranch }}
         ${{ elseif eq(parameters.dependencySource, 'tag-builds') }}:
           pipelineId: ${{ parameters.taggedPipelineIdentifiers[dependency] }}
           latestFromBranch: false
@@ -252,6 +285,37 @@ steps:
         for file in amdclang amdclang++ amdclang-cl amdclang-cpp amdflang amdlld aompcc mygpu mycpu offload-arch; do
           sudo ln -s $(Agent.BuildDirectory)/rocm/llvm/bin/$file $(Agent.BuildDirectory)/rocm/bin/$file
         done
+# dlopen calls within a ctest or pytest sequence runs into issues when shared library symlink convention is not followed
+# the convention is as follows:
+# unversioned .so is a symlink to major version .so
+# major version .so is a symlink to detailed version .so
+# HIP libraries do not follow this convention, and each .so is a copy of each other
+# changing the library structure to follow the symlink convention resolves some test failures
+- ${{ if eq(parameters.setupHIPLibrarySymlinks, true) }}:
+  - task: Bash@3
+    displayName: Setup symlinks for hip libraries
+    inputs:
+      targetType: inline
+      workingDirectory: $(Agent.BuildDirectory)/rocm/lib
+      script: |
+        LIBRARIES=("libamdhip64" "libhiprtc-builtins" "libhiprtc")
+        for LIB_NAME in "${LIBRARIES[@]}"; do
+            VERSIONED_SO=$(ls ${LIB_NAME}.so.* 2>/dev/null | grep -E "${LIB_NAME}\.so\.[0-9]+\.[0-9]+\.[0-9]+(-.*)?" | sort -V | tail -n 1)
+            if [[ -z "$VERSIONED_SO" ]]; then
+                continue
+            fi
+            MAJOR_VERSION=$(echo "$VERSIONED_SO" | grep -oP "${LIB_NAME}\.so\.\K[0-9]+")
+            if [[ -e "${LIB_NAME}.so.${MAJOR_VERSION}" && ! -L "${LIB_NAME}.so.${MAJOR_VERSION}" ]]; then
+                rm -f "${LIB_NAME}.so.${MAJOR_VERSION}"
+            fi
+            if [[ -e "${LIB_NAME}.so" && ! -L "${LIB_NAME}.so" ]]; then
+                rm -f "${LIB_NAME}.so"
+            fi
+            ln -sf "$VERSIONED_SO" "${LIB_NAME}.so.${MAJOR_VERSION}"
+            ln -sf "${LIB_NAME}.so.${MAJOR_VERSION}" "${LIB_NAME}.so"
+            echo "Symlinks created for $LIB_NAME:"
+            ls -l ${LIB_NAME}.so*
+        done
 - task: Bash@3
   displayName: 'List downloaded ROCm files'
   inputs:

.azuredevops/templates/steps/gpu-diagnostics.yml

@@ -0,0 +1,54 @@
+# Diagnostics for GPU-enabled systems
+parameters:
+- name: runRocminfo
+  type: boolean
+  default: true
+
+steps:
+- ${{ if eq(parameters.runRocminfo, true) }}:
+  - task: Bash@3
+    displayName: 'rocminfo'
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: $(Agent.BuildDirectory)/rocm/bin/rocminfo || true
+  - task: Bash@3
+    displayName: 'rocm_agent_enumerator'
+    continueOnError: true
+    inputs:
+      targetType: inline
+      script: $(Agent.BuildDirectory)/rocm/bin/rocm_agent_enumerator || true
+- task: Bash@3
+  displayName: 'List DRI devices'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: ls -la /dev/dri/ || true
+- task: Bash@3
+  displayName: 'List amdgpu/rocm/mesa packages'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: apt list --installed | grep -E 'amdgpu|rocm|mesa' || true
+- task: Bash@3
+  displayName: 'List GPU processes'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      ls /sys/class/kfd/kfd/proc/ || true
+      sudo lsof | grep amdgpu || true
+- task: Bash@3
+  displayName: 'System snapshot'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: top -bn1 || true
+- task: Bash@3
+  displayName: 'List dmesg'
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      echo 'rocm-ci: $(Build.DefinitionName) $(System.DefinitionId)' | sudo tee /dev/kmsg || true
+      sudo dmesg || true

.azuredevops/templates/steps/local-artifact-download.yml

@@ -29,17 +29,17 @@ steps:
         definition: ${{ parameters.definitionId }}
         buildId: ${{ parameters.buildId }}
       itemPattern: '**/*${{ parameters.gpuTarget }}*'
-      targetPath: $(System.ArtifactsDirectory)
+      targetPath: $(Pipeline.Workspace)/d
   - task: ExtractFiles@1
     displayName: 'Extract Pipeline Build'
     inputs:
-      archiveFilePatterns: '$(System.ArtifactsDirectory)/**/*.tar.gz'
+      archiveFilePatterns: '$(Pipeline.Workspace)/d/**/*.tar.gz'
       destinationFolder: '$(Agent.BuildDirectory)/rocm'
       cleanDestinationFolder: false
       overwriteExistingFiles: true
   - task: DeleteFiles@1
     displayName: 'Clean up Compressed Pipeline Build'
     inputs:
-      SourceFolder: '$(System.ArtifactsDirectory)'
+      SourceFolder: '$(Pipeline.Workspace)/d'
       Contents: '/**/*.tar.xz'
       RemoveDotFiles: true

.azuredevops/templates/steps/manifest.yml

@@ -0,0 +1,140 @@
+parameters:
+- name: artifactName
+  type: string
+  default: 'drop'
+- name: gpuTarget
+  type: string
+  default: ''
+
+steps:
+- task: Bash@3
+  displayName: Set up current_repo values
+  condition: always()
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      sudo apt-get install -y jq
+
+      # RESOURCES_REPOSITORIES is a runtime variable (not an env var!) that contains quotations and newlines
+      # So we need to save it to a file to properly preserve its formatting and contents
+      cat <<EOF > resources.repositories
+      $(RESOURCES_REPOSITORIES)
+      EOF
+      cat resources.repositories
+
+      IS_TAG_BUILD=$(jq 'has("release_repo")' resources.repositories)
+      if [ "$IS_TAG_BUILD" = "true" ]; then
+        REPO_TYPE="release_repo" # Triggered by a ROCm/ROCm tag-builds file
+      else
+        REPO_TYPE="self" # Triggered by component repo's rocm-ci.yml file
+      fi
+
+      echo "##vso[task.setvariable variable=current_repo.id;]$(jq .$REPO_TYPE.id resources.repositories | tr -d '"')"
+      echo "##vso[task.setvariable variable=current_repo.name;]$(jq .$REPO_TYPE.name resources.repositories | tr -d '"')"
+      echo "##vso[task.setvariable variable=current_repo.ref;]$(jq .$REPO_TYPE.ref resources.repositories | tr -d '"')"
+      echo "##vso[task.setvariable variable=current_repo.url;]$(jq .$REPO_TYPE.url resources.repositories | tr -d '"')"
+      echo "##vso[task.setvariable variable=current_repo.version;]$(jq .$REPO_TYPE.version resources.repositories | tr -d '"')"
+- task: Bash@3
+  displayName: Create manifest.json
+  condition: always()
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      manifest_json=$(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.json
+
+      dependencies=()
+      for manifest_file in $(Pipeline.Workspace)/d/**/manifest_*.json; do
+        echo "Processing $manifest_file"
+        cat $manifest_file
+        current=$(jq '.current' "$manifest_file")
+        dependencies+=("$current")
+      done
+      dependencies_json=$(printf '%s\n' "${dependencies[@]}" | jq -s '.')
+
+      jq -n \
+        --arg buildNumber "$(Build.BuildNumber)" \
+        --arg buildId "$(Build.BuildId)" \
+        --arg repoId "$(current_repo.id)" \
+        --arg repoName "$(current_repo.name)" \
+        --arg repoRef "$(current_repo.ref)" \
+        --arg repoUrl "$(current_repo.url)" \
+        --arg repoVersion "$(current_repo.version)" \
+        --argjson dependencies "$dependencies_json" \
+        '{
+          current: {
+            buildNumber: $buildNumber,
+            buildId: $buildId,
+            repoId: $repoId,
+            repoName: $repoName,
+            repoRef: $repoRef,
+            repoUrl: $repoUrl,
+            repoVersion: $repoVersion
+          },
+          dependencies: $dependencies
+        }' > $manifest_json
+
+      dependencies_rows=$(cat $manifest_json | \
+        jq -r '
+          .dependencies[] | 
+          "<tr><td>" + .buildNumber + "</td>" + 
+          "<td><a href=\"https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=" + .buildId + "\">" + .buildId + "</a></td>" + 
+          "<td><a href=\"" + .repoUrl + "\">" + .repoName + "</a></td>" + 
+          "<td><a href=\"" + .repoUrl + "/tree/" + .repoRef + "\">" + .repoRef + "</a></td>" + 
+          "<td><a href=\"" + .repoUrl + "/commit/" + .repoVersion + "\">" + .repoVersion + "</a></td></tr>"
+        ')
+      dependencies_rows=$(echo $dependencies_rows)
+      echo "##vso[task.setvariable variable=dependencies_rows;]$dependencies_rows"
+
+      cat $manifest_json
+- task: Bash@3
+  displayName: Create manifest.html
+  condition: always()
+  continueOnError: true
+  inputs:
+    targetType: inline
+    script: |
+      manifest_html=$(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.html
+      cat <<EOF > $manifest_html
+      <html>
+      <h1>Manifest</h1>
+      <h2>Current</h2>
+      <table border="1">
+      <tr>
+        <th>Build Number</th>
+        <th>Build ID</th>
+        <th>Repo Name</th>
+        <th>Repo Ref</th>
+        <th>Repo Version</th>
+      </tr>
+      <tr>
+        <td>$(Build.BuildNumber)</td>
+        <td><a href="https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$(Build.BuildId)">$(Build.BuildId)</a></td>
+        <td><a href="$(current_repo.url)">$(current_repo.name)</a></td>
+        <td><a href="$(current_repo.url)/tree/$(current_repo.ref)">$(current_repo.ref)</a></td>
+        <td><a href="$(current_repo.url)/commit/$(current_repo.version)">$(current_repo.version)</a></td>
+      </tr>
+      </table>
+      <h2>Dependencies</h2>
+      <table border="1">
+      <tr>
+        <th>Build Number</th>
+        <th>Build ID</th>
+        <th>Repo Name</th>
+        <th>Repo Ref</th>
+        <th>Repo Version</th>
+      </tr>
+      $(dependencies_rows)
+      </table>
+      </html>
+      EOF
+
+      cat $manifest_html
+- task: PublishHtmlReport@1
+  displayName: Publish manifest.html
+  condition: always()
+  continueOnError: true
+  inputs:
+    tabName: Manifest
+    reportDir: $(Build.ArtifactStagingDirectory)/manifest_$(Build.DefinitionName)_$(Build.SourceBranchName)_$(Build.BuildId)_$(Build.BuildNumber)_ubuntu2204_${{ parameters.artifactName }}_${{ parameters.gpuTarget }}.html

.azuredevops/templates/steps/miopen-get-ck-build.yml

@@ -0,0 +1,72 @@
+parameters:
+- name: gpuTarget
+  type: string
+  default: ''
+
+steps:
+- task: Bash@3
+  name: downloadCKBuild
+  displayName: Download specific CK build
+  continueOnError: true
+  env:
+    CXX: $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
+    CC: $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
+  inputs:
+    targetType: inline
+    workingDirectory: $(Build.SourcesDirectory)
+    script: |
+      AZ_API="https://dev.azure.com/ROCm-CI/ROCm-CI/_apis"
+      GH_API="https://api.github.com/repos/ROCm"
+      ARTIFACT_NAME="composablekernel.${{ parameters.gpuTarget }}"
+      EXIT_CODE=0
+
+      # The commits that MIOpen reference are all merge commits from CK/develop to CK/amd-develop
+      # These commits are present on CK/amd-develop but not on CK/develop
+      # Ex-CI only builds CK/develop, so we need to find a commit present on both CK/develop and CK/amd-develop
+
+      CK_COMMIT=$(grep 'ROCm/composable_kernel' requirements.txt | sed -E 's/.*@([a-f0-9]{40}).*/\1/')
+      echo "Fetching CK build ID for commit $CK_COMMIT"
+      CK_COMMIT_URL="$GH_API/composable_kernel/commits/${CK_COMMIT}"
+      PARENT_COMMIT=$(curl -s $CK_COMMIT_URL | jq '.parents[1].sha' | tr -d '"')
+      echo "Found parent commit: $PARENT_COMMIT"
+      PARENT_CHECKS_URL="$GH_API/composable_kernel/commits/${PARENT_COMMIT}/check-runs"
+      CK_BUILD_ID=$(curl -s $PARENT_CHECKS_URL | \
+        jq '.check_runs[] | select(.name == "composable_kernel" and .app.slug == "azure-pipelines") | .details_url' | \
+        tr -d '"' | grep -oP 'buildId=\K\d+')
+
+      if [ -z "$CK_BUILD_ID" ]; then
+        echo "Did not find specific CK build ID"
+        LATEST_BUILD_URL="$AZ_API/build/builds?definitions=$(COMPOSABLE_KERNEL_PIPELINE_ID)&status=completed&result=succeeded&\$top=1&api-version=7.1"
+        CK_BUILD_ID=$(curl -s $LATEST_BUILD_URL | jq '.value[0].id')
+        echo "Found latest CK build ID: $CK_BUILD_ID"
+        EXIT_CODE=1
+      fi
+
+      AZURE_URL="$AZ_API/build/builds/$CK_BUILD_ID/artifacts?artifactName=$ARTIFACT_NAME&api-version=7.1"
+      ARTIFACT_URL=$(curl -s $AZURE_URL | jq '.resource.downloadUrl' | tr -d '"')
+
+      if [ -z "$ARTIFACT_URL" ]; then
+        echo "Did not find specific CK build artifact"
+        LATEST_BUILD_URL="$AZ_API/build/builds?definitions=$(COMPOSABLE_KERNEL_PIPELINE_ID)&status=completed&result=succeeded&\$top=1&api-version=7.1"
+        CK_BUILD_ID=$(curl -s $LATEST_BUILD_URL | jq '.value[0].id')
+        echo "Found latest CK build ID: $CK_BUILD_ID"
+        AZURE_URL="$AZ_API/build/builds/$CK_BUILD_ID/artifacts?artifactName=$ARTIFACT_NAME&api-version=7.1"
+        ARTIFACT_URL=$(curl -s $AZURE_URL | jq '.resource.downloadUrl' | tr -d '"')
+        EXIT_CODE=2
+      elif [ $EXIT_CODE -eq 0 ]; then
+        echo "Found specific CK build ID: $CK_BUILD_ID"
+      fi
+
+      echo "Downloading CK artifact from $ARTIFACT_URL"
+      wget -nv $ARTIFACT_URL -O $(System.ArtifactsDirectory)/ck.zip
+      unzip $(System.ArtifactsDirectory)/ck.zip -d $(System.ArtifactsDirectory)
+      mkdir -p $(Agent.BuildDirectory)/rocm
+      tar -zxvf $(System.ArtifactsDirectory)/$ARTIFACT_NAME/*.tar.gz -C $(Agent.BuildDirectory)/rocm
+      rm -r $(System.ArtifactsDirectory)/ck.zip $(System.ArtifactsDirectory)/$ARTIFACT_NAME
+
+      if [ $EXIT_CODE -ne 0 ]; then
+        BUILD_COMMIT=$(curl -s $AZ_API/build/builds/$CK_BUILD_ID | jq '.sourceVersion' | tr -d '"')
+        echo "WARNING: couldn't find a CK build for commit $CK_COMMIT"
+        echo "Instead used latest CK build $CK_BUILD_ID for commit $BUILD_COMMIT"
+      fi
+      exit $EXIT_CODE

.azuredevops/templates/steps/test.yml

@@ -26,25 +26,23 @@ parameters:
 - name: testPublishResults
   type: boolean
   default: true
-- name: reloadAMDGPU
-  type: boolean
-  default: false
+- name: allowPartiallySucceededBuilds
+  type: object
+  default:
+    - amdsmi
+    - aomp
+    - HIPIFY
+    - MIVisionX
+    - rocm-cmake
+    - rocm_smi_lib
+    - roctracer
 
 steps:
-# Avoids occasional AMDGPU driver issues with opening /dev/kfd
-- ${{ if parameters.reloadAMDGPU }}:
-  - task: Bash@3
-    displayName: Unload and reload AMDGPU
-    inputs:
-      targetType: inline
-      script: |
-        sudo modprobe -r amdgpu
-        sudo modprobe amdgpu
 # run test, continue on failure to publish results
 # and to publish build artifacts
 - task: Bash@3
   displayName: '${{ parameters.componentName }} Test'
-  continueOnError: true
+  continueOnError: ${{ containsValue(parameters.allowPartiallySucceededBuilds, parameters.componentName) }}
   inputs:
     targetType: inline
     script: ${{ parameters.testExecutable }} ${{ parameters.testParameters }}
@@ -52,8 +50,8 @@ steps:
 - ${{ if parameters.testPublishResults }}:
   - task: PublishTestResults@2
     displayName: '${{ parameters.componentName }} Publish Results'
+    condition: succeededOrFailed()
     inputs:
       searchFolder: ${{ parameters.testDir }}
       testResultsFormat: ${{ parameters.testOutputFormat }}
       testResultsFiles: '**/${{ parameters.testOutputFile }}'
-    condition: succeededOrFailed()

.azuredevops/variables-global.yml

@@ -1,6 +1,8 @@
 # specify non-secret global variables reused across pipelines here
 
 variables:
+- name: RESOURCES_REPOSITORIES
+  value: $[ convertToJson(resources.repositories) ]
 - name: CI_ROOT_PATH
   value: /.azuredevops
 - name: CI_COMPONENT_PATH
@@ -32,7 +34,7 @@ variables:
 - name: LATEST_DOCKER_VERSION
   value: 6.1
 - name: KEYRING_VERSION
-  value: 6.1
+  value: 6.3
 - name: AMDMIGRAPHX_GFX942_TEST_PIPELINE_ID
   value: 197
 - name: AMDMIGRAPHX_PIPELINE_ID
@@ -79,6 +81,10 @@ variables:
   value: 93
 - name: HIP_TAGGED_PIPELINE_ID
   value: 31
+- name: HIP_TESTS_PIPELINE_ID
+  value: 233
+- name: HIP_TESTS_TAGGED_PIPELINE_ID
+  value: 220
 - name: HIPBLAS_COMMON_PIPELINE_ID
   value: 223
 - name: HIPBLAS_COMMON_TAGGED_PIPELINE_ID
@@ -163,6 +169,10 @@ variables:
   value: 241
 - name: OMNIPERF_TAGGED_PIPELINE_ID
   value: 242
+- name: OMNITRACE_PIPELINE_ID
+  value: 253
+- name: OMNITRACE_TAGGED_PIPELINE_ID
+  value: 252
 - name: RCCL_GFX942_TEST_PIPELINE_ID
   value: 184
 - name: RCCL_PIPELINE_ID
@@ -209,6 +219,10 @@ variables:
   value: 134
 - name: ROCGDB_TAGGED_PIPELINE_ID
   value: 50
+- name: ROCJPEG_PIPELINE_ID
+  value: 262
+- name: ROCJPEG_TAGGED_PIPELINE_ID
+  value: 263
 - name: ROCM_BANDWIDTH_TEST_PIPELINE_ID
   value: 88
 - name: ROCM_BANDWIDTH_TEST_TAGGED_PIPELINE_ID
@@ -223,6 +237,10 @@ variables:
   value: 22
 - name: ROCM_EXAMPLES_GFX942_TEST_PIPELINE_ID
   value: 204
+- name: ROCM_EXAMPLES_PIPELINE_ID
+  value: 216
+- name: ROCM_EXAMPLES_TAGGED_PIPELINE_ID
+  value: 245
 - name: ROCM_SMI_LIB_PIPELINE_ID
   value: 96
 - name: ROCM_SMI_LIB_TAGGED_PIPELINE_ID
@@ -247,14 +265,30 @@ variables:
   value: 20
 - name: ROCPROFILER_GFX942_TEST_PIPELINE_ID
   value: 190
-- name: ROCPROFILER_PIPELINE_ID
-  value: 143
+- name: ROCPROFILER_COMPUTE_PIPELINE_ID
+  value: 257
+- name: ROCPROFILER_COMPUTE_TAGGED_PIPELINE_ID
+  value: 258
 - name: ROCPROFILER_REGISTER_PIPELINE_ID
   value: 1
 - name: ROCPROFILER_REGISTER_TAGGED_PIPELINE_ID
   value: 25
+- name: ROCPROFILER_SDK_PIPELINE_ID
+  value: 246
+- name: ROCPROFILER_SDK_TAGGED_PIPELINE_ID
+  value: 234
+- name: ROCPROFILER_SYSTEMS_PIPELINE_ID
+  value: 255
+- name: ROCPROFILER_SYSTEMS_TAGGED_PIPELINE_ID
+  value: 254
+- name: ROCPROFILER_PIPELINE_ID
+  value: 143
 - name: ROCPROFILER_TAGGED_PIPELINE_ID
   value: 28
+- name: ROCPYDECODE_PIPELINE_ID
+  value: 239
+- name: ROCPYDECODE_TAGGED_PIPELINE_ID
+  value: 232
 - name: ROCR_DEBUG_AGENT_PIPELINE_ID
   value: 136
 - name: ROCR_DEBUG_AGENT_TAGGED_PIPELINE_ID
@@ -309,3 +343,5 @@ variables:
   value: 78
 - name: RPP_TAGGED_PIPELINE_ID
   value: 39
+- name: BOOST_DEPENDENCY_PIPELINE_ID
+  value: 250

.github/CODEOWNERS

@@ -5,3 +5,4 @@ docs/ @amd-aakash @jlgreathouse @samjwu @yhuiYH @ROCm/rocm-documentation
 *.rst @amd-aakash @jlgreathouse @samjwu @yhuiYH @ROCm/rocm-documentation
 # External CI
 /.azuredevops/ @ROCm/external-ci
+tools/rocm-build/ @ROCm/rocm-devops

.wordlist.txt

@@ -13,6 +13,7 @@ AMDMIGraphX
 AMI
 AOCC
 AOMP
+AOTriton
 APBDIS
 APIC
 APIs
@@ -25,6 +26,7 @@ ASm
 ATI
 AddressSanitizer
 AlexNet
+Andrej
 Arb
 Autocast
 BARs
@@ -72,6 +74,7 @@ Conda
 ConnectX
 CuPy
 Dashboarding
+DBRX
 DDR
 DF
 DGEMM
@@ -89,6 +92,8 @@ Dask
 DataFrame
 DataLoader
 DataParallel
+Debian
+DeepSeek
 DeepSpeed
 Dependabot
 Deprecations
@@ -105,6 +110,7 @@ FFT
 FFTs
 FFmpeg
 FHS
+FIXME
 FMA
 FP
 FX
@@ -125,10 +131,12 @@ GDS
 GEMM
 GEMMs
 GFortran
+Gemma
 GiB
 GIM
 GL
 GLXT
+Gloo
 GMI
 GPG
 GPR
@@ -147,6 +155,8 @@ HGX
 HIPCC
 HIPExtension
 HIPIFY
+HIPification
+HIPify
 HPC
 HPCG
 HPE
@@ -158,6 +168,8 @@ HWS
 Haswell
 Higgs
 Hyperparameters
+Huggingface
+ICD
 ICV
 IDE
 IDEs
@@ -180,14 +192,17 @@ Interop
 Intersphinx
 Intra
 Ioffe
+JAX's
 Jinja
 JSON
 Jupyter
 KFD
 KFDTest
-KiB
+KMD
 KV
 KVM
+Karpathy's
+KiB
 Keras
 Khronos
 LAPACK
@@ -208,11 +223,13 @@ MiB
 MIGraphX
 MIOpen
 MIOpenGEMM
+MIOpen's
 MIVisionX
 MLM
 MMA
 MMIO
 MMIOH
+MMU
 MNIST
 MPI
 MSVC
@@ -236,6 +253,8 @@ MyEnvironment
 MyST
 NBIO
 NBIOs
+NCCL
+NCF
 NIC
 NICs
 NLI
@@ -277,10 +296,12 @@ OpenVX
 OpenXLA
 Oversubscription
 PagedAttention
+Pallas
 PCC
 PCI
 PCIe
 PEFT
+PEQT
 PIL
 PILImage
 POR
@@ -295,7 +316,9 @@ PipelineParallel
 PnP
 PowerEdge
 PowerShell
+Profiler's
 PyPi
+Pytest
 PyTorch
 Qcycles
 Qwen
@@ -303,14 +326,17 @@ RAII
 RAS
 RCCL
 RDC
+RDC's
 RDMA
 RDNA
 README
 RHEL
+RMW
 RNN
 RNNs
 ROC
 ROCProfiler
+ROCT
 ROCTracer
 ROCclr
 ROCdbgapi
@@ -322,6 +348,7 @@ ROCmSoftwarePlatform
 ROCmValidationSuite
 ROCprofiler
 ROCr
+RPP
 RST
 RW
 Radeon
@@ -329,6 +356,7 @@ RelWithDebInfo
 Req
 Rickle
 RoCE
+Runfile
 Ryzen
 SALU
 SBIOS
@@ -341,6 +369,7 @@ SENDMSG
 SGPR
 SGPRs
 SHA
+SHARK's
 SIGQUIT
 SIMD
 SIMDs
@@ -374,6 +403,7 @@ TCR
 TF
 TFLOPS
 TP
+TPS
 TPU
 TPUs
 TSME
@@ -384,9 +414,14 @@ TensorFlow
 TensorParallel
 ToC
 TorchAudio
+torchaudio
+TorchElastic
 TorchMIGraphX
+torchrec
 TorchScript
 TorchServe
+torchserve
+torchtext
 TorchVision
 TransferBench
 TrapStatus
@@ -450,10 +485,12 @@ api
 atmi
 atomics
 autogenerated
+autotune
 avx
 awk
 backend
 backends
+benchmarked
 benchmarking
 bfloat
 bilinear
@@ -491,11 +528,15 @@ copyable
 cpp
 csn
 cuBLAS
+cuda
+cuDNN
+cudnn
 cuFFT
 cuLIB
 cuRAND
 cuSOLVER
 cuSPARSE
+customizations
 cTDP
 dataset
 datasets
@@ -505,6 +546,8 @@ datatypes
 dbgapi
 de
 deallocation
+debuggability
+debian
 denoise
 denoised
 denoises
@@ -519,6 +562,9 @@ devsel
 dimensionality
 disambiguates
 distro
+distros
+dkms
+dtype
 el
 embeddings
 enablement
@@ -539,6 +585,7 @@ gRPC
 galb
 gcc
 gdb
+gemm
 gfortran
 gfx
 githooks
@@ -551,8 +598,10 @@ heterogenous
 hipBLAS
 hipBLASLt
 hipBLASLt's
+hipblaslt
 hipCUB
 hipFFT
+hipFORT
 hipLIB
 hipRAND
 hipSOLVER
@@ -574,6 +623,7 @@ hpp
 hsa
 hsakmt
 hyperparameter
+hyperparameters
 iDRAC
 ib_core
 inband
@@ -594,10 +644,13 @@ ipo
 jax
 kdb
 kfd
+kv
 latencies
+len
 libfabric
 libjpeg
 libs
+linalg
 linearized
 linter
 linux
@@ -620,6 +673,8 @@ mutex
 mvffr
 namespace
 namespaces
+nanoGPT
+num
 numref
 ocl
 opencl
@@ -628,8 +683,11 @@ openmp
 openssl
 optimizers
 os
+oversubscription
 pageable
+pallas
 parallelization
+parallelizing
 parameterization
 passthrough
 perfcounter
@@ -642,6 +700,7 @@ prebuilt
 precompiled
 preconditioner
 preconfigured
+preemptible
 prefetch
 prefetchable
 prefill
@@ -658,10 +717,13 @@ profilers
 protobuf
 pseudorandom
 py
+recommender
+recommenders
 quantile
 quantizer
 quasirandom
 queueing
+radeon
 rccl
 rdc
 rdma
@@ -683,6 +745,8 @@ rocALUTION
 rocBLAS
 rocDecode
 rocFFT
+rocHPCG
+rocJPEG
 rocLIB
 rocMLIR
 rocPRIM
@@ -707,16 +771,21 @@ rocsolver
 rocsparse
 rocthrust
 roctracer
+rst
 runtime
 runtimes
 sL
 scalability
 scalable
+scipy
+seealso
 sendmsg
+seqs
 serializers
 shader
 sharding
 sigmoid
+single-node
 sm
 smi
 softmax
@@ -753,8 +822,10 @@ txt
 uarch
 uncached
 uncorrectable
+underoptimized
 unhandled
 uninstallation
+unmapped
 unsqueeze
 unstacking
 unswitching
@@ -774,6 +845,8 @@ vectorize
 vectorized
 vectorizer
 vectorizes
+virtualize
+virtualized
 vjxb
 voxel
 walkthrough

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 - 2024 Advanced Micro Devices, Inc. All rights reserved.
+Copyright (c) 2023 - 2025 Advanced Micro Devices, Inc. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -50,7 +50,7 @@ The following example shows how to use the repo tool to download the ROCm source
 ```bash
 mkdir -p ~/ROCm/
 cd ~/ROCm/
-~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.2.x
+~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.3.x
 ~/bin/repo sync
 ```
 
@@ -76,8 +76,8 @@ The Build time will reduce significantly if we limit the GPU Architecture/s agai
 
 mkdir -p ~/WORKSPACE/      # Or any folder name other than WORKSPACE
 cd ~/WORKSPACE/
-export ROCM_VERSION=6.2.1
-~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.2.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
+export ROCM_VERSION=6.3.1
+~/bin/repo init -u http://github.com/ROCm/ROCm.git -b roc-6.3.x -m tools/rocm-build/rocm-${ROCM_VERSION}.xml
 ~/bin/repo sync
 
 # --------------------------------------
@@ -178,25 +178,9 @@ Source code for our documentation is located in the `/docs` folder of most ROCm
 
 The ROCm documentation homepage is [rocm.docs.amd.com](https://rocm.docs.amd.com).
 
-### Building the documentation
-
-For a quick-start build, use the following code. For more options and detail, refer to
-[Building documentation](./docs/contribute/building.md).
-
-```bash
-cd docs
-pip3 install -r sphinx/requirements.txt
-python3 -m sphinx -T -E -b html -d _build/doctrees -D language=en . _build/html
-```
-
-Alternatively, CMake build is supported.
-
-```bash
-cmake -B build
-cmake --build build --target=doc
-```
+For information on how to contribute to the ROCm documentation, see [Contributing to the ROCm documentation](https://rocm.docs.amd.com/en/latest/contribute/contributing.html).
 
 ## Older ROCm releases
 
 For release information for older ROCm releases, refer to the
-[CHANGELOG](./CHANGELOG.md).
+[ROCm release history](https://rocm.docs.amd.com/en/latest/release/versions.html).

RELEASE.md

@@ -1,77 +1,92 @@
-# ROCm 6.2.1 release notes
+<!-- Do not edit this file!                                 -->
+<!-- This file is autogenerated with                        -->
+<!--   tools/autotag/tag_script.py                          -->
+<!-- Disable lints since this is an auto-generated file.    -->
+<!-- markdownlint-disable blanks-around-headers             -->
+<!-- markdownlint-disable no-duplicate-header               -->
+<!-- markdownlint-disable no-blanks-blockquote              -->
+<!-- markdownlint-disable ul-indent                         -->
+<!-- markdownlint-disable no-trailing-spaces                -->
+<!-- markdownlint-disable reference-links-images            -->
+<!-- markdownlint-disable no-missing-space-atx              -->
+<!-- spellcheck-disable                                     -->
+# ROCm 6.3.1 release notes
 
 The release notes provide a summary of notable changes since the previous ROCm release.
 
-- [Release highlights](release-highlights)
+- [Release highlights](#release-highlights)
 
-- [Operating system and hardware support changes](operating-system-and-hardware-support-changes)
+- [Operating system and hardware support changes](#operating-system-and-hardware-support-changes)
 
-- [ROCm components versioning](rocm-components)
+- [ROCm components versioning](#rocm-components)
 
-- [Detailed component changes](detailed-component-changes)
+- [Detailed component changes](#detailed-component-changes)
 
-- [ROCm known issues](rocm-known-issues)
+- [ROCm known issues](#rocm-known-issues)
 
-- [ROCm upcoming changes](rocm-upcoming-changes)
+- [ROCm resolved issues](#rocm-resolved-issues)
 
-The [Compatibility matrix](https://rocm.docs.amd.com/en/docs-6.2.1/compatibility/compatibility-matrix.html)
-provides the full list of supported hardware, operating systems, ecosystems, third-party components, and ROCm components for each ROCm release.
-
-Release notes for previous ROCm releases are available in earlier versions of the documentation.
-See the [ROCm documentation release history](https://rocm.docs.amd.com/en/latest/release/versions.html).
-
-## Release highlights
-
-The following are notable new features and improvements in ROCm 6.2.1. For changes to individual components, see [Detailed component changes](#detailed-component-changes).
-
-### rocAL major version change
-
-The new version of rocAL introduces many new features, but does not modify any of the existing public API functions. However, the version number was incremented from 1.3 to 2.0.
-Applications linked to version 1.3 must be recompiled to link against version 2.0.
-
-See [the rocAL detailed changes](#rocal-2-0-0) for more information.
-
-### New support for FBGEMM (Facebook General Matrix Multiplication)
-
-As of ROCm 6.2.1, ROCm supports Facebook General Matrix Multiplication (FBGEMM) and the related FBGEMM_GPU library. 
-
-FBGEMM is a low-precision, high-performance CPU kernel library for convolution and matrix multiplication. It is used for server-side inference and as a back end for PyTorch quantized operators. FBGEMM_GPU includes a collection of PyTorch GPU operator libraries for training and inference. For more information, see the ROCm [Model acceleration libraries guide](https://rocm.docs.amd.com/en/docs-6.2.1/how-to/llm-fine-tuning-optimization/model-acceleration-libraries.html)
-and [PyTorch's FBGEMM GitHub repository](https://github.com/pytorch/FBGEMM).
-
-### ROCm Offline Installer Creator changes
-
-The [ROCm Offline Installer Creator 6.2.1](https://rocm.docs.amd.com/projects/install-on-linux/en/docs-6.2.1/install/rocm-offline-installer.html) introduces several new features and improvements including:
-
-* Logging support for create and install logs
-* More stringent checks for Linux versions and distributions
-* Updated prerequisite repositories
-* Fixed CTest issues
-
-### ROCm documentation changes 
-
-The HIP documentation has been updated with several new topics aimed at improving usability
-and providing more detailed information.
-
-* The Programming Model Reference and Understanding the Programming Model topics in HIP have been consolidated into one topic,
-[HIP programming model (conceptual)](https://rocm.docs.amd.com/projects/HIP/en/docs-6.2.1/understand/programming_model.html). 
-
-* The [HIP virtual memory management](https://rocm.docs.amd.com/projects/HIP/en/docs-6.2.1/how-to/virtual_memory.html) and [HIP virtual memory management API](https://rocm.docs.amd.com/projects/HIP/en/docs-6.2.1/reference/virtual_memory_reference.html) topics have been added.
+- [ROCm upcoming changes](#rocm-upcoming-changes)
 
 ```{note}
-To contribute to ROCm documentation, see the [ROCm documentation contribution guidelines](https://rocm.docs.amd.com/en/latest/contribute/contributing.html).
+If you’re using Radeon™ PRO or Radeon GPUs in a workstation setting with a
+display connected, continue to use ROCm 6.2.3. See the [Use ROCm on Radeon GPUs](https://rocm.docs.amd.com/projects/radeon/en/latest/index.html)
+documentation to verify compatibility and system requirements.
 ```
+## Release highlights
+
+The following are notable new features and improvements in ROCm 6.3.1. For changes to individual components, see
+[Detailed component changes](#detailed-component-changes).
+
+### Per queue resiliency for Instinct MI300 accelerators
+
+The AMDGPU driver now includes enhanced resiliency for misbehaving applications on AMD Instinct MI300 accelerators. This helps isolate the impact of misbehaving applications, ensuring other workloads running on the same accelerator are unaffected.
+
+### ROCm Runfile Installer
+
+ROCm 6.3.1 introduces the ROCm Runfile Installer, with initial support for Ubuntu 22.04. The ROCm Runfile Installer facilitates ROCm installation without using a native Linux package management system, with or without network or internet access. For more information, see the [ROCm Runfile Installer documentation](https://rocm.docs.amd.com/projects/install-on-linux/en/docs-6.3.1/install/rocm-runfile-installer.html).
+
+### ROCm documentation updates
+
+ROCm documentation continues to be updated to provide clearer and more comprehensive guidance for a wider variety of user needs and use cases.
+
+* Added documentation on training a model with ROCm Megatron-LM. AMD offers a Docker image for MI300X accelerators
+  containing essential components to get started, including ROCm libraries, PyTorch, and Megatron-LM utilities. See
+  [Training a model using ROCm Megatron-LM](https://rocm.docs.amd.com/en/docs-6.3.1/how-to/rocm-for-ai/train-a-model.html)
+  to get started.
+
+  The new ROCm Megatron-LM training Docker accompanies the [ROCm vLLM inference
+  Docker](https://rocm.docs.amd.com/en/docs-6.3.1/how-to/performance-validation/mi300x/vllm-benchmark.html)
+  as a set of ready-to-use containerized solutions to get started with using ROCm
+  for AI.
+
+* Updated the [Instinct MI300X workload tuning
+  guide](https://rocm.docs.amd.com/en/docs-6.3.1/how-to/tuning-guides/mi300x/workload.html) with more current optimization
+  strategies. The updated sections include guidance on vLLM optimization, PyTorch TunableOp, and hipBLASLt tuning.
+
+* HIP graph-safe libraries operate safely in HIP execution graphs. [HIP graphs](https://rocm.docs.amd.com/projects/HIP/en/docs-6.3.1/how-to/hip_runtime_api/hipgraph.html#how-to-hip-graph) are an alternative way of executing tasks on a GPU that can provide performance benefits over launching kernels using the standard method via streams. A topic that shows whether a [ROCm library is graph-safe](https://rocm.docs.amd.com/en/docs-6.3.1/reference/graph-safe-support.html) has been added.
+
+* The [Device memory](https://rocm.docs.amd.com/projects/HIP/en/docs-6.3.1/how-to/hip_runtime_api/memory_management/device_memory.html) topic in the HIP memory management section has been updated.
+
+* The HIP documentation has expanded with new resources for developers:
+  * [Multi device management](https://rocm.docs.amd.com/projects/HIP/en/docs-6.3.1/how-to/hip_runtime_api/multi_device.html)
+  * [OpenGL interoperability](https://rocm.docs.amd.com/projects/HIP/en/docs-6.3.1/how-to/hip_runtime_api/opengl_interop.html)
 
 ## Operating system and hardware support changes
 
-ROCm 6.2.1 adds support for Ubuntu 24.04.1 (kernel: 6.8 [GA]).
+ROCm 6.3.1 adds support for Debian 12 (kernel: 6.1). Debian is supported only on AMD Instinct accelerators. See the installation instructions at [Debian native installation](https://rocm.docs.amd.com/projects/install-on-linux/en/docs-6.3.1/install/native-install/debian.html).
 
-See the [Compatibility matrix](https://rocm.docs.amd.com/en/docs-6.2.1/compatibility/compatibility-matrix.html) for the full list of supported operating systems and hardware architectures.
+ROCm 6.3.1 enables support for AMD Instinct MI325X accelerator. For more information, see [AMD Instinct™ MI325X Accelerators](https://www.amd.com/en/products/accelerators/instinct/mi300/mi325x.html).
+
+See the [Compatibility
+matrix](https://rocm.docs.amd.com/en/docs-6.3.1/compatibility/compatibility-matrix.html)
+for more information about operating system and hardware compatibility.
 
 ## ROCm components
 
-The following table lists the versions of ROCm components for ROCm 6.2.1, including any version changes from 6.2.0 to 6.2.1. 
-
-Click the component's updated version to go to a detailed list of its changes. Click <i class="fab fa-github fa-lg"></i> to go to the component's source code on GitHub.
+The following table lists the versions of ROCm components for ROCm 6.3.1, including any version
+changes from 6.3.0 to 6.3.1. Click the component's updated version to go to a list of its changes.
+Click {fab}`github` to go to the component's source code on GitHub.
 
 <div class="pst-scrollable-table-container">
     <table id="rocm-rn-components" class="table">
@@ -90,333 +105,302 @@ Click the component's updated version to go to a detailed list of its changes. C
         </colgroup>
         <tbody class="rocm-components-libs rocm-components-ml">
             <tr>
-                <th rowspan="8">Libraries</th>
-                <th rowspan="8">Machine learning and computer vision</th>
-                <td><a href="https://rocm.docs.amd.com/projects/composable_kernel/en/docs-6.2.1">Composable Kernel</a>
-                </td>
+                <th rowspan="9">Libraries</th>
+                <th rowspan="9">Machine learning and computer vision</th>
+                <td><a href="https://rocm.docs.amd.com/projects/composable_kernel/en/docs-6.3.1/index.html">Composable Kernel</a></td>
                 <td>1.1.0</td>
-                <td><a href="https://github.com/ROCm/composable_kernel/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://github.com/ROCm/composable_kernel"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/AMDMIGraphX/en/docs-6.2.1">MIGraphX</a></td>
-                <td>2.10</td>
-                <td><a href="https://github.com/ROCm/AMDMIGraphX/releases/tag/rocm-6.2.1"><i class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/AMDMIGraphX/en/docs-6.3.1/index.html">MIGraphX</a></td>
+                <td>2.11.0</td>
+                <td><a href="https://github.com/ROCm/AMDMIGraphX"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/MIOpen/en/docs-6.2.1">MIOpen</a></td>
-                <td>3.2.0</td>
-                <td><a href="https://github.com/ROCm/MIOpen/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/MIOpen/en/docs-6.3.1/index.html">MIOpen</a></td>
+                <td>3.3.0</td>
+                <td><a href="https://github.com/ROCm/MIOpen"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/MIVisionX/en/docs-6.2.1">MIVisionX</a></td>
-                <td>3.0.0</td>
-                <td><a href="https://github.com/ROCm/MIVisionX/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/MIVisionX/en/docs-6.3.1/index.html">MIVisionX</a></td>
+                <td>3.1.0&nbsp;&Rightarrow;&nbsp;<a href="#mivisionx-3-1-0">3.1.0</a></td>
+                <td><a href="https://github.com/ROCm/MIVisionX"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocAL/en/docs-6.2.1">rocAL</a></td>
-                <td>1.0.0&nbsp;&Rightarrow;&nbsp;<a href="#rocal-2-0-0">2.0.0</a></td>
-                <td><a href="https://github.com/ROCm/rocAL/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocAL/en/docs-6.3.1/index.html">rocAL</a></td>
+                <td>2.1.0</td>
+                <td><a href="https://github.com/ROCm/rocAL"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocDecode/en/docs-6.2.1">rocDecode</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocDecode/en/docs-6.3.1/index.html">rocDecode</a></td>
+                <td>0.8.0</td>
+                <td><a href="https://github.com/ROCm/rocDecode"><i class="fab fa-github fa-lg"></i></a></td>
+            </tr>
+            <tr>
+                <td><a href="https://rocm.docs.amd.com/projects/rocJPEG/en/docs-6.3.1/index.html">rocJPEG</a></td>
                 <td>0.6.0</td>
-                <td><a href="https://github.com/ROCm/rocDecode/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://github.com/ROCm/rocJPEG"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocPyDecode/en/docs-6.2.1">rocPyDecode</a></td>
-                <td>0.1.0</td>
-                <td><a href="https://github.com/ROCm/rocPyDecode/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocPyDecode/en/docs-6.3.1/index.html">rocPyDecode</a></td>
+                <td>0.2.0</td>
+                <td><a href="https://github.com/ROCm/rocPyDecode"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rpp/en/docs-6.2.1">RPP</a></td>
-                <td>1.8.0</td>
-                <td><a href="https://github.com/ROCm/rpp/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rpp/en/docs-6.3.1/index.html">RPP</a></td>
+                <td>1.9.1</td>
+                <td><a href="https://github.com/ROCm/rpp"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
-        <tbody class="rocm-components-libs rocm-components-communication">
+        <tbody class="rocm-components-libs rocm-components-communication tbody-reverse-zebra">
             <tr>
                 <th rowspan="1"></th>
                 <th rowspan="1">Communication</th>
-                <td><a href="https://rocm.docs.amd.com/projects/rccl/en/docs-6.2.1">RCCL</a></td>
-                <td>2.20.5&nbsp;&Rightarrow;&nbsp;<a href="#rccl-2-20-5">2.20.5</a></td>
-                <td><a href="https://github.com/ROCm/rccl/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rccl/en/docs-6.3.1/index.html">RCCL</a></td>
+                <td>2.21.5&nbsp;&Rightarrow;&nbsp;<a href="#rccl-2-21-5">2.21.5</a></td>
+                <td><a href="https://github.com/ROCm/rccl"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
-        <tbody class="rocm-components-libs rocm-components-math tbody-reverse-zebra">
+        <tbody class="rocm-components-libs rocm-components-math">
             <tr>
                 <th rowspan="16"></th>
                 <th rowspan="16">Math</th>
-                <td><a href="https://rocm.docs.amd.com/projects/hipBLAS/en/docs-6.2.1">hipBLAS</a></td>
-                <td>2.2.0</td>
-                <td><a href="https://github.com/ROCm/hipBLAS/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipBLAS/en/docs-6.3.1/index.html">hipBLAS</a></td>
+                <td>2.3.0</td>
+                <td><a href="https://github.com/ROCm/hipBLAS"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipBLASLt/en/docs-6.2.1">hipBLASLt</a></td>
-                <td>0.8.0</td>
-                <td><a href="https://github.com/ROCm/hipBLASLt/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipBLASLt/en/docs-6.3.1/index.html">hipBLASLt</a></td>
+                <td>0.10.0</td>
+                <td><a href="https://github.com/ROCm/hipBLASLt"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipFFT/en/docs-6.2.1">hipFFT</a></td>
-                <td>1.0.15</td>
-                <td><a href="https://github.com/ROCm/hipFFT/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipFFT/en/docs-6.3.1/index.html">hipFFT</a></td>
+                <td>1.0.17</td>
+                <td><a href="https://github.com/ROCm/hipFFT"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipfort/en/docs-6.2.1">hipfort</a></td>
-                <td>0.4.0</td>
-                <td><a href="https://github.com/ROCm/hipfort/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipfort/en/docs-6.3.1/index.html">hipfort</a></td>
+                <td>0.5.0</td>
+                <td><a href="https://github.com/ROCm/hipfort"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipRAND/en/docs-6.2.1">hipRAND</a></td>
-                <td>2.11.0</td>
-                <td><a href="https://github.com/ROCm/hipRAND/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipRAND/en/docs-6.3.1/index.html">hipRAND</a></td>
+                <td>2.11.1</td>
+                <td><a href="https://github.com/ROCm/hipRAND"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipSOLVER/en/docs-6.2.1">hipSOLVER</a></td>
-                <td>2.2.0</td>
-                <td><a href="https://github.com/ROCm/hipSOLVER/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipSOLVER/en/docs-6.3.1/index.html">hipSOLVER</a></td>
+                <td>2.3.0</td>
+                <td><a href="https://github.com/ROCm/hipSOLVER"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipSPARSE/en/docs-6.2.1">hipSPARSE</a></td>
-                <td>3.1.1</td>
-                <td><a href="https://github.com/ROCm/hipSPARSE/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipSPARSE/en/docs-6.3.1/index.html">hipSPARSE</a></td>
+                <td>3.1.2</td>
+                <td><a href="https://github.com/ROCm/hipSPARSE"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipSPARSELt/en/docs-6.2.1">hipSPARSELt</a></td>
-                <td>0.2.1</td>
-                <td><a href="https://github.com/ROCm/hipSPARSELt/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipSPARSELt/en/docs-6.3.1/index.html">hipSPARSELt</a></td>
+                <td>0.2.2</td>
+                <td><a href="https://github.com/ROCm/hipSPARSELt"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocALUTION/en/docs-6.2.1">rocALUTION</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocALUTION/en/docs-6.3.1/index.html">rocALUTION</a></td>
+                <td>3.2.1</td>
+                <td><a href="https://github.com/ROCm/rocALUTION"><i class="fab fa-github fa-lg"></i></a></td>
+            </tr>
+            <tr>
+                <td><a href="https://rocm.docs.amd.com/projects/rocBLAS/en/docs-6.3.1/index.html">rocBLAS</a></td>
+                <td>4.3.0</td>
+                <td><a href="https://github.com/ROCm/rocBLAS"><i class="fab fa-github fa-lg"></i></a></td>
+            </tr>
+            <tr>
+                <td><a href="https://rocm.docs.amd.com/projects/rocFFT/en/docs-6.3.1/index.html">rocFFT</a></td>
+                <td>1.0.31</td>
+                <td><a href="https://github.com/ROCm/rocFFT"><i class="fab fa-github fa-lg"></i></a></td>
+            </tr>
+            <tr>
+                <td><a href="https://rocm.docs.amd.com/projects/rocRAND/en/docs-6.3.1/index.html">rocRAND</a></td>
                 <td>3.2.0</td>
-                <td><a href="https://github.com/ROCm/rocALUTION/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://github.com/ROCm/rocRAND"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocBLAS/en/docs-6.2.1">rocBLAS</a></td>
-                <td>4.1.2&nbsp;&Rightarrow;&nbsp;<a href="#rocblas-4-2-1">4.2.1</a></td>
-                <td><a href="https://github.com/ROCm/rocBLAS/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocSOLVER/en/docs-6.3.1/index.html">rocSOLVER</a></td>
+                <td>3.27.0</td>
+                <td><a href="https://github.com/ROCm/rocSOLVER"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocFFT/en/docs-6.2.1">rocFFT</a></td>
-                <td>1.0.28&nbsp;&Rightarrow;&nbsp;<a href="#rocfft-1-0-29">1.0.29</a></td>
-                <td><a href="https://github.com/ROCm/rocFFT/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocSPARSE/en/docs-6.3.1/index.html">rocSPARSE</a></td>
+                <td>3.3.0</td>
+                <td><a href="https://github.com/ROCm/rocSPARSE"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocRAND/en/docs-6.2.1">rocRAND</a></td>
-                <td>3.1.0</td>
-                <td><a href="https://github.com/ROCm/rocRAND/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocWMMA/en/docs-6.3.1/index.html">rocWMMA</a></td>
+                <td>1.6.0</td>
+                <td><a href="https://github.com/ROCm/rocWMMA"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocSOLVER/en/docs-6.2.1">rocSOLVER</a></td>
-                <td>3.26.0</td>
-                <td><a href="https://github.com/ROCm/rocSOLVER/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
-            </tr>
-            <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocSPARSE/en/docs-6.2.1">rocSPARSE</a></td>
-                <td>3.2.0</td>
-                <td><a href="https://github.com/ROCm/rocSPARSE/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
-            </tr>
-            <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocWMMA/en/docs-6.2.1">rocWMMA</a></td>
-                <td>1.5.0</td>
-                <td><a href="https://github.com/ROCm/rocWMMA/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
-            </tr>
-            <tr>
-                <td><a href="https://github.com/ROCm/Tensile">Tensile</a></td>
-                <td>4.41.0</td>
-                <td><a href="https://github.com/ROCm/tensile/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/Tensile/en/docs-6.3.1/src/index.html">Tensile</a></td>
+                <td>4.42.0</td>
+                <td><a href="https://github.com/ROCm/Tensile"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
-        <tbody class="rocm-components-libs rocm-components-primitives tbody-reverse-zebra">
+        <tbody class="rocm-components-libs rocm-components-primitives">
             <tr>
                 <th rowspan="4"></th>
                 <th rowspan="4">Primitives</th>
-                <td><a href="https://rocm.docs.amd.com/projects/hipCUB/en/docs-6.2.1">hipCUB</a></td>
-                <td>3.2.0</td>
-                <td><a href="https://github.com/ROCm/hipCUB/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipCUB/en/docs-6.3.1/index.html">hipCUB</a></td>
+                <td>3.3.0</td>
+                <td><a href="https://github.com/ROCm/hipCUB"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/hipTensor/en/docs-6.2.1">hipTensor</a></td>
-                <td>1.3.0</td>
-                <td><a href="https://github.com/ROCm/hipTensor/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/hipTensor/en/docs-6.3.1/index.html">hipTensor</a></td>
+                <td>1.4.0</td>
+                <td><a href="https://github.com/ROCm/hipTensor"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocPRIM/en/docs-6.2.1">rocPRIM</a></td>
-                <td>3.2.0&nbsp;&Rightarrow;&nbsp;<a href="#rocprim-3-2-1">3.2.1</a></td>
-                <td><a href="https://github.com/ROCm/rocPRIM/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocPRIM/en/docs-6.3.1/index.html">rocPRIM</a></td>
+                <td>3.3.0</td>
+                <td><a href="https://github.com/ROCm/rocPRIM"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocThrust/en/docs-6.2.1">rocThrust</a></td>
-                <td>3.1.0</td>
-                <td><a href="https://github.com/ROCm/rocThrust/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocThrust/en/docs-6.3.1/index.html">rocThrust</a></td>
+                <td>3.3.0</td>
+                <td><a href="https://github.com/ROCm/rocThrust"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
-        <tbody class="rocm-components-tools rocm-components-system tbody-reverse-zebra">
+        <tbody class="rocm-components-tools rocm-components-system">
             <tr>
-                <th rowspan="6">Tools</th>
-                <th rowspan="6">System management</th>
-                <td><a href="https://rocm.docs.amd.com/projects/amdsmi/en/docs-6.2.1">AMD SMI</a></td>
-                <td>24.6.2&nbsp;&Rightarrow;&nbsp;<a href="#amd-smi-24-6-3">24.6.3</a></td>
-                <td><a href="https://github.com/ROCm/amdsmi/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <th rowspan="7">Tools</th>
+                <th rowspan="7">System management</th>
+                <td><a href="https://rocm.docs.amd.com/projects/amdsmi/en/docs-6.3.1/index.html">AMD SMI</a></td>
+                <td>24.7.1&nbsp;&Rightarrow;&nbsp;<a href="#amd-smi-24-7-1">24.7.1</a></td>
+                <td><a href="https://github.com/ROCm/amdsmi"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocminfo/en/docs-6.2.1">rocminfo</a></td>
-                <td>1.0.0</td>
-                <td><a href="https://github.com/ROCm/rocminfo/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
-            </tr>
-            <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rdc/en/docs-6.2.1">ROCm Data Center Tool</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rdc/en/docs-6.3.1/index.html">ROCm Data Center Tool</a></td>
                 <td>0.3.0</td>
-                <td><a href="https://github.com/ROCm/rdc/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://github.com/ROCm/rdc"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocm_smi_lib/en/docs-6.2.1">ROCm SMI</a></td>
-                <td>7.3.0&nbsp;&Rightarrow;&nbsp;<a href="#rocm-smi-7-3-0">7.3.0</a></td>
-                <td><a href="https://github.com/ROCm/rocm_smi_lib/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
-            </tr>
-            <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/ROCmValidationSuite/en/docs-6.2.1">ROCm Validation Suite</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocminfo/en/docs-6.3.1/index.html">rocminfo</a></td>
                 <td>1.0.0</td>
-                <td><a href="https://github.com/ROCm/ROCmValidationSuite/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://github.com/ROCm/rocminfo"><i class="fab fa-github fa-lg"></i></a></td>
+            </tr>
+            <tr>
+                <td><a href="https://rocm.docs.amd.com/projects/rocm_smi_lib/en/docs-6.3.1/index.html">ROCm SMI</a></td>
+                <td>7.4.0</td>
+                <td><a href="https://github.com/ROCm/rocm_smi_lib"><i class="fab fa-github fa-lg"></i></a></td>
+            </tr>
+            <tr>
+                <td><a href="https://rocm.docs.amd.com/projects/ROCmValidationSuite/en/docs-6.3.1/index.html">ROCmValidationSuite</a></td>
+                <td>1.1.0</td>
+                <td><a href="https://github.com/ROCm/ROCmValidationSuite"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
-        <tbody class="rocm-components-tools rocm-components-perf">
+        <tbody class="rocm-components-tools rocm-components-perf tbody-reverse-zebra">
             <tr>
                 <th rowspan="6"></th>
                 <th rowspan="6">Performance</th>
-                <td><a href="https://rocm.docs.amd.com/projects/omniperf/en/docs-6.2.1">Omniperf</a></td>
-                <td>2.0.1&nbsp;&Rightarrow;&nbsp;<a href="#omniperf-2-0-1">2.0.1</a></td>
-                <td><a href="https://github.com/ROCm/omniperf/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
-            </tr>
-            <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/omnitrace/en/docs-6.2.1">Omnitrace</a></td>
-                <td>1.11.2&nbsp;&Rightarrow;&nbsp;<a href="#omnitrace-1-11-2">1.11.2</a></td>
-                <td><a href="https://github.com/ROCm/omnitrace/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
-            </tr>
-            <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocm_bandwidth_test/en/docs-6.2.1">ROCm Bandwidth
+                <td><a href="https://rocm.docs.amd.com/projects/rocm_bandwidth_test/en/docs-6.3.1/index.html">ROCm Bandwidth
                         Test</a></td>
                 <td>1.4.0</td>
-                <td><a href="https://github.com/ROCm/rocm_bandwidth_test/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/rocm_bandwidth_test/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocprofiler/en/docs-6.2.1/">ROCProfiler</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/rocprofiler-compute/en/docs-6.3.1/index.html">ROCm Compute Profiler</a></td>
+                <td>3.0.0&nbsp;&Rightarrow;&nbsp<a href="#rocm-compute-profiler-3-0-0">3.0.0</a></td>
+                <td><a href="https://github.com/ROCm/rocprofiler-compute"><i
+                            class="fab fa-github fa-lg"></i></a></td>
+            </tr>
+            <tr>
+                <td><a href="https://rocm.docs.amd.com/projects/rocprofiler-systems/en/docs-6.3.1/index.html">ROCm Systems Profiler</a></td>
+                <td>0.1.0&nbsp;&Rightarrow;&nbsp<a href="#rocm-systems-profiler-0-1-0">0.1.0</a></td>
+                <td><a href="https://github.com/ROCm/rocprofiler-systems"><i
+                            class="fab fa-github fa-lg"></i></a></td>
+            </tr>
+            <tr>
+                <td><a href="https://rocm.docs.amd.com/projects/rocprofiler/en/docs-6.3.1/index.html">ROCProfiler</a></td>
                 <td>2.0.0</td>
-                <td><a href="https://github.com/ROCm/ROCProfiler/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/ROCProfiler/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocprofiler-sdk/en/docs-6.2.1">ROCprofiler-SDK</a></td>
-                <td>0.4.0</td>
-                <td><a href="https://github.com/ROCm/rocprofiler-sdk/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/rocprofiler-sdk/en/docs-6.3.1/index.html">ROCprofiler-SDK</a></td>
+                 <td>0.5.0&nbsp;&Rightarrow;&nbsp;<a href="#rocprofiler-sdk-0-5-0">0.5.0</a></td>
+                <td><a href="https://github.com/ROCm/rocprofiler-sdk/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr >
-                <td><a href="https://rocm.docs.amd.com/projects/roctracer/en/docs-6.2.1/">ROCTracer</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/roctracer/en/docs-6.3.1/index.html">ROCTracer</a></td>
                 <td>4.1.0</td>
-                <td><a href="https://github.com/ROCm/ROCTracer/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/ROCTracer/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
-        <tbody class="rocm-components-tools rocm-components-dev">
+        <tbody class="rocm-components-tools rocm-components-dev tbody-reverse-zebra">
             <tr>
                 <th rowspan="5"></th>
                 <th rowspan="5">Development</th>
-                <td><a href="https://rocm.docs.amd.com/projects/HIPIFY/en/docs-6.2.1/">HIPIFY</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/HIPIFY/en/docs-6.3.1/index.html">HIPIFY</a></td>
                 <td>18.0.0&nbsp;&Rightarrow;&nbsp;<a href="#hipify-18-0-0">18.0.0</a></td>
-                <td><a href="https://github.com/ROCm/HIPIFY/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/HIPIFY/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/ROCdbgapi/en/docs-6.2.1">ROCdbgapi</a></td>
-                <td>0.76.0</td>
-                <td><a href="https://github.com/ROCm/ROCdbgapi/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/ROCdbgapi/en/docs-6.3.1/index.html">ROCdbgapi</a></td>
+                <td>0.77.0</td>
+                <td><a href="https://github.com/ROCm/ROCdbgapi/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/ROCmCMakeBuildTools/en/docs-6.2.1/">ROCm CMake</a></td>
-                <td>0.13.0</td>
-                <td><a href="https://github.com/ROCm/rocm-cmake/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://rocm.docs.amd.com/projects/ROCmCMakeBuildTools/en/docs-6.3.1/index.html">ROCm CMake</a></td>
+                <td>0.14.0</td>
+                <td><a href="https://github.com/ROCm/rocm-cmake/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/ROCgdb/en/docs-6.2.1">ROCm Debugger (ROCgdb)</a>
+                <td><a href="https://rocm.docs.amd.com/projects/ROCgdb/en/docs-6.3.1/index.html">ROCm Debugger (ROCgdb)</a>
                 </td>
-                <td>14.2</td>
-                <td><a href="https://github.com/ROCm/ROCgdb/releases/tag/rocm-6.2.1"><i
+                <td>15.2</td>
+                <td><a href="https://github.com/ROCm/ROCgdb/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/rocr_debug_agent/en/docs-6.2.1">ROCr Debug Agent</a>
+                <td><a href="https://rocm.docs.amd.com/projects/rocr_debug_agent/en/docs-6.3.1/index.html">ROCr Debug Agent</a>
                 </td>
                 <td>2.0.3</td>
-                <td><a href="https://github.com/ROCm/rocr_debug_agent/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/rocr_debug_agent/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
-        <tbody class="rocm-components-compilers tbody-reverse-zebra">
+        <tbody class="rocm-components-compilers">
             <tr>
                 <th rowspan="2" colspan="2">Compilers</th>
-                <td><a href="https://rocm.docs.amd.com/projects/HIPCC/en/docs-6.2.1">HIPCC</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/HIPCC/en/docs-6.3.1/index.html">HIPCC</a></td>
                 <td>1.1.1</td>
-                <td><a href="https://github.com/ROCm/llvm-project/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/llvm-project/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/llvm-project/en/docs-6.2.1">llvm-project</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/llvm-project/en/docs-6.3.1/index.html">llvm-project</a></td>
                 <td>18.0.0</td>
-                <td><a href="https://github.com/ROCm/llvm-project/releases/tag/rocm-6.2.1"><i
+                <td><a href="https://github.com/ROCm/llvm-project/"><i
                             class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
-        <tbody class="rocm-components-runtimes tbody-reverse-zebra">
+        <tbody class="rocm-components-runtimes">
             <tr>
                 <th rowspan="2" colspan="2">Runtimes</th>
-                <td><a href="https://rocm.docs.amd.com/projects/HIP/en/docs-6.2.1">HIP</a></td>
-                <td>6.2&nbsp;&Rightarrow;&nbsp;<a href="#hip-6-2-1">6.2.1</a></td>
-                <td><a href="https://github.com/ROCm/HIP/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/HIP/en/docs-6.3.1/index.html">HIP</a></td>
+                <td>6.3.0&nbsp;&Rightarrow;&nbsp;<a href="#hip-6-3-1">6.3.1</a></td>
+                <td><a href="https://github.com/ROCm/HIP/"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
             <tr>
-                <td><a href="https://rocm.docs.amd.com/projects/ROCR-Runtime/en/docs-6.2.1">ROCr Runtime</a></td>
+                <td><a href="https://rocm.docs.amd.com/projects/ROCR-Runtime/en/docs-6.3.1/index.html">ROCr Runtime</a></td>
                 <td>1.14.0</td>
-                <td><a href="https://github.com/ROCm/ROCR-Runtime/releases/tag/rocm-6.2.1"><i
-                            class="fab fa-github fa-lg"></i></a></td>
+                <td><a href="https://github.com/ROCm/ROCR-Runtime/"><i class="fab fa-github fa-lg"></i></a></td>
             </tr>
         </tbody>
     </table>
@@ -426,171 +410,152 @@ Click the component's updated version to go to a detailed list of its changes. C
 
 The following sections describe key changes to ROCm components.
 
-### **AMD SMI** (24.6.3)
+### **AMD SMI** (24.7.1)
 
-#### Changes
+#### Changed
 
-* Added `amd-smi static --ras` on Guest VMs. Guest VMs can view enabled/disabled RAS features on Host cards.
-
-#### Removals
-
-* Removed `amd-smi metric --ecc` & `amd-smi metric --ecc-blocks` on Guest VMs. Guest VMs do not support getting current ECC counts from the Host cards.
+* `amd-smi monitor` displays `VCLOCK` and `DCLOCK` instead of `ENC_CLOCK` and `DEC_CLOCK`.
 
 #### Resolved issues
 
-* Fixed TypeError in `amd-smi process -G`.
-* Updated CLI error strings to handle empty and invalid GPU/CPU inputs.
-* Fixed Guest VM showing passthrough options.
-* Fixed firmware formatting where leading 0s were missing.
+* Fixed `amd-smi monitor`'s reporting of encode and decode information. `VCLOCK` and `DCLOCK` are
+  now associated with both `ENC_UTIL` and `DEC_UTIL`.
 
-### **HIP** (6.2.1)
+```{note}
+See the full [AMD SMI changelog](https://github.com/ROCm/amdsmi/blob/6.3.x/CHANGELOG.md) for more details and examples.
+```
 
-#### Resolved issues
+### **HIP** (6.3.1)
 
-* Soft hang when using `AMD_SERIALIZE_KERNEL`
-* Memory leak in `hipIpcCloseMemHandle`
+#### Added
+
+* An activeQueues set that tracks only the queues that have a command submitted to them, which allows fast iteration in `waitActiveStreams`.
+
+#### Optimized
+
+* Mechanism of preventing `hipLaunchKernel` latency degradation with number of idle streams is implemented for performance improvement.
 
 ### **HIPIFY** (18.0.0)
 
-#### Changes
+#### Added
 
-* Added CUDA 12.5.1 support.
-* Added cuDNN 9.2.1 support.
-* Added LLVM 18.1.8 support.
-* Added `hipBLAS` 64-bit APIs support.
-* Added Support for math constants `math_constants.h`.
-
-### **Omniperf** (2.0.1)
-
-#### Changes
-
-* Enabled rocprofv1 for MI300 hardware.
-* Added dependency checks on application launch.
-* Updated Omniperf packaging.
-* Rolled back Grafana version in Dockerfile for Angular plugin compatibility.
-* Added GPU model distinction for MI300 systems.
-* Refactored and updated documemtation.
+* Support for: 
+  * NVIDIA CUDA 12.6.2
+  * cuDNN 9.5.1
+  * LLVM 19.1.3
+  * Full `hipBLAS` 64-bit APIs
+  * Full `rocBLAS` 64-bit APIs
 
 #### Resolved issues
 
-* Fixed an issue with analysis output.
-* Fixed issues with profiling multi-process and multi-GPU applications.
+* Added missing support for device intrinsics and built-ins: `__all_sync`, `__any_sync`, `__ballot_sync`, `__activemask`, `__match_any_sync`, `__match_all_sync`, `__shfl_sync`, `__shfl_up_sync`, `__shfl_down_sync`, and `__shfl_xor_sync`.
 
-#### Optimizations
+### **MIVisionX** (3.1.0)
 
-* Reduced running time of Omniperf when profiling.
-* Improved console logging.
+#### Changed
 
-### **Omnitrace** (1.11.2)
-
-#### Known issues
-
-Perfetto can no longer open Omnitrace proto files. Loading Perfetto trace output `.proto` files in the latest version of `ui.perfetto.dev` can result in a dialog with the message, "Oops, something went wrong! Please file a bug." The information in the dialog will refer to an "Unknown field type." The workaround is to open the files with the previous version of the Perfetto UI found at [https://ui.perfetto.dev/v46.0-35b3d9845/#!/](https://ui.perfetto.dev/v46.0-35b3d9845/#!/).
-
-See [issue #3767](https://github.com/ROCm/ROCm/issues/3767) on GitHub.
-
-### **RCCL** (2.20.5)
-
-#### Known issues
-
-On systems running Linux kernel 6.8.0, such as Ubuntu 24.04, Direct Memory Access (DMA) transfers between the GPU and NIC are disabled and impacts multi-node RCCL performance.
-This issue was reproduced with RCCL 2.20.5 (ROCm 6.2.0 and 6.2.1) on systems with Broadcom Thor-2 NICs and affects other systems with RoCE networks using Linux 6.8.0 or newer.
-Older RCCL versions are also impacted.
-
-This issue will be addressed in a future ROCm release.
-
-See [issue #3772](https://github.com/ROCm/ROCm/issues/3772) on GitHub.
-
-### **rocAL** (2.0.0)
-
-#### Changes
- 
-* The new version of rocAL introduces many new features, but does not modify any of the existing public API functions.However, the version number was incremented from 1.3 to 2.0.
-  Applications linked to version 1.3 must be recompiled to link against version 2.0.
-* Added development and test packages.
-* Added C++ rocAL audio unit test and Python script to run and compare the outputs.
-* Added Python support for audio decoders.
-* Added Pytorch iterator for audio.
-* Added Python audio unit test and support to verify outputs.
-* Added rocDecode for HW decode.
-* Added support for: 
-    * Audio loader and decoder, which uses libsndfile library to decode wav files
-    * Audio augmentation - PreEmphasis filter, Spectrogram, ToDecibels, Resample, NonSilentRegionDetection, MelFilterBank 
-    * Generic augmentation - Slice, Normalize
-    * Reading from file lists in file reader
-    * Downmixing audio channels during decoding
-    * TensorTensorAdd and TensorScalarMultiply operations
-    * Uniform and Normal distribution nodes
-* Image to tensor updates
-* ROCm install - use case graphics removed
-
-#### Known issues
- 
-* Dependencies are not installed with the rocAL package installer. Dependencies must be installed with the prerequisite setup script provided. See the [rocAL README on GitHub](https://github.com/ROCm/rocAL/blob/docs/6.2.1/README.md#prerequisites-setup-script) for details.
-
-### **rocBLAS** (4.2.1)
-
-#### Removals
-
-* Removed Device_Memory_Allocation.pdf link in documentation.
+* AMD Clang is now the default CXX and C compiler.
+* The dependency on rocDecode has been removed and automatic rocDecode installation is now disabled in the setup script.
 
 #### Resolved issues
 
-* Fixed error/warning message during `rocblas_set_stream()` call.
-
-### **rocFFT** (1.0.29)
-
-#### Optimizations
-
-* Implemented 1D kernels for factorizable sizes less than 1024.
-
-### **ROCm SMI** (7.3.0)
-
-#### Optimizations
-
-* Improved handling of UnicodeEncodeErrors with non UTF-8 locales. Non UTF-8 locales were causing crashes on UTF-8 special characters.
-
-#### Resolved issues
-
-* Fixed an issue where the Compute Partition tests segfaulted when AMDGPU was loaded with optional parameters.
+* Canny failure on Instinct MI300 has been fixed.
+* Ubuntu 24.04 CTest failures have been fixed.
 
 #### Known issues
 
-* When setting CPX as a partition mode, there is a DRM node limit of 64. This is a known limitation when multiple drivers are using the DRM nodes. The `ls /sys/class/drm` command can be used to see the number of DRM nodes, and the following steps can be used to remove unnecessary drivers:
-        
-    1. Unload AMDGPU: `sudo rmmod amdgpu`.
-    2. Remove any unnecessary drivers using `rmmod`. For example, to remove an AST driver, run `sudo rmmod ast`.
-    3. Reload AMDGPU using `modprobe`: `sudo modprobe amdgpu`.
+* CentOS, Red Hat, and SLES requires the manual installation of `OpenCV` and `FFMPEG`. 
+* Hardware decode requires that ROCm is installed with `--usecase=graphics`.
 
-### **rocPRIM** (3.2.1)
+#### Upcoming changes
 
-#### Optimizations
+* Optimized audio augmentations support for VX_RPP.
 
-* Improved performance of `block_reduce_warp_reduce` when warp size equals block size.
+### **RCCL** (2.21.5)
+
+#### Changed
+
+* Enhanced the user documentation.
+
+#### Resolved Issues
+
+* Corrected some user help strings in `install.sh`.
+
+### **ROCm Compute Profiler** (3.0.0)
+
+#### Resolved issues
+
+* Fixed a minor issue for users upgrading to ROCm 6.3 from 6.2 post-rename from `omniperf`.
+  See [ROCm Compute Profiler and ROCm Systems Profiler post-upgrade issues](#rocm-compute-profiler-and-rocm-systems-profiler-post-upgrade-issues).
+
+### **ROCm Systems Profiler** (0.1.0)
+
+#### Added
+
+* Improvements to support OMPT target offload.
+
+#### Resolved issues
+
+* Fixed an issue with generated Perfetto files. See [issue #3767](https://github.com/ROCm/ROCm/issues/3767) for more information.
+
+* Fixed an issue with merging multiple `.proto` files.
+
+* Fixed an issue causing GPU resource data to be missing from traces of Instinct MI300A systems.
+
+* Fixed a minor issue for users upgrading to ROCm 6.3 from 6.2 post-rename from `omnitrace`.
+  See [ROCm Compute Profiler and ROCm Systems Profiler post-upgrade issues](#rocm-compute-profiler-and-rocm-systems-profiler-post-upgrade-issues).
+
+### **ROCprofiler-SDK** (0.5.0)
+
+#### Added
+
+* SIMD_UTILIZATION metric.
+* New <a href="https://rocm.docs.amd.com/projects/rdc/en/docs-6.3.1/index.html">ROCm Data Center (RDC)</a> ops metrics.
 
 ## ROCm known issues
 
-ROCm known issues are tracked on [GitHub](https://github.com/ROCm/ROCm/labels/Verified%20Issue). Known issues related to
-individual components are listed in the [Detailed component changes](detailed-component-changes) section.
+ROCm known issues are noted on {fab}`github` [GitHub](https://github.com/ROCm/ROCm/labels/Verified%20Issue). For known
+issues related to individual components, review the [Detailed component changes](#detailed-component-changes).
 
-### Instinct MI300X GPU recovery failure on uncorrectable errors
+### PCI Express Qualification Tool failure on Debian 12
 
-For the AMD Instinct MI300X accelerator, GPU recovery resets triggered by uncorrectable errors (UE) might not complete
-successfully, which can result in the system being left in an undefined state. A system reboot is needed to recover from
-this state. Additionally, error logging might fail in these situations, hindering diagnostics.
+The PCI Express Qualification Tool (PEQT) module present in the ROCm Validation Suite (RVS) might fail due to the segmentation issue in Debian 12 (bookworm). This will result in failure to determine the characteristics of the PCIe interconnect between the host platform and the GPU like support for Gen 3 atomic completers, DMA transfer statistics, link speed, and link width. The standard PCIe command `lspci` can be used as an alternative to view the characteristics of the PCIe bus interconnect with the GPU. This issue is under investigation and will be addressed in a future release. See [GitHub issue #4175](https://github.com/ROCm/ROCm/issues/4175).
 
-This issue is under investigation and will be resolved in a future ROCm release.
+## ROCm resolved issues
 
-See [issue #3766](https://github.com/ROCm/ROCm/issues/3766) on GitHub.
+The following are previously known issues resolved in this release. For resolved issues related to
+individual components, review the [Detailed component changes](#detailed-component-changes).
+
+### Instinct MI300 series: backward weights convolution performance issue
+
+Fixed a performance issue affecting certain tensor shapes during backward weights convolution when using FP16 or FP32 data types on Instinct MI300 series accelerators. See [GitHub issue #4080](https://github.com/ROCm/ROCm/issues/4080).
+
+### ROCm Compute Profiler and ROCm Systems Profiler post-upgrade issues
+
+Packaging metadata for ROCm Compute Profiler (`rocprofiler-compute`) and ROCm Systems Profiler
+(`rocprofiler-systems`) has been updated to handle the renaming from Omniperf and Omnitrace,
+respectively. This fixes minor issues when upgrading from ROCm 6.2 to 6.3. For more information, see the GitHub issues
+[#4082](https://github.com/ROCm/ROCm/issues/4082) and
+[#4083](https://github.com/ROCm/ROCm/issues/4083).
+
+### Stale file due to OpenCL ICD loader deprecation
+
+When upgrading from ROCm 6.2.x to ROCm 6.3.0, the issue of removal of the `rocm-icd-loader` package
+leaving a stale file in the old `rocm-6.2.x` directory has been resolved. The stale files left during
+the upgrade from ROCm 6.2.x to ROCm 6.3.0 will be removed when upgrading to ROCm 6.3.1. For more
+information, see [GitHub issue #4084](https://github.com/ROCm/ROCm/issues/4084).
 
 ## ROCm upcoming changes
 
 The following changes to the ROCm software stack are anticipated for future releases.
 
-### rocm-llvm-alt
+### AMDGPU wavefront size compiler macro deprecation
 
-The `rocm-llvm-alt` package will be removed in an upcoming release. Users relying on the functionality provided by the closed-source compiler should transition to the open-source compiler. Once the `rocm-llvm-alt` package is removed, any compilation requesting functionality provided by the closed-source compiler will result in a Clang warning: "*[AMD] proprietary optimization compiler has been removed*".
+The `__AMDGCN_WAVEFRONT_SIZE__` macro will be deprecated in an upcoming
+release. It is recommended to remove any use of this macro. For more information, see [AMDGPU
+support](https://rocm.docs.amd.com/projects/llvm-project/en/docs-6.3.1/LLVM/clang/html/AMDGPUSupport.html).
 
-### rccl-rdma-sharp-plugins
+### HIPCC Perl scripts deprecation
+
+The HIPCC Perl scripts (`hipcc.pl` and `hipconfig.pl`) will be removed in an upcoming release.
 
-The RCCL plugin package, `rccl-rdma-sharp-plugins`, will be removed in an upcoming ROCm release. 

default.xml

@@ -1,17 +1,14 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <manifest>
     <remote name="rocm-org" fetch="https://github.com/ROCm/" />
-    <default revision="refs/tags/rocm-6.2.1"
+    <default revision="refs/tags/rocm-6.3.1"
      remote="rocm-org"
      sync-c="true"
      sync-j="4" />
 <!--list of projects for ROCm-->
     <project name="ROCK-Kernel-Driver" />
     <project name="ROCR-Runtime" />
-    <project name="ROCT-Thunk-Interface" />
     <project name="amdsmi" />
-    <project name="omniperf" />
-    <project name="omnitrace" />
     <project name="rdc" />
     <project name="rocm_bandwidth_test" />
     <project name="rocm_smi_lib" />
@@ -21,6 +18,8 @@
     <project name="rocprofiler" />
     <project name="rocprofiler-register" />
     <project name="rocprofiler-sdk" />
+    <project name="rocprofiler-compute" />
+    <project name="rocprofiler-systems" />
     <project name="roctracer" />
 <!--HIP Projects-->
     <project name="HIP" />
@@ -42,6 +41,7 @@
     <project groups="mathlibs" name="ROCmValidationSuite" />
     <project groups="mathlibs" name="Tensile" />
     <project groups="mathlibs" name="composable_kernel" />
+    <project groups="mathlibs" name="hipBLAS-common" />
     <project groups="mathlibs" name="hipBLAS" />
     <project groups="mathlibs" name="hipBLASLt" />
     <project groups="mathlibs" name="hipCUB" />
@@ -57,6 +57,7 @@
     <project groups="mathlibs" name="rocALUTION" />
     <project groups="mathlibs" name="rocBLAS" />
     <project groups="mathlibs" name="rocDecode" />
+    <project groups="mathlibs" name="rocJPEG" />
     <project groups="mathlibs" name="rocPyDecode" />
     <project groups="mathlibs" name="rocFFT" />
     <project groups="mathlibs" name="rocPRIM" />
@@ -67,6 +68,7 @@
     <project groups="mathlibs" name="rocWMMA" />
     <project groups="mathlibs" name="rocm-cmake" />
     <project groups="mathlibs" name="rpp" />
+    <project groups="mathlibs" name="TransferBench" />
 <!-- Projects for OpenMP-Extras -->
     <project name="aomp" path="openmp-extras/aomp" />
     <project name="aomp-extras" path="openmp-extras/aomp-extras" />

docs/about/license.md

@@ -25,15 +25,15 @@ additional licenses. Please review individual repositories for more information.
 <!-- spellcheck-disable -->
 | Component | License |
 |:---------------------|:-------------------------|
-| [AMD Common Language Runtime (CLR)](https://github.com/ROCm/clr) | [MIT](https://github.com/ROCm/clr/blob/develop/LICENCE) |
-| [AMD SMI](https://github.com/ROCm/amdsmi) | [MIT](https://github.com/ROCm/amdsmi/blob/develop/LICENSE) |
+| [AMD Compute Language Runtime (CLR)](https://github.com/ROCm/clr) | [MIT](https://github.com/ROCm/clr/blob/amd-staging/LICENCE) |
+| [AMD SMI](https://github.com/ROCm/amdsmi) | [MIT](https://github.com/ROCm/amdsmi/blob/amd-staging/LICENSE) |
 | [aomp](https://github.com/ROCm/aomp/) | [Apache 2.0](https://github.com/ROCm/aomp/blob/aomp-dev/LICENSE) |
 | [aomp-extras](https://github.com/ROCm/aomp-extras/) | [MIT](https://github.com/ROCm/aomp-extras/blob/aomp-dev/LICENSE) |
 | [Code Object Manager (Comgr)](https://github.com/ROCm/llvm-project/tree/amd-staging/amd/comgr) | [The University of Illinois/NCSA](https://github.com/ROCm/llvm-project/blob/amd-staging/amd/comgr/LICENSE.txt) |
 | [Composable Kernel](https://github.com/ROCm/composable_kernel) | [MIT](https://github.com/ROCm/composable_kernel/blob/develop/LICENSE) |
 | [half](https://github.com/ROCm/half/) | [MIT](https://github.com/ROCm/half/blob/rocm/LICENSE.txt) |
-| [HIP](https://github.com/ROCm/HIP/) | [MIT](https://github.com/ROCm/HIP/blob/develop/LICENSE.txt) |
-| [hipamd](https://github.com/ROCm/clr/tree/develop/hipamd) | [MIT](https://github.com/ROCm/clr/blob/develop/hipamd/LICENSE.txt) |
+| [HIP](https://github.com/ROCm/HIP/) | [MIT](https://github.com/ROCm/HIP/blob/amd-staging/LICENSE.txt) |
+| [hipamd](https://github.com/ROCm/clr/tree/amd-staging/hipamd) | [MIT](https://github.com/ROCm/clr/blob/amd-staging/hipamd/LICENSE.txt) |
 | [hipBLAS](https://github.com/ROCm/hipBLAS/) | [MIT](https://github.com/ROCm/hipBLAS/blob/develop/LICENSE.md) |
 | [hipBLASLt](https://github.com/ROCm/hipBLASLt/) | [MIT](https://github.com/ROCm/hipBLASLt/blob/develop/LICENSE.md) |
 | [HIPCC](https://github.com/ROCm/llvm-project/tree/amd-staging/amd/hipcc) | [MIT](https://github.com/ROCm/llvm-project/blob/amd-staging/amd/hipcc/LICENSE.txt) |
@@ -52,47 +52,46 @@ additional licenses. Please review individual repositories for more information.
 | [MIGraphX](https://github.com/ROCm/AMDMIGraphX/) | [MIT](https://github.com/ROCm/AMDMIGraphX/blob/develop/LICENSE) |
 | [MIOpen](https://github.com/ROCm/MIOpen/) | [MIT](https://github.com/ROCm/MIOpen/blob/develop/LICENSE.txt) |
 | [MIVisionX](https://github.com/ROCm/MIVisionX/) | [MIT](https://github.com/ROCm/MIVisionX/blob/develop/LICENSE.txt) |
-| [Omniperf](https://github.com/ROCm/omniperf) | [MIT](https://github.com/ROCm/omniperf/blob/main/LICENSE) |
-| [Omnitrace](https://github.com/ROCm/omnitrace) | [MIT](https://github.com/ROCm/omnitrace/blob/main/LICENSE) |
 | [rocAL](https://github.com/ROCm/rocAL) | [MIT](https://github.com/ROCm/rocAL/blob/develop/LICENSE.txt) |
 | [rocALUTION](https://github.com/ROCm/rocALUTION/) | [MIT](https://github.com/ROCm/rocALUTION/blob/develop/LICENSE.md) |
 | [rocBLAS](https://github.com/ROCm/rocBLAS/) | [MIT](https://github.com/ROCm/rocBLAS/blob/develop/LICENSE.md) |
 | [ROCdbgapi](https://github.com/ROCm/ROCdbgapi/) | [MIT](https://github.com/ROCm/ROCdbgapi/blob/amd-staging/LICENSE.txt) |
 | [rocDecode](https://github.com/ROCm/rocDecode) | [MIT](https://github.com/ROCm/rocDecode/blob/develop/LICENSE) |
 | [rocFFT](https://github.com/ROCm/rocFFT/) | [MIT](https://github.com/ROCm/rocFFT/blob/develop/LICENSE.md) |
-| [ROCgdb](https://github.com/ROCm/ROCgdb/) | [GNU General Public License v2.0](https://github.com/ROCm/ROCgdb/blob/amd-master/COPYING) |
+| [ROCgdb](https://github.com/ROCm/ROCgdb/) | [GNU General Public License v3.0](https://github.com/ROCm/ROCgdb/blob/amd-staging/COPYING3) |
+| [rocJPEG](https://github.com/ROCm/rocJPEG/) | [MIT](https://github.com/ROCm/rocJPEG/blob/develop/LICENSE) |
 | [ROCK-Kernel-Driver](https://github.com/ROCm/ROCK-Kernel-Driver/) | [GPL 2.0 WITH Linux-syscall-note](https://github.com/ROCm/ROCK-Kernel-Driver/blob/master/COPYING) |
 | [rocminfo](https://github.com/ROCm/rocminfo/) | [The University of Illinois/NCSA](https://github.com/ROCm/rocminfo/blob/amd-staging/License.txt) |
-| [ROCm Bandwidth Test](https://github.com/ROCm/rocm_bandwidth_test/) | [The University of Illinois/NCSA](https://github.com/ROCm/rocm_bandwidth_test/blob/master/LICENSE.txt) |
+| [ROCm Bandwidth Test](https://github.com/ROCm/rocm_bandwidth_test/) | [MIT](https://github.com/ROCm/rocm_bandwidth_test/blob/master/LICENSE.txt) |
 | [ROCm CMake](https://github.com/ROCm/rocm-cmake/) | [MIT](https://github.com/ROCm/rocm-cmake/blob/develop/LICENSE) |
 | [ROCm Communication Collectives Library (RCCL)](https://github.com/ROCm/rccl/) | [Custom](https://github.com/ROCm/rccl/blob/develop/LICENSE.txt) |
 | [ROCm-Core](https://github.com/ROCm/rocm-core) | [MIT](https://github.com/ROCm/rocm-core/blob/master/copyright) |
-| [ROCm Data Center (RDC)](https://github.com/ROCm/rdc/) | [MIT](https://github.com/ROCm/rdc/blob/develop/LICENSE) |
+| [ROCm Compute Profiler](https://github.com/ROCm/rocprofiler-compute) | [MIT](https://github.com/ROCm/rocprofiler-compute/blob/amd-staging/LICENSE) |
+| [ROCm Data Center (RDC)](https://github.com/ROCm/rdc/) | [MIT](https://github.com/ROCm/rdc/blob/amd-staging/LICENSE) |
 | [ROCm-Device-Libs](https://github.com/ROCm/llvm-project/tree/amd-staging/amd/device-libs) | [The University of Illinois/NCSA](https://github.com/ROCm/llvm-project/blob/amd-staging/amd/device-libs/LICENSE.TXT) |
-| [ROCm-OpenCL-Runtime](https://github.com/ROCm/clr/tree/develop/opencl) | [MIT](https://github.com/ROCm/clr/blob/develop/opencl/LICENSE.txt) |
+| [ROCm-OpenCL-Runtime](https://github.com/ROCm/clr/tree/amd-staging/opencl) | [MIT](https://github.com/ROCm/clr/blob/amd-staging/opencl/LICENSE.txt) |
 | [ROCm Performance Primitives (RPP)](https://github.com/ROCm/rpp) | [MIT](https://github.com/ROCm/rpp/blob/develop/LICENSE) |
-| [ROCm SMI Lib](https://github.com/ROCm/rocm_smi_lib/) | [MIT](https://github.com/ROCm/rocm_smi_lib/blob/develop/License.txt) |
+| [ROCm SMI Lib](https://github.com/ROCm/rocm_smi_lib/) | [MIT](https://github.com/ROCm/rocm_smi_lib/blob/amd-staging/License.txt) |
+| [ROCm Systems Profiler](https://github.com/ROCm/rocprofiler-systems) | [MIT](https://github.com/ROCm/rocprofiler-systems/blob/amd-staging/LICENSE) |
 | [ROCm Validation Suite](https://github.com/ROCm/ROCmValidationSuite/) | [MIT](https://github.com/ROCm/ROCmValidationSuite/blob/master/LICENSE) |
 | [rocPRIM](https://github.com/ROCm/rocPRIM/) | [MIT](https://github.com/ROCm/rocPRIM/blob/develop/LICENSE.txt) |
-| [ROCProfiler](https://github.com/ROCm/rocprofiler/) | [MIT](https://github.com/ROCm/rocprofiler/blob/amd-master/LICENSE) |
+| [ROCProfiler](https://github.com/ROCm/rocprofiler/) | [MIT](https://github.com/ROCm/rocprofiler/blob/amd-staging/LICENSE) |
 | [ROCprofiler-SDK](https://github.com/ROCm/rocprofiler-sdk) | [MIT](https://github.com/ROCm/rocprofiler-sdk/blob/amd-mainline/LICENSE) |
 | [rocPyDecode](https://github.com/ROCm/rocPyDecode) | [MIT](https://github.com/ROCm/rocPyDecode/blob/develop/LICENSE) |
 | [rocRAND](https://github.com/ROCm/rocRAND/) | [MIT](https://github.com/ROCm/rocRAND/blob/develop/LICENSE.txt) |
 | [ROCr Debug Agent](https://github.com/ROCm/rocr_debug_agent/) | [The University of Illinois/NCSA](https://github.com/ROCm/rocr_debug_agent/blob/amd-staging/LICENSE.txt) |
-| [ROCR-Runtime](https://github.com/ROCm/ROCR-Runtime/) | [The University of Illinois/NCSA](https://github.com/ROCm/ROCR-Runtime/blob/master/LICENSE.txt) |
+| [ROCR-Runtime](https://github.com/ROCm/ROCR-Runtime/) | [The University of Illinois/NCSA](https://github.com/ROCm/ROCR-Runtime/blob/amd-staging/LICENSE.txt) |
 | [rocSOLVER](https://github.com/ROCm/rocSOLVER/) | [BSD-2-Clause](https://github.com/ROCm/rocSOLVER/blob/develop/LICENSE.md) |
 | [rocSPARSE](https://github.com/ROCm/rocSPARSE/) | [MIT](https://github.com/ROCm/rocSPARSE/blob/develop/LICENSE.md) |
 | [rocThrust](https://github.com/ROCm/rocThrust/) | [Apache 2.0](https://github.com/ROCm/rocThrust/blob/develop/LICENSE) |
 | [ROCTracer](https://github.com/ROCm/roctracer/) | [MIT](https://github.com/ROCm/roctracer/blob/amd-master/LICENSE) |
-| [ROCT-Thunk-Interface](https://github.com/ROCm/ROCT-Thunk-Interface/) | [MIT](https://github.com/ROCm/ROCT-Thunk-Interface/blob/master/LICENSE.md) |
 | [rocWMMA](https://github.com/ROCm/rocWMMA/) | [MIT](https://github.com/ROCm/rocWMMA/blob/develop/LICENSE.md) |
 | [Tensile](https://github.com/ROCm/Tensile/) | [MIT](https://github.com/ROCm/Tensile/blob/develop/LICENSE.md) |
 | [TransferBench](https://github.com/ROCm/TransferBench) | [MIT](https://github.com/ROCm/TransferBench/blob/develop/LICENSE.md) |
 
 Open sourced ROCm components are released via public GitHub
 repositories, packages on [https://repo.radeon.com](https://repo.radeon.com) and other distribution channels.
-Proprietary products are only available on [https://repo.radeon.com](https://repo.radeon.com). Currently, only
-one component of ROCm, `rocm-llvm-alt` is governed by a proprietary license.
+Proprietary products are only available on [https://repo.radeon.com](https://repo.radeon.com).
 Proprietary components are organized in a proprietary subdirectory in the package
 repositories to distinguish from open sourced packages.
 
@@ -100,7 +99,7 @@ repositories to distinguish from open sourced packages.
 The following additional terms and conditions apply to your use of ROCm technical documentation.
 ```
 
-©2023 - 2024 Advanced Micro Devices, Inc. All rights reserved.
+©2023 - 2025 Advanced Micro Devices, Inc. All rights reserved.
 
 The information presented in this document is for informational purposes only
 and may contain technical inaccuracies, omissions, and typographical errors. The

docs/compatibility/compatibility-matrix-historical-6.0.csv

@@ -1,117 +1,129 @@
-ROCm Version,6.2.1,6.2.0, 6.1.5, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
-      :ref:`Operating Systems & kernels <OS-kernel-versions>`,"Ubuntu 24.04.1, 24.04",Ubuntu 24.04,,,,,,
-      ,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4, 22.04.3","Ubuntu 22.04.5, 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3, 22.04.2","Ubuntu 22.04.4, 22.04.3, 22.04.2"
-      ,,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
-      ,"RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.4 [#red-hat94-past-60]_, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
-      ,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
-      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
-      ,,,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
-      ,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,Oracle Linux 8.9 [#oracle89-past-60]_,,,
-      ,.. _architecture-support-compatibility-matrix-past-60:,,,,,,,
-      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
-      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
-      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
-      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
-      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
-      ,.. _gpu-support-compatibility-matrix-past-60:,,,,,,,
-      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
-      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
-      ,gfx942 [#mi300_621-past-60]_,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
-      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
-      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
-      ,,,,,,,,
-      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix-past-60:,,,,,,,
-      :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1","2.14.0, 2.13.1, 2.12.1"
-      :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
-      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
-      ,,,,,,,,
-      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix-past-60:,,,,,,,
-      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
-      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
-      ,,,,,,,,
-      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix-past-60:,,,,,,,
-      Thrust,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-      CUB,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-      ,,,,,,,,
-      KFD & USER SPACE [#kfd_support-past-60]_,.. _kfd-userspace-support-compatibility-matrix-past-60:,,,,,,,
-      Tested user space versions,"6.1.x, 6.0.x","6.1.x, 6.0.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x","6.2.x, 6.0.x, 5.7.x, 5.6.x","6.2.x, 6.0.x, 5.7.x, 5.6.x"
-      ,,,,,,,,
-      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix-past-60:,,,,,,,
-      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
-      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.10.0,2.9.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
-      :doc:`MIOpen <miopen:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`MIVisionX <mivisionx:index>`,3.0.0,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
-      :doc:`rocAL <rocal:index>`,2.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
-      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,0.1.0,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`RPP <rpp:index>`,1.8.0,1.8.0,1.5.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
-      ,,,,,,,,
-      COMMUNICATION,.. _commlibs-support-compatibility-matrix-past-60:,,,,,,,
-      :doc:`RCCL <rccl:index>`,2.20.5,2.20.5,2.18.6,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
-      ,,,,,,,,
-      MATH LIBS,.. _mathlibs-support-compatibility-matrix-past-60:,,,,,,,
-      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
-      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
-      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.8.0,0.7.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
-      :doc:`hipFFT <hipfft:index>`,1.0.15,1.0.14,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
-      :doc:`hipFORT <hipfort:index>`,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
-      :doc:`hipRAND <hiprand:index>`,2.11.0,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
-      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.2.0,2.1.1,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
-      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.1.1,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.1,0.2.0,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
-      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.2.0,3.1.1,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
-      :doc:`rocBLAS <rocblas:index>`,4.2.1,4.2.0,4.1.2,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
-      :doc:`rocFFT <rocfft:index>`,1.0.29,1.0.28,1.0.27,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
-      :doc:`rocRAND <rocrand:index>`,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
-      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.26.0,3.25.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
-      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.2.0,3.1.2,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
-      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.5.0,1.4.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
-      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.40.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
-      ,,,,,,,,
-      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix-past-60:,,,,,,,
-      :doc:`hipCUB <hipcub:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.3.0,1.2.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
-      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`rocThrust <rocthrust:index>`,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      ,,,,,,,,
-      SUPPORT LIBS,,,,,,,,
-      `hipother <https://github.com/ROCm/hipother>`_,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.1,6.2.0,6.1.5,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
-      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.4.05,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
-      ,,,,,,,,
-      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix-past-60:,,,,,,,
-      :doc:`AMD SMI <amdsmi:index>`,24.6.3,24.6.2,24.5.1,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
-      :doc:`ROCm Data Center Tool <rdc:index>`,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
-      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.3.0,7.2.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
-      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.1,rocm-6.2.0,rocm-6.1.5,rocm-6.1.2,rocm-6.1.1,rocm-6.1.0,rocm-6.0.2,rocm-6.0.0
-      ,,,,,,,,
-      PERFORMANCE TOOLS,,,,,,,,
-      :doc:`Omniperf <omniperf:index>`,2.0.1,2.0.1,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`Omnitrace <omnitrace:index>`,1.11.2,1.11.2,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
-      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60201,2.0.60200,2.0.60105,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
-      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,0.4.0,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCTracer <roctracer:index>`,4.1.60201,4.1.60200,4.1.60105,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
-      ,,,,,,,,
-      DEVELOPMENT TOOLS,,,,,,,,
-      :doc:`HIPIFY <hipify:index>`,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.13.0,0.12.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
-      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
-      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.2.0,14.1.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
-      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.3.0,0.3.0,0.3.0,0.3.0,N/A,N/A
-      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
-      ,,,,,,,,
-      COMPILERS,.. _compilers-support-compatibility-matrix-past-60:,,,,,,,
-      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
-      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      :doc:`llvm-project <llvm-project:index>`,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      ,,,,,,,,
-      RUNTIMES,.. _runtime-support-compatibility-matrix-past-60:,,,,,,,
-      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      :doc:`HIP <hip:index>`,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
-      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.14.0,1.13.0,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0
+ROCm Version,6.3.1,6.3.0,6.2.4,6.2.2,6.2.1,6.2.0, 6.1.5, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
+      :ref:`Operating systems & kernels <OS-kernel-versions>`,Ubuntu 24.04.2,Ubuntu 24.04.2,"Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04",Ubuntu 24.04,,,,,,
+      ,Ubuntu 22.04.5,Ubuntu 22.04.5,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3, 22.04.2","Ubuntu 22.04.4, 22.04.3, 22.04.2"
+      ,,,,,,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
+      ,"RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
+      ,RHEL 8.10,RHEL 8.10,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
+      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
+      ,,,,,,,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
+      ,Oracle Linux 8.10 [#mic300x-past-60]_,Oracle Linux 8.10 [#mic300x-past-60]_,Oracle Linux 8.9 [#mic300x-past-60]_,Oracle Linux 8.9 [#mic300x-past-60]_,Oracle Linux 8.9 [#mic300x-past-60]_,Oracle Linux 8.9 [#mic300x-past-60]_,Oracle Linux 8.9 [#mic300x-past-60]_,Oracle Linux 8.9 [#mic300x-past-60]_,Oracle Linux 8.9 [#mic300x-past-60]_,,,
+,Debian 12 [#single-node-past-60]_,,,,,,,,,,,
+,Azure Linux 3.0 [#mic300x-past-60]_,,,,,,,,,,,
+      ,.. _architecture-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
+      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
+      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
+      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
+      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
+      ,.. _gpu-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
+      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
+      ,gfx942,gfx942,gfx942 [#mi300_624-past-60]_,gfx942 [#mi300_622-past-60]_,gfx942 [#mi300_621-past-60]_,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
+      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
+      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
+      ,,,,,,,,,,,,
+      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      :doc:`PyTorch <../compatibility/ml-compatibility/pytorch-compatibility>`,"2.4, 2.3, 2.2, 2.1, 2.0, 1.13","2.4, 2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
+      :doc:`TensorFlow <../compatibility/ml-compatibility/tensorflow-compatibility>`,"2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1","2.14.0, 2.13.1, 2.12.1"
+      :doc:`JAX <../compatibility/ml-compatibility/jax-compatibility>`,0.4.31,0.4.31,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
+      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
+      ,,,,,,,,,,,,
+      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
+      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
+      ,,,,,,,,,,,,
+      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      Thrust,2.3.2,2.3.2,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+      CUB,2.3.2,2.3.2,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+,,,,,,,,,,,,
+      ,,,,,,,,,,,,
+      KMD & USER SPACE [#kfd_support-past-60]_,.. _kfd-userspace-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      Tested user space versions,"6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.2.x, 6.1.x, 6.0.x, 5.7.x, 5.6.x","6.2.x, 6.1.x, 6.0.x, 5.7.x, 5.6.x"
+      ,,,,,,,,,,,,
+      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
+      :doc:`MIGraphX <amdmigraphx:index>`,2.11.0,2.11.0,2.10.0,2.10.0,2.10.0,2.10.0,2.9.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
+      :doc:`MIOpen <miopen:index>`,3.3.0,3.3.0,3.2.0,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`MIVisionX <mivisionx:index>`,3.1.0,3.1.0,3.0.0,3.0.0,3.0.0,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
+      :doc:`rocAL <rocal:index>`,2.1.0,2.1.0,2.0.0,2.0.0,2.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`rocDecode <rocdecode:index>`,0.8.0,0.8.0,0.6.0,0.6.0,0.6.0,0.6.0,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
+      :doc:`rocJPEG <rocjpeg:index>`,0.6.0,0.6.0,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A
+      :doc:`rocPyDecode <rocpydecode:index>`,0.2.0,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0,N/A,N/A,N/A,N/A,N/A,N/A
+      :doc:`RPP <rpp:index>`,1.9.1,1.9.1,1.8.0,1.8.0,1.8.0,1.8.0,1.5.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
+      ,,,,,,,,,,,,
+      COMMUNICATION,.. _commlibs-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      :doc:`RCCL <rccl:index>`,2.21.5,2.21.5,2.20.5,2.20.5,2.20.5,2.20.5,2.18.6,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
+      ,,,,,,,,,,,,
+      MATH LIBS,.. _mathlibs-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
+      :doc:`hipBLAS <hipblas:index>`,2.3.0,2.3.0,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
+      :doc:`hipBLASLt <hipblaslt:index>`,0.10.0,0.10.0,0.8.0,0.8.0,0.8.0,0.8.0,0.7.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
+      :doc:`hipFFT <hipfft:index>`,1.0.17,1.0.17,1.0.16,1.0.15,1.0.15,1.0.14,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
+      :doc:`hipfort <hipfort:index>`,0.5.0,0.5.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
+      :doc:`hipRAND <hiprand:index>`,2.11.1,2.11.0,2.11.1,2.11.0,2.11.0,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
+,,,,,,,,,,,,
+      :doc:`hipSOLVER <hipsolver:index>`,2.3.0,2.3.0,2.2.0,2.2.0,2.2.0,2.2.0,2.1.1,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
+      :doc:`hipSPARSE <hipsparse:index>`,3.1.2,3.1.2,3.1.1,3.1.1,3.1.1,3.1.1,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.2,0.2.2,0.2.1,0.2.1,0.2.1,0.2.1,0.2.0,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
+      :doc:`rocALUTION <rocalution:index>`,3.2.1,3.2.1,3.2.1,3.2.0,3.2.0,3.2.0,3.1.1,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
+      :doc:`rocBLAS <rocblas:index>`,4.3.0,4.3.0,4.2.4,4.2.1,4.2.1,4.2.0,4.1.2,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
+      :doc:`rocFFT <rocfft:index>`,1.0.31,1.0.31,1.0.30,1.0.29,1.0.29,1.0.28,1.0.27,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
+      :doc:`rocRAND <rocrand:index>`,3.2.0,3.2.0,3.1.1,3.1.0,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
+      :doc:`rocSOLVER <rocsolver:index>`,3.27.0,3.27.0,3.26.2,3.26.0,3.26.0,3.26.0,3.25.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
+      :doc:`rocSPARSE <rocsparse:index>`,3.3.0,3.3.0,3.2.1,3.2.0,3.2.0,3.2.0,3.1.2,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
+      :doc:`rocWMMA <rocwmma:index>`,1.6.0,1.6.0,1.5.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
+      :doc:`Tensile <tensile:src/index>`,4.42.0,4.42.0,4.41.0,4.41.0,4.41.0,4.41.0,4.40.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
+      ,,,,,,,,,,,,
+      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      :doc:`hipCUB <hipcub:index>`,3.3.0,3.3.0,3.2.1,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`hipTensor <hiptensor:index>`,1.4.0,1.4.0,1.3.0,1.3.0,1.3.0,1.3.0,1.2.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
+      :doc:`rocPRIM <rocprim:index>`,3.3.0,3.3.0,3.2.2,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`rocThrust <rocthrust:index>`,3.3.0,3.3.0,3.1.1,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      ,,,,,,,,,,,,
+      SUPPORT LIBS,,,,,,,,,,,,
+      `hipother <https://github.com/ROCm/hipother>`_,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+,,,,,,,,,,,,
+      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.3.1,6.3.0,6.2.4,6.2.2,6.2.1,6.2.0,6.1.5,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
+      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,20240607.5.7,20240607.5.7,20240607.4.05,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
+      ,,,,,,,,,,,,
+      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      :doc:`AMD SMI <amdsmi:index>`,24.7.1,24.7.1,24.6.3,24.6.3,24.6.3,24.6.2,24.5.1,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
+      :doc:`ROCm Data Center Tool <rdc:index>`,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
+      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.4.0,7.4.0,7.3.0,7.3.0,7.3.0,7.3.0,7.2.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
+      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,1.1.0,1.1.0,1.0.60204,1.0.60202,1.0.60201,1.0.60200,1.0.60105,1.0.60102,1.0.60101,1.0.60100,1.0.60002,1.0.60000
+      ,,,,,,,,,,,,
+      PERFORMANCE TOOLS,,,,,,,,,,,,
+      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
+      :doc:`ROCm Compute Profiler <rocprofiler-compute:index>`,3.0.0,3.0.0,2.0.1,2.0.1,2.0.1,2.0.1,N/A,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCm Systems Profiler <rocprofiler-systems:index>`,0.1.0,0.1.0,1.11.2,1.11.2,1.11.2,1.11.2,N/A,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60301,2.0.60300,2.0.60204,2.0.60202,2.0.60201,2.0.60200,2.0.60105,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
+,,,,,,,,,,,,
+      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.5.0,0.5.0,0.4.0,0.4.0,0.4.0,0.4.0,N/A,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCTracer <roctracer:index>`,4.1.60301,4.1.60300,4.1.60204,4.1.60202,4.1.60201,4.1.60200,4.1.60105,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
+,,,,,,,,,,,,
+      ,,,,,,,,,,,,
+      DEVELOPMENT TOOLS,,,,,,,,,,,,
+      :doc:`HIPIFY <hipify:index>`,18.0.0.24491,18.0.0.24455,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+,,,,,,,,,,,,
+      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.14.0,0.14.0,0.13.0,0.13.0,0.13.0,0.13.0,0.12.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
+      :doc:`ROCdbgapi <rocdbgapi:index>`,0.77.0,0.77.0,0.76.0,0.76.0,0.76.0,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
+      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,15.2.0,15.2.0,14.2.0,14.2.0,14.2.0,14.2.0,14.1.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
+      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.3.0,0.3.0,0.3.0,0.3.0,N/A,N/A
+      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
+      ,,,,,,,,,,,,
+      COMPILERS,.. _compilers-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,N/A,N/A,N/A,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
+      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24491,18.0.0.24455,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`llvm-project <llvm-project:index>`,18.0.0.24455,18.0.0.24491,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24455,18.0.0.24491,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+,,,,,,,,,,,,
+      ,,,,,,,,,,,,
+      RUNTIMES,.. _runtime-support-compatibility-matrix-past-60:,,,,,,,,,,,
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+,,,,,,,,,,,,
+      :doc:`HIP <hip:index>`,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+,,,,,,,,,,,,
+      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
+      :doc:`ROCr Runtime <rocr-runtime:index>`,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.13.0,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0

docs/compatibility/compatibility-matrix.rst

@@ -10,24 +10,30 @@ Use this matrix to view the ROCm compatibility and system requirements across su
 
 You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-rocm-compatibility-matrix>`.
 
+Accelerators and GPUs listed in the following table support compute workloads (no display
+information or graphics). If you’re using ROCm with AMD Radeon or Radeon Pro GPUs for graphics
+workloads, see the `Use ROCm on Radeon GPU documentation
+<https://rocm.docs.amd.com/projects/radeon/en/latest/docs/compatibility.html>`_ to verify
+compatibility and system requirements.
+
 .. |br| raw:: html
 
    <br/>
 
 .. container:: format-big-table
 
-  .. csv-table:: 
-      :header: "ROCm Version", "6.2.1", "6.2.0", "6.1.0"
+  .. csv-table::
+      :header: "ROCm Version", "6.3.1", "6.3.0", "6.2.0"
       :stub-columns: 1
 
-      :ref:`Operating Systems & kernels <OS-kernel-versions>`,"Ubuntu 24.04.1, 24.04",Ubuntu 24.04,
-      ,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.4, 22.04.3"
-      ,,,"Ubuntu 20.04.6, 20.04.5"
-      ,"RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4 [#red-hat94]_, 9.3, 9.2"
-      ,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8"
-      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4"
-      ,,,CentOS 7.9
-      ,Oracle Linux 8.9 [#oracle89]_,Oracle Linux 8.9 [#oracle89]_,
+      :ref:`Operating systems & kernels <OS-kernel-versions>`,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04
+      ,Ubuntu 22.04.5,Ubuntu 22.04.5,"Ubuntu 22.04.5, 22.04.4"
+      ,"RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.4, 9.3"
+      ,RHEL 8.10,RHEL 8.10,"RHEL 8.10, 8.9"
+      ,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5"
+      ,Oracle Linux 8.10 [#mi300x]_,Oracle Linux 8.10 [#mi300x]_,Oracle Linux 8.9 [#mi300x]_
+      ,Debian 12 [#single-node]_,,
+      ,Azure Linux 3.0 [#mi300x]_,,
       ,.. _architecture-support-compatibility-matrix:,,
       :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3
       ,CDNA2,CDNA2,CDNA2
@@ -37,154 +43,149 @@ You can also refer to the :ref:`past versions of ROCm compatibility matrix<past-
       ,.. _gpu-support-compatibility-matrix:,,
       :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100
       ,gfx1030,gfx1030,gfx1030
-      ,gfx942 [#mi300_621]_,gfx942 [#mi300_620]_, gfx942 [#mi300_610]_
+      ,gfx942,gfx942,gfx942 [#mi300_620]_
       ,gfx90a,gfx90a,gfx90a
       ,gfx908,gfx908,gfx908
       ,,,
       FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix:,,
-      :doc:`PyTorch <rocm-install-on-linux:install/3rd-party/pytorch-install>`,"2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <rocm-install-on-linux:install/3rd-party/tensorflow-install>`,"2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15, 2.14, 2.13"
-      :doc:`JAX <rocm-install-on-linux:install/3rd-party/jax-install>`,0.4.26,0.4.26,0.4.26
+      :doc:`PyTorch <../compatibility/ml-compatibility/pytorch-compatibility>`,"2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13"
+      :doc:`TensorFlow <../compatibility/ml-compatibility/tensorflow-compatibility>`,"2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.16.1, 2.15.1, 2.14.1"
+      :doc:`JAX <../compatibility/ml-compatibility/jax-compatibility>`,0.4.31,0.4.31,0.4.26
       `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.17.3,1.17.3,1.17.3
       ,,,
       THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix:,,
       `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0
-      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.14.1
+      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.15.0
       ,,,
       THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix:,,
-      Thrust,2.2.0,2.2.0,2.1.0
-      CUB,2.2.0,2.2.0,2.1.0
+      Thrust,2.3.2,2.3.2,2.2.0
+      CUB,2.3.2,2.3.2,2.2.0
       ,,,
-      KFD & USER SPACE [#kfd_support]_,.. _kfd-userspace-support-compatibility-matrix:,,
-      Tested user space versions,"6.1.x, 6.0.x","6.1.x, 6.0.x","6.2.x, 6.0.x, 5.7.x"
+      KMD & USER SPACE [#kfd_support]_,.. _kfd-userspace-support-compatibility-matrix:,,
+      Tested user space versions,"6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x, 6.0.x"
       ,,,
       ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix:,,
       :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0
-      :doc:`MIGraphX <amdmigraphx:index>`,2.10.0,2.10.0,2.9.0
-      :doc:`MIOpen <miopen:index>`,3.2.0,3.2.0,3.1.0
-      :doc:`MIVisionX <mivisionx:index>`,3.0.0,3.0.0,2.5.0
-      :doc:`rocAL <rocal:index>`,2.0.0,1.0.0,1.0.0
-      :doc:`rocDecode <rocdecode:index>`,0.6.0,0.6.0,0.5.0
-      :doc:`rocPyDecode <rocpydecode:index>`,0.1.0,0.1.0,N/A
-      :doc:`RPP <rpp:index>`,1.8.0,1.8.0,1.5.0
+      :doc:`MIGraphX <amdmigraphx:index>`,2.11.0,2.11.0,2.10.0
+      :doc:`MIOpen <miopen:index>`,3.3.0,3.3.0,3.2.0
+      :doc:`MIVisionX <mivisionx:index>`,3.1.0,3.1.0,3.0.0
+      :doc:`rocAL <rocal:index>`,2.1.0,2.1.0,1.0.0
+      :doc:`rocDecode <rocdecode:index>`,0.8.0,0.8.0,0.6.0
+      :doc:`rocJPEG <rocjpeg:index>`,0.6.0,0.6.0,N/A
+      :doc:`rocPyDecode <rocpydecode:index>`,0.2.0,0.2.0,0.1.0
+      :doc:`RPP <rpp:index>`,1.9.1,1.9.1,1.8.0
       ,,,
       COMMUNICATION,.. _commlibs-support-compatibility-matrix:,,
-      :doc:`RCCL <rccl:index>`,2.20.5,2.20.5,2.18.6
+      :doc:`RCCL <rccl:index>`,2.21.5,2.21.5,2.20.5
       ,,,
       MATH LIBS,.. _mathlibs-support-compatibility-matrix:,,
       `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0
-      :doc:`hipBLAS <hipblas:index>`,2.2.0,2.2.0,2.1.0
-      :doc:`hipBLASLt <hipblaslt:index>`,0.8.0,0.8.0,0.7.0
-      :doc:`hipFFT <hipfft:index>`,1.0.15,1.0.14,1.0.14
-      :doc:`hipFORT <hipfort:index>`,0.4.0,0.4.0,0.4.0
-      :doc:`hipRAND <hiprand:index>`,2.11.0,2.11.0,2.10.16
-      :doc:`hipSOLVER <hipsolver:index>`,2.2.0,2.2.0,2.1.0
-      :doc:`hipSPARSE <hipsparse:index>`,3.1.1,3.1.1,3.0.1
-      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.1,0.2.1,0.1.0
-      :doc:`rocALUTION <rocalution:index>`,3.2.0,3.2.0,3.1.1
-      :doc:`rocBLAS <rocblas:index>`,4.2.1,4.2.0,4.1.0
-      :doc:`rocFFT <rocfft:index>`,1.0.29,1.0.28,1.0.26
-      :doc:`rocRAND <rocrand:index>`,3.1.0,3.1.0,3.0.1
-      :doc:`rocSOLVER <rocsolver:index>`,3.26.0,3.26.0,3.25.0
-      :doc:`rocSPARSE <rocsparse:index>`,3.2.0,3.2.0,3.1.2
-      :doc:`rocWMMA <rocwmma:index>`,1.5.0,1.5.0,1.4.0
-      `Tensile <https://github.com/ROCm/Tensile>`_,4.40.0,4.40.0,4.40.0
+      :doc:`hipBLAS <hipblas:index>`,2.3.0,2.3.0,2.2.0
+      :doc:`hipBLASLt <hipblaslt:index>`,0.10.0,0.10.0,0.8.0
+      :doc:`hipFFT <hipfft:index>`,1.0.17,1.0.17,1.0.14
+      :doc:`hipfort <hipfort:index>`,0.5.0,0.5.0,0.4.0
+      :doc:`hipRAND <hiprand:index>`,2.11.1,2.11.0,2.11.0
+      :doc:`hipSOLVER <hipsolver:index>`,2.3.0,2.3.0,2.2.0
+      :doc:`hipSPARSE <hipsparse:index>`,3.1.2,3.1.2,3.1.1
+      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.2,0.2.2,0.2.1
+      :doc:`rocALUTION <rocalution:index>`,3.2.1,3.2.1,3.2.0
+      :doc:`rocBLAS <rocblas:index>`,4.3.0,4.3.0,4.2.0
+      :doc:`rocFFT <rocfft:index>`,1.0.31,1.0.31,1.0.28
+      :doc:`rocRAND <rocrand:index>`,3.2.0,3.2.0,3.1.0
+      :doc:`rocSOLVER <rocsolver:index>`,3.27.0,3.27.0,3.26.0
+      :doc:`rocSPARSE <rocsparse:index>`,3.3.0,3.3.0,3.2.0
+      :doc:`rocWMMA <rocwmma:index>`,1.6.0,1.6.0,1.5.0
+      :doc:`Tensile <tensile:src/index>`,4.42.0,4.42.0,4.41.0
       ,,,
       PRIMITIVES,.. _primitivelibs-support-compatibility-matrix:,,
-      :doc:`hipCUB <hipcub:index>`,3.2.0,3.2.0,3.1.0
-      :doc:`hipTensor <hiptensor:index>`,1.3.0,1.3.0,1.2.0
-      :doc:`rocPRIM <rocprim:index>`,3.2.0,3.2.0,3.1.0
-      :doc:`rocThrust <rocthrust:index>`,3.1.0,3.0.1,3.0.1
+      :doc:`hipCUB <hipcub:index>`,3.3.0,3.3.0,3.2.0
+      :doc:`hipTensor <hiptensor:index>`,1.4.0,1.4.0,1.3.0
+      :doc:`rocPRIM <rocprim:index>`,3.3.0,3.3.0,3.2.0
+      :doc:`rocThrust <rocthrust:index>`,3.3.0,3.3.0,3.0.1
       ,,,
       SUPPORT LIBS,,,
-      `hipother <https://github.com/ROCm/hipother>`_,6.2.41134,6.2.41133,6.1.40091
-      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.2.1,6.2.0,6.1.0
-      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,20240607.4.05,20240607.1.4246,20240125.3.30
+      `hipother <https://github.com/ROCm/hipother>`_,6.3.42133,6.3.42131,6.2.41133
+      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.3.1,6.3.0,6.2.0
+      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,N/A [#ROCT-rocr]_,N/A [#ROCT-rocr]_,20240607.1.4246
       ,,,
       SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix:,,
-      :doc:`AMD SMI <amdsmi:index>`,24.6.3,24.6.2,24.4.1
+      :doc:`AMD SMI <amdsmi:index>`,24.7.1,24.7.1,24.6.2
       :doc:`ROCm Data Center Tool <rdc:index>`,0.3.0,0.3.0,0.3.0
       :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0
-      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.3.0,7.3.0,7.0.0
-      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,rocm-6.2.1,rocm-6.2.0,rocm-6.1.0
+      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.4.0,7.4.0,7.3.0
+      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,1.1.0,1.1.0,1.0.60200
       ,,,
       PERFORMANCE TOOLS,,,
-      :doc:`Omniperf <omniperf:index>`,2.0.1,2.0.1,N/A
-      :doc:`Omnitrace <omnitrace:index>`,1.11.2,1.11.2,N/A
       :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0
-      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60201,2.0.60200,2.0.60100
-      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.4.0,0.4.0,N/A
-      :doc:`ROCTracer <roctracer:index>`,4.1.60201,4.1.60200,4.1.60100
+      :doc:`ROCm Compute Profiler <rocprofiler-compute:index>`,3.0.0,3.0.0,2.0.1
+      :doc:`ROCm Systems Profiler <rocprofiler-systems:index>`,0.1.0,0.1.0,1.11.2
+      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60301,2.0.60300,2.0.60200
+      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.5.0,0.5.0,0.4.0
+      :doc:`ROCTracer <roctracer:index>`,4.1.60301,4.1.60300,4.1.60200
       ,,,
       DEVELOPMENT TOOLS,,,
-      :doc:`HIPIFY <hipify:index>`,18.0.0.24355,18.0.0.24232,17.0.0.24103
-      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.13.0,0.13.0,0.12.0
-      :doc:`ROCdbgapi <rocdbgapi:index>`,0.76.0,0.76.0,0.71.0
-      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,14.2.0,14.2.0,14.1.0
-      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.3.0
+      :doc:`HIPIFY <hipify:index>`,18.0.0.24491,18.0.0.24455,18.0.0.24232
+      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.14.0,0.14.0,0.13.0
+      :doc:`ROCdbgapi <rocdbgapi:index>`,0.77.0,0.77.0,0.76.0
+      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,15.2.0,15.2.0,14.2.0
+      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.4.0
       :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.3,2.0.3,2.0.3
       ,,,
       COMPILERS,.. _compilers-support-compatibility-matrix:,,
-      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,0.5.0
-      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.0.0
-      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24355,18.0.0.24232,17.0.0.24103
-      :doc:`llvm-project <llvm-project:index>`,18.0.0.24355,18.0.0.24232,17.0.0.24103
-      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24355,18.0.0.24232,17.0.0.24103
+      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,N/A
+      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.1.1
+      `Flang <https://github.com/ROCm/flang>`_,18.0.0.24491,18.0.0.24455,18.0.0.24232
+      :doc:`llvm-project <llvm-project:index>`,18.0.0.24491,18.0.0.24455,18.0.0.24232
+      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,18.0.0.24491,18.0.0.24455,18.0.0.24232
       ,,,
       RUNTIMES,.. _runtime-support-compatibility-matrix:,,
-      :doc:`AMD CLR <hip:understand/amd_clr>`,6.2.41134,6.2.41133,6.1.40091
-      :doc:`HIP <hip:index>`,6.2.41134,6.2.41133,6.1.40091
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.3.42133,6.3.42131,6.2.41133
+      :doc:`HIP <hip:index>`,6.3.42133,6.3.42131,6.2.41133
       `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0
-      :doc:`ROCR-Runtime <rocr-runtime:index>`,1.14.0,1.13.0,1.13.0
+      :doc:`ROCr Runtime <rocr-runtime:index>`,1.14.0,1.14.0,1.13.0
 
 
 .. rubric:: Footnotes
 
-.. [#red-hat94] **For ROCm 6.1** - RHEL 9.4 is supported only on AMD Instinct MI300A.
-.. [#oracle89] Oracle Linux is supported only on AMD Instinct MI300X.
-.. [#mi300_621] **For ROCm 6.2.1** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
+.. [#mi300x] Oracle Linux and Azure Linux are supported only on AMD Instinct MI300X.
+.. [#single-node] Debian 12 is supported only on AMD Instinct MI300X for single-node functionality. 
 .. [#mi300_620] **For ROCm 6.2.0** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
-.. [#mi300_610] **For ROCm 6.1.0** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4.
-.. [#kfd_support] ROCm provides forward and backward compatibility between the Kernel Fusion Driver (KFD) and its user space software for +/- 2 releases. The tested user space versions on this page were accurate as of the time of initial ROCm release. For the most up-to-date information, see the latest version of this information at `User and kernel-space support matrix <https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/user-kernel-space-compat-matrix.html>`_.
-
+.. [#kfd_support] ROCm provides forward and backward compatibility between the AMD Kernel-mode GPU Driver (KMD) and its user space software for +/- 2 releases. These are the compatibility combinations that are currently supported.
+.. [#ROCT-rocr] Starting from ROCm 6.3.0, the ROCT Thunk Interface is included as part of the ROCr runtime package.
 
 .. _OS-kernel-versions:
 
 Operating systems and kernel versions
 *************************************
 
-Use this look up table to confirm which operating system and kernel versions are supported with ROCm.
+Use this lookup table to confirm which operating system and kernel versions are supported with ROCm.
 
 .. csv-table:: 
    :header: "OS", "Version", "Kernel"
    :widths: 40, 20, 40
    :stub-columns: 1
 
-   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 24.04.1, "6.8 GA"
+   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 24.04.2, "6.8 GA, 6.11 HWE"
    , 24.04, "6.8 GA"
-   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 22.04.05, "5.15 GA, 6.8 HWE"
-   , 22.04.04, "5.15 GA, 6.5 HWE"
-   , 22.04.03, "5.15 GA, 6.2 HWE"
-   , 22.04.02, "5.15 GA, 5.19 HWE"
-   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 20.04.06, "5.15 HWE"
-   , 20.04.05, "5.15 HWE"
    ,,
-   `Red Hat Enterprise Linux (RHEL) <https://access.redhat.com/articles/3078#RHEL9>`_, 9.4, 5.14.0
+   `Ubuntu <https://ubuntu.com/about/release-cycle#ubuntu-kernel-release-cycle>`_, 22.04.5, "5.15 GA, 6.8 HWE"
+   , 22.04.4, "5.15 GA, 6.5 HWE"
+   ,,
+   `Red Hat Enterprise Linux (RHEL) <https://access.redhat.com/articles/3078#RHEL9>`_, 9.5, 5.14.0
+   ,9.4, 5.14.0
    ,9.3, 5.14.0
-   ,9.2, 5.14.0
    ,,
    `Red Hat Enterprise Linux (RHEL) <https://access.redhat.com/articles/3078#RHEL8>`_, 8.10, 4.18.0
    ,8.9, 4.18.0
-   ,8.8, 4.18.0
-   ,,
-   `CentOS <https://access.redhat.com/articles/3078#RHEL7>`_, 7.9, 3.10
    ,,
    `SUSE Linux Enterprise Server (SLES) <https://www.suse.com/support/kb/doc/?id=000019587#SLE15SP4>`_, 15 SP6, 6.4.0
    ,15 SP5, 5.14.21
-   ,15 SP4, 5.14.21
    ,,
-   `Oracle Linux <https://blogs.oracle.com/scoter/post/oracle-linux-and-unbreakable-enterprise-kernel-uek-releases>`_, 8.9, 5.15.0
- 
+   `Oracle Linux <https://blogs.oracle.com/scoter/post/oracle-linux-and-unbreakable-enterprise-kernel-uek-releases>`_, 8.10, 5.15.0
+   ,8.9, 5.15.0
+   ,,
+   `Debian <https://www.debian.org/download>`_,12, 6.1
+   `Azure Linux <https://techcommunity.microsoft.com/blog/linuxandopensourceblog/azure-linux-3-0-now-in-preview-on-azure-kubernetes-service-v1-31/4287229>`_,3.0, 6.6
 
 ..
    Footnotes and ref anchors in below historical tables should be appended with "-past-60", to differentiate from the 
@@ -207,14 +208,15 @@ Expand for full historical view of:
 
    .. csv-table::
       :file: compatibility-matrix-historical-6.0.csv
-      :widths: 20,10,10,10,10,10,10,10,10
       :header-rows: 1
       :stub-columns: 1
-   
+
    .. rubric:: Footnotes
 
-   .. [#red-hat94-past-60] RHEL 9.4 is supported only on AMD Instinct MI300A.
-   .. [#oracle89-past-60] Oracle Linux is supported only on AMD Instinct MI300X.
+   .. [#mic300x-past-60] Oracle Linux and Azure Linux are supported only on AMD Instinct MI300X.
+   .. [#single-node-past-60] Debian 12 is supported only on AMD Instinct MI300X for single-node functionality. 
+   .. [#mi300_624-past-60] **For ROCm 6.2.4** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
+   .. [#mi300_622-past-60] **For ROCm 6.2.2** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
    .. [#mi300_621-past-60] **For ROCm 6.2.1** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
    .. [#mi300_620-past-60] **For ROCm 6.2.0** - MI300X (gfx942) is supported on listed operating systems *except* Ubuntu 22.04.5 [6.8 HWE] and Ubuntu 22.04.4 [6.5 HWE].
    .. [#mi300_612-past-60] **For ROCm 6.1.2** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4 and Oracle Linux.
@@ -222,4 +224,5 @@ Expand for full historical view of:
    .. [#mi300_610-past-60] **For ROCm 6.1.0** - MI300A (gfx942) is supported on Ubuntu 22.04.4, RHEL 9.4, RHEL 9.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.4.
    .. [#mi300_602-past-60] **For ROCm 6.0.2** - MI300A (gfx942) is supported on Ubuntu 22.04.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.3.
    .. [#mi300_600-past-60] **For ROCm 6.0.0** - MI300A (gfx942) is supported on Ubuntu 22.04.3, RHEL 8.9, and SLES 15 SP5. MI300X (gfx942) is only supported on Ubuntu 22.04.3.
-   .. [#kfd_support-past-60] ROCm provides forward and backward compatibility between the Kernel Fusion Driver (KFD) and its user space software for +/- 2 releases. The tested user space versions on this page were accurate as of the time of initial ROCm release. For the most up-to-date information, see the latest version of this information at `User and kernel-space support matrix <https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/user-kernel-space-compat-matrix.html>`_.
+   .. [#kfd_support-past-60] ROCm provides forward and backward compatibility between the AMD Kernel-mode GPU Driver (KMD) and its user space software for +/- 2 releases. These are the compatibility combinations that are currently supported.
+   .. [#ROCT-rocr-past-60] Starting from ROCm 6.3.0, the ROCT Thunk Interface is included as part of the ROCr runtime package.

docs/compatibility/ml-compatibility/jax-compatibility.rst

@@ -0,0 +1,663 @@
+.. meta::
+   :description: JAX compatibility
+   :keywords: GPU, JAX compatibility
+
+*******************************************************************************
+JAX compatibility
+*******************************************************************************
+
+JAX provides a NumPy-like API, which combines automatic differentiation and the
+Accelerated Linear Algebra (XLA) compiler to achieve high-performance machine
+learning at scale.
+
+JAX uses composable transformations of Python and NumPy through just-in-time (JIT) compilation,
+automatic vectorization, and parallelization. To learn about JAX, including profiling and
+optimizations, see the official `JAX documentation
+<https://jax.readthedocs.io/en/latest/notebooks/quickstart.html>`_.
+
+ROCm support for JAX is upstreamed and users can build the official source code with ROCm
+support:
+
+- ROCm JAX release:
+
+  - Offers AMD-validated and community :ref:`Docker images <jax-docker-compat>` with ROCm and JAX pre-installed.
+
+  - ROCm JAX repository: `<https://github.com/ROCm/jax>`__
+
+  - See the :doc:`ROCm JAX installation guide <rocm-install-on-linux:install/3rd-party/jax-install>`
+    to get started.
+
+- Official JAX release:
+
+  - Official JAX repository: `<https://github.com/jax-ml/jax>`__
+
+  - See the `AMD GPU (Linux) installation section
+    <https://jax.readthedocs.io/en/latest/installation.html#amd-gpu-linux>`_ in the JAX
+    documentation.
+
+.. note::
+
+   AMD releases official `ROCm JAX Docker images <https://hub.docker.com/r/rocm/jax>`_
+   quarterly alongside new ROCm releases. These images undergo full AMD testing.
+   `Community ROCm JAX Docker images <https://hub.docker.com/r/rocm/jax-community>`_
+   follow upstream JAX releases and use the latest available ROCm version.
+
+.. _jax-docker-compat:
+
+Docker image compatibility
+================================================================================
+
+.. |docker-icon| raw:: html
+
+   <i class="fab fa-docker"></i>
+
+AMD validates and publishes ready-made `JAX <https://hub.docker.com/r/rocm/jax/>`_
+images with ROCm backends on Docker Hub. The following Docker image tags and
+associated inventories are validated for
+`ROCm 6.3.1 <https://repo.radeon.com/rocm/apt/6.3.1/>`_. Click the |docker-icon|
+icon to view the image on Docker Hub.
+
+.. list-table:: JAX Docker image components
+    :header-rows: 1
+
+    * - Docker image
+      - JAX
+      - Linux
+      - Python
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/jax/rocm6.3.1-jax0.4.31-py3.12/images/sha256-085a0cd5207110922f1fca684933a9359c66d42db6c5aba4760ed5214fdabde0"><i class="fab fa-docker fa-lg"></i> rocm/jax</a>
+
+      - `0.4.31 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.31>`_
+      - Ubuntu 24.04
+      - `3.12.7 <https://www.python.org/downloads/release/python-3127/>`_
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/jax/rocm6.3.1-jax0.4.31-py3.10/images/sha256-f88eddad8f47856d8640b694da4da347ffc1750d7363175ab7dc872e82b43324"><i class="fab fa-docker fa-lg"></i> rocm/jax</a>
+
+      - `0.4.31 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.31>`_
+      - Ubuntu 22.04
+      - `3.10.14 <https://www.python.org/downloads/release/python-31014/>`_
+
+AMD publishes community `JAX <https://hub.docker.com/r/rocm/jax-community>`_
+images with ROCm backends on Docker Hub. The following Docker image tags and
+associated inventories are tested for `ROCm 6.2.4 <https://repo.radeon.com/rocm/apt/6.2.4/>`_.
+
+.. list-table:: JAX community Docker image components
+    :header-rows: 1
+
+    * - Docker image
+      - JAX
+      - Linux
+      - Python
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/jax-community/rocm6.2.4-jax0.4.35-py3.12.7/images/sha256-a6032d89c07573b84c44e42c637bf9752b1b7cd2a222d39344e603d8f4c63beb?context=explore"><i class="fab fa-docker fa-lg"></i> rocm/jax-community</a>
+
+      - `0.4.35 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.35>`_
+      - Ubuntu 22.04
+      - `3.12.7 <https://www.python.org/downloads/release/python-3127/>`_
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/jax-community/rocm6.2.4-jax0.4.35-py3.11.10/images/sha256-d462f7e445545fba2f3b92234a21beaa52fe6c5f550faabcfdcd1bf53486d991?context=explore"><i class="fab fa-docker fa-lg"></i> rocm/jax-community</a>
+
+      - `0.4.35 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.35>`_
+      - Ubuntu 22.04
+      - `3.11.10 <https://www.python.org/downloads/release/python-31110/>`_
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/jax-community/rocm6.2.4-jax0.4.35-py3.10.15/images/sha256-6f2d4d0f529378d9572f0e8cfdcbc101d1e1d335bd626bb3336fff87814e9d60?context=explore"><i class="fab fa-docker fa-lg"></i> rocm/jax-community</a>
+
+      - `0.4.35 <https://github.com/ROCm/jax/releases/tag/rocm-jax-v0.4.35>`_
+      - Ubuntu 22.04
+      - `3.10.15 <https://www.python.org/downloads/release/python-31015/>`_
+
+Critical ROCm libraries for JAX
+================================================================================
+
+The functionality of JAX with ROCm is determined by its underlying library
+dependencies. These critical ROCm components affect the capabilities,
+performance, and feature set available to developers.
+
+.. list-table::
+    :header-rows: 1
+
+    * - ROCm library
+      - Version
+      - Purpose
+      - Used in
+    * - `hipBLAS <https://github.com/ROCm/hipBLAS>`_
+      - 2.3.0
+      - Provides GPU-accelerated Basic Linear Algebra Subprograms (BLAS) for
+        matrix and vector operations.
+      - Matrix multiplication in ``jax.numpy.matmul``, ``jax.lax.dot`` and
+        ``jax.lax.dot_general``, operations like ``jax.numpy.dot``, which
+        involve vector and matrix computations and batch matrix multiplications
+        ``jax.numpy.einsum`` with matrix-multiplication patterns algebra
+        operations.
+    * - `hipBLASLt <https://github.com/ROCm/hipBLASLt>`_
+      - 0.10.0
+      - hipBLASLt is an extension of hipBLAS, providing additional
+        features like epilogues fused into the matrix multiplication kernel or
+        use of integer tensor cores.
+      - Matrix multiplication in ``jax.numpy.matmul`` or ``jax.lax.dot``, and
+        the XLA (Accelerated Linear Algebra) use hipBLASLt for optimized matrix
+        operations, mixed-precision support, and hardware-specific
+        optimizations.
+    * - `hipCUB <https://github.com/ROCm/hipCUB>`_
+      - 3.3.0
+      - Provides a C++ template library for parallel algorithms for reduction,
+        scan, sort and select.
+      - Reduction functions (``jax.numpy.sum``, ``jax.numpy.mean``, 
+        ``jax.numpy.prod``, ``jax.numpy.max`` and ``jax.numpy.min``), prefix sum
+        (``jax.numpy.cumsum``, ``jax.numpy.cumprod``) and sorting
+        (``jax.numpy.sort``, ``jax.numpy.argsort``).
+    * - `hipFFT <https://github.com/ROCm/hipFFT>`_
+      - 1.0.17
+      - Provides GPU-accelerated Fast Fourier Transform (FFT) operations.
+      - Used in functions like ``jax.numpy.fft``.
+    * - `hipRAND <https://github.com/ROCm/hipRAND>`_
+      - 2.11.0
+      - Provides fast random number generation for GPUs.
+      - The ``jax.random.uniform``, ``jax.random.normal``,
+        ``jax.random.randint`` and ``jax.random.split``.
+    * - `hipSOLVER <https://github.com/ROCm/hipSOLVER>`_
+      - 2.3.0
+      - Provides GPU-accelerated solvers for linear systems, eigenvalues, and
+        singular value decompositions (SVD).
+      - Solving linear systems (``jax.numpy.linalg.solve``), matrix
+        factorizations, SVD (``jax.numpy.linalg.svd``) and eigenvalue problems 
+        (``jax.numpy.linalg.eig``).
+    * - `hipSPARSE <https://github.com/ROCm/hipSPARSE>`_
+      - 3.1.2
+      - Accelerates operations on sparse matrices, such as sparse matrix-vector
+        or matrix-matrix products.
+      - Sparse matrix multiplication (``jax.numpy.matmul``), sparse
+        matrix-vector and matrix-matrix products
+        (``jax.experimental.sparse.dot``), sparse linear system solvers and
+        sparse data handling.
+    * - `hipSPARSELt <https://github.com/ROCm/hipSPARSELt>`_
+      - 0.2.2
+      - Accelerates operations on sparse matrices, such as sparse matrix-vector
+        or matrix-matrix products.
+      - Sparse matrix multiplication (``jax.numpy.matmul``), sparse
+        matrix-vector and matrix-matrix products
+        (``jax.experimental.sparse.dot``) and sparse linear system solvers.
+    * - `MIOpen <https://github.com/ROCm/MIOpen>`_
+      - 3.3.0
+      - Optimized for deep learning primitives such as convolutions, pooling,
+        normalization, and activation functions.
+      - Speeds up convolutional neural networks (CNNs), recurrent neural
+        networks (RNNs), and other layers. Used in operations like
+        ``jax.nn.conv``, ``jax.nn.relu``, and ``jax.nn.batch_norm``.
+    * - `RCCL <https://github.com/ROCm/rccl>`_
+      - 2.21.5
+      - Optimized for multi-GPU communication for operations like  all-reduce,
+        broadcast, and scatter.
+      - Distribute computations across multiple GPU with ``pmap`` and
+        ``jax.distributed``. XLA automatically uses rccl when executing
+        operations across multiple GPUs on AMD hardware.
+    * - `rocThrust <https://github.com/ROCm/rocThrust>`_
+      - 3.3.0
+      - Provides a C++ template library for parallel algorithms like sorting,
+        reduction, and scanning.
+      - Reduction operations like ``jax.numpy.sum``, ``jax.pmap`` for
+        distributed training, which involves parallel reductions or
+        operations like ``jax.numpy.cumsum`` can use rocThrust.
+
+Supported and unsupported features
+===============================================================================
+
+The following table maps GPU-accelerated JAX modules to their supported
+ROCm and JAX versions.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Module
+      - Description
+      - Since JAX
+      - Since ROCm
+    * - ``jax.numpy``
+      - Implements the NumPy API, using the primitives in ``jax.lax``.
+      - 0.1.56
+      - 5.0.0
+    * - ``jax.scipy``
+      - Provides GPU-accelerated and differentiable implementations of many
+        functions from the SciPy library, leveraging JAX's transformations
+        (e.g., ``grad``, ``jit``, ``vmap``).
+      - 0.1.56
+      - 5.0.0
+    * - ``jax.lax``
+      - A library of primitives operations that underpins libraries such as
+        ``jax.numpy.`` Transformation rules, such as Jacobian-vector product
+        (JVP) and batching rules, are typically defined as transformations on
+        ``jax.lax`` primitives.
+      - 0.1.57
+      - 5.0.0
+    * - ``jax.random``
+      - Provides a number of routines for deterministic generation of sequences
+        of pseudorandom numbers.
+      - 0.1.58
+      - 5.0.0
+    * - ``jax.sharding``
+      - Allows to define partitioning and distributing arrays across multiple
+        devices.
+      - 0.3.20
+      - 5.1.0
+    * - ``jax.dlpack``
+      - For exchanging tensor data between JAX and other libraries that support the
+        DLPack standard.
+      - 0.1.57
+      - 5.0.0
+    * - ``jax.distributed``
+      - Enables the scaling of computations across multiple devices on a single
+        machine or across multiple machines.
+      - 0.1.74
+      - 5.0.0
+    * - ``jax.dtypes``
+      - Provides utilities for working with and managing data types in JAX
+        arrays and computations.
+      - 0.1.66
+      - 5.0.0 
+    * - ``jax.image``
+      - Contains image manipulation functions like resize, scale and translation.
+      - 0.1.57
+      - 5.0.0
+    * - ``jax.nn``
+      - Contains common functions for neural network libraries.
+      - 0.1.56
+      - 5.0.0
+    * - ``jax.ops``
+      - Computes the minimum, maximum, sum or product within segments of an
+        array.
+      - 0.1.57
+      - 5.0.0
+    * - ``jax.profiler``
+      - Contains JAX’s tracing and time profiling features.
+      - 0.1.57
+      - 5.0.0
+    * - ``jax.stages``
+      - Contains interfaces to stages of the compiled execution process.
+      - 0.3.4
+      - 5.0.0
+    * - ``jax.tree``
+      - Provides utilities for working with tree-like container data structures.
+      - 0.4.26
+      - 5.6.0
+    * - ``jax.tree_util``
+      - Provides utilities for working with nested data structures, or
+        ``pytrees``.
+      - 0.1.65
+      - 5.0.0
+    * - ``jax.typing``
+      - Provides JAX-specific static type annotations.
+      - 0.3.18
+      - 5.1.0
+    * - ``jax.extend``
+      - Provides modules for access to JAX internal machinery module. The
+        ``jax.extend`` module defines a library view of some of JAX’s internal
+        components.
+      - 0.4.15
+      - 5.5.0
+    * - ``jax.example_libraries``
+      - Serves as a collection of example code and libraries that demonstrate
+        various capabilities of JAX.
+      - 0.1.74
+      - 5.0.0
+    * - ``jax.experimental``
+      - Namespace for experimental features and APIs that are in development or
+        are not yet fully stable for production use.
+      - 0.1.56
+      - 5.0.0
+    * - ``jax.lib``
+      - Set of internal tools and types for bridging between JAX’s Python
+        frontend and its XLA backend.
+      - 0.4.6
+      - 5.3.0
+    * - ``jax_triton``
+      - Library that integrates the Triton deep learning compiler with JAX.
+      - jax_triton 0.2.0 
+      - 6.2.4
+
+jax.scipy module
+-------------------------------------------------------------------------------
+
+A SciPy-like API for scientific computing.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Module
+      - Since JAX
+      - Since ROCm
+    * - ``jax.scipy.cluster``
+      - 0.3.11
+      - 5.1.0
+    * - ``jax.scipy.fft``
+      - 0.1.71
+      - 5.0.0
+    * - ``jax.scipy.integrate``
+      - 0.4.15
+      - 5.5.0
+    * - ``jax.scipy.interpolate``
+      - 0.1.76
+      - 5.0.0
+    * - ``jax.scipy.linalg``
+      - 0.1.56
+      - 5.0.0
+    * - ``jax.scipy.ndimage``
+      - 0.1.56
+      - 5.0.0
+    * - ``jax.scipy.optimize``
+      - 0.1.57
+      - 5.0.0
+    * - ``jax.scipy.signal``
+      - 0.1.56
+      - 5.0.0
+    * - ``jax.scipy.spatial.transform``
+      - 0.4.12
+      - 5.4.0
+    * - ``jax.scipy.sparse.linalg``
+      - 0.1.56
+      - 5.0.0
+    * - ``jax.scipy.special``
+      - 0.1.56
+      - 5.0.0
+    * - ``jax.scipy.stats``
+      - 0.1.56
+      - 5.0.0
+
+jax.scipy.stats module
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. list-table::
+   :header-rows: 1
+
+   * - Module
+     - Since JAX
+     - Since ROCm
+   * - ``jax.scipy.stats.bernouli``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.beta``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.betabinom``
+     - 0.1.61
+     - 5.0.0
+   * - ``jax.scipy.stats.binom``
+     - 0.4.14
+     - 5.4.0
+   * - ``jax.scipy.stats.cauchy``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.chi2``
+     - 0.1.61
+     - 5.0.0
+   * - ``jax.scipy.stats.dirichlet``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.expon``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.gamma``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.gennorm``
+     - 0.3.15
+     - 5.2.0
+   * - ``jax.scipy.stats.geom``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.laplace``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.logistic``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.multinomial``
+     - 0.3.18
+     - 5.1.0
+   * - ``jax.scipy.stats.multivariate_normal``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.nbinom``
+     - 0.1.72
+     - 5.0.0
+   * - ``jax.scipy.stats.norm``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.pareto``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.poisson``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.t``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.truncnorm``
+     - 0.4.0
+     - 5.3.0
+   * - ``jax.scipy.stats.uniform``
+     - 0.1.56
+     - 5.0.0
+   * - ``jax.scipy.stats.vonmises``
+     - 0.4.2
+     - 5.3.0
+   * - ``jax.scipy.stats.wrapcauchy``
+     - 0.4.20
+     - 5.6.0
+
+jax.extend module
+-------------------------------------------------------------------------------
+
+Modules for JAX extensions.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Module
+      - Since JAX
+      - Since ROCm
+    * - ``jax.extend.ffi``
+      - 0.4.30
+      - 6.0.0
+    * - ``jax.extend.linear_util``
+      - 0.4.17
+      - 5.6.0
+    * - ``jax.extend.mlir``
+      - 0.4.26
+      - 5.6.0
+    * - ``jax.extend.random``
+      - 0.4.15
+      - 5.5.0
+
+jax.experimental module
+-------------------------------------------------------------------------------
+
+Experimental modules and APIs.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Module
+      - Since JAX
+      - Since ROCm
+    * - ``jax.experimental.checkify``
+      - 0.1.75
+      - 5.0.0
+    * - ``jax.experimental.compilation_cache.compilation_cache``
+      - 0.1.68
+      - 5.0.0
+    * - ``jax.experimental.custom_partitioning``
+      - 0.4.0
+      - 5.3.0
+    * - ``jax.experimental.jet``
+      - 0.1.56
+      - 5.0.0
+    * - ``jax.experimental.key_reuse``
+      - 0.4.26
+      - 5.6.0
+    * - ``jax.experimental.mesh_utils``
+      - 0.1.76
+      - 5.0.0
+    * - ``jax.experimental.multihost_utils``
+      - 0.3.2
+      - 5.0.0
+    * - ``jax.experimental.pallas``
+      - 0.4.15
+      - 5.5.0
+    * - ``jax.experimental.pjit``
+      - 0.1.61
+      - 5.0.0
+    * - ``jax.experimental.serialize_executable``
+      - 0.4.0
+      - 5.3.0
+    * - ``jax.experimental.shard_map``
+      - 0.4.3
+      - 5.3.0
+    * - ``jax.experimental.sparse``
+      - 0.1.75
+      - 5.0.0
+
+.. list-table::
+    :header-rows: 1
+
+    * - API
+      - Since JAX
+      - Since ROCm
+    * - ``jax.experimental.enable_x64``
+      - 0.1.60
+      - 5.0.0
+    * - ``jax.experimental.disable_x64``
+      - 0.1.60
+      - 5.0.0
+
+jax.experimental.pallas module
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Module for Pallas, a JAX extension for custom kernels.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Module
+      - Since JAX
+      - Since ROCm
+    * - ``jax.experimental.pallas.mosaic_gpu``
+      - 0.4.31
+      - 6.1.3
+    * - ``jax.experimental.pallas.tpu``
+      - 0.4.15
+      - 5.5.0
+    * - ``jax.experimental.pallas.triton``
+      - 0.4.32
+      - 6.1.3
+
+jax.experimental.sparse module
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Experimental support for sparse matrix operations.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Module
+      - Since JAX
+      - Since ROCm
+    * - ``jax.experimental.sparse.linalg``
+      - 0.3.15
+      - 5.2.0
+    * - ``jax.experimental.sparse.sparsify``
+      - 0.3.25
+      - ❌
+
+.. list-table::
+    :header-rows: 1
+
+    * - ``sparse`` data structure API
+      - Since JAX
+      - Since ROCm
+    * - ``jax.experimental.sparse.BCOO``
+      - 0.1.72
+      - 5.0.0
+    * - ``jax.experimental.sparse.BCSR``
+      - 0.3.20
+      - 5.1.0
+    * - ``jax.experimental.sparse.CSR``
+      - 0.1.75
+      - 5.0.0
+    * - ``jax.experimental.sparse.NM``
+      - 0.4.27
+      - 5.6.0
+    * - ``jax.experimental.sparse.COO``
+      - 0.1.75
+      - 5.0.0
+
+Unsupported JAX features
+------------------------
+
+The following are GPU-accelerated JAX features not currently supported by
+ROCm.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Feature
+      - Description
+      - Since JAX
+    * - Mixed Precision with TF32
+      - Mixed precision with TF32 is used for matrix multiplications,
+        convolutions, and other linear algebra operations, particularly in
+        deep learning workloads like CNNs and transformers.
+      - 0.2.25
+    * - RNN support
+      - Currently only LSTM with double bias is supported with float32 input
+        and weight.
+      - 0.3.25
+    * - XLA int4 support
+      - 4-bit integer (int4) precision in the XLA compiler.
+      - 0.4.0
+    * - ``jax.experimental.sparsify``
+      - Converts a dense matrix to a sparse matrix representation.
+      - Experimental
+
+Use cases and recommendations
+================================================================================
+
+* The `nanoGPT in JAX <https://rocm.blogs.amd.com/artificial-intelligence/nanoGPT-JAX/README.html>`_
+  blog explores the implementation and training of a Generative Pre-trained
+  Transformer (GPT) model in JAX, inspired by Andrej Karpathy’s PyTorch-based
+  nanoGPT. By comparing how essential GPT components—such as self-attention
+  mechanisms and optimizers—are realized in PyTorch and JAX, also highlight
+  JAX’s unique features.
+
+* The `Optimize GPT Training: Enabling Mixed Precision Training in JAX using
+  ROCm on AMD GPUs <https://rocm.blogs.amd.com/artificial-intelligence/jax-mixed-precision/README.html>`_
+  blog post provides a comprehensive guide on enhancing the training efficiency
+  of GPT models by implementing mixed precision techniques in JAX, specifically
+  tailored for AMD GPUs utilizing the ROCm platform.
+
+* The `Supercharging JAX with Triton Kernels on AMD GPUs <https://rocm.blogs.amd.com/artificial-intelligence/jax-triton/README.html>`_
+  blog demonstrates how to develop a custom fused dropout-activation kernel for
+  matrices using Triton, integrate it with JAX, and benchmark its performance
+  using ROCm.
+
+* The `Distributed fine-tuning with JAX on AMD GPUs <https://rocm.blogs.amd.com/artificial-intelligence/distributed-sft-jax/README.html>`_
+  outlines the process of fine-tuning a Bidirectional Encoder Representations
+  from Transformers (BERT)-based large language model (LLM) using JAX for a text
+  classification task. The blog post discuss techniques for parallelizing the
+  fine-tuning across multiple AMD GPUs and assess the model's performance on a
+  holdout dataset. During the fine-tuning, a BERT-base-cased transformer model
+  and the General Language Understanding Evaluation (GLUE) benchmark dataset was
+  used on a multi-GPU setup.
+
+* The `MI300X workload optimization guide <https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/workload.html>`_
+  provides detailed guidance on optimizing workloads for the AMD Instinct MI300X
+  accelerator using ROCm. The page is aimed at helping users achieve optimal
+  performance for deep learning and other high-performance computing tasks on
+  the MI300X GPU.
+
+For more use cases and recommendations, see `ROCm JAX blog posts <https://rocm.blogs.amd.com/blog/tag/jax.html>`_.

docs/compatibility/ml-compatibility/pytorch-compatibility.rst

@@ -0,0 +1,922 @@
+.. meta::
+    :description: PyTorch compatibility
+    :keywords: GPU, PyTorch compatibility
+
+********************************************************************************
+PyTorch compatibility
+********************************************************************************
+
+`PyTorch <https://pytorch.org/>`_ is an open-source tensor library designed for
+deep learning. PyTorch on ROCm provides mixed-precision and large-scale training
+using `MIOpen <https://github.com/ROCm/MIOpen>`_ and
+`RCCL <https://github.com/ROCm/rccl>`_ libraries.
+
+ROCm support for PyTorch is upstreamed into the official PyTorch repository. Due
+to independent compatibility considerations, this results in two distinct
+release cycles for PyTorch on ROCm:
+
+- ROCm PyTorch release:
+
+  - Provides the latest version of ROCm but doesn't immediately support the latest stable PyTorch
+    version.
+
+  - Offers :ref:`Docker images <pytorch-docker-compat>` with ROCm and PyTorch
+    pre-installed.
+
+  - ROCm PyTorch repository: `<https://github.com/ROCm/pytorch>`__
+
+  - See the :doc:`ROCm PyTorch installation guide <rocm-install-on-linux:install/3rd-party/pytorch-install>` to get started.
+
+- Official PyTorch release:
+
+  - Provides the latest stable version of PyTorch but doesn't immediately support the latest ROCm version.
+
+  - Official PyTorch repository: `<https://github.com/pytorch/pytorch>`__
+
+  - See the `Nightly and latest stable version installation guide <https://pytorch.org/get-started/locally/>`_
+    or `Previous versions <https://pytorch.org/get-started/previous-versions/>`_ to get started.
+
+The upstream PyTorch includes an automatic HIPification solution that automatically generates HIP
+source code from the CUDA backend. This approach allows PyTorch to support ROCm without requiring
+manual code modifications.
+
+Development of ROCm is aligned with the stable release of PyTorch while upstream PyTorch testing uses
+the stable release of ROCm to maintain consistency.
+
+.. _pytorch-docker-compat:
+
+Docker image compatibility
+================================================================================
+
+.. |docker-icon| raw:: html
+
+   <i class="fab fa-docker"></i>
+
+AMD validates and publishes ready-made `PyTorch <https://hub.docker.com/r/rocm/pytorch>`_
+images with ROCm backends on Docker Hub. The following Docker image tags and
+associated inventories are validated for `ROCm 6.3.0 <https://repo.radeon.com/rocm/apt/6.3/>`_.
+Click the |docker-icon| icon to view the image on Docker Hub.
+
+.. list-table:: PyTorch Docker image components
+    :header-rows: 1
+    :class: docker-image-compatibility
+
+    * - Docker
+      - PyTorch
+      - Ubuntu
+      - Python
+      - Apex
+      - torchvision
+      - TensorBoard
+      - MAGMA
+      - UCX
+      - OMPI
+      - OFED
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu24.04_py3.12_pytorch_release_2.4.0/images/sha256-98ddf20333bd01ff749b8092b1190ee369a75d3b8c71c2fac80ffdcb1a98d529?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `2.4.0 <https://github.com/ROCm/pytorch/tree/release/2.4>`_
+      - 24.04
+      - `3.12 <https://www.python.org/downloads/release/python-3128/>`_
+      - `1.4.0 <https://github.com/ROCm/apex/tree/release/1.4.0>`_
+      - `0.19.0 <https://github.com/pytorch/vision/tree/v0.19.0>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+      - `4.0.7 <https://github.com/open-mpi/ompi/tree/v4.0.7>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu22.04_py3.10_pytorch_release_2.4.0/images/sha256-402c9b4f1a6b5a81c634a1932b56cbe01abb699cfcc7463d226276997c6cf8ea?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `2.4.0 <https://github.com/ROCm/pytorch/tree/release/2.4>`_
+      - 22.04
+      - `3.10 <https://www.python.org/downloads/release/python-31016/>`_
+      - `1.4.0 <https://github.com/ROCm/apex/tree/release/1.4.0>`_
+      - `0.19.0 <https://github.com/pytorch/vision/tree/v0.19.0>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+      - `4.0.7 <https://github.com/open-mpi/ompi/tree/v4.0.7>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu22.04_py3.9_pytorch_release_2.4.0/images/sha256-e0608b55d408c3bfe5c19fdd57a4ced3e0eb3a495b74c309980b60b156c526dd?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `2.4.0 <https://github.com/ROCm/pytorch/tree/release/2.4>`_
+      - 22.04
+      - `3.9 <https://www.python.org/downloads/release/python-3918/>`_
+      - `1.4.0 <https://github.com/ROCm/apex/tree/release/1.4.0>`_
+      - `0.19.0 <https://github.com/pytorch/vision/tree/v0.19.0>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+      - `4.0.7 <https://github.com/open-mpi/ompi/tree/v4.0.7>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu22.04_py3.10_pytorch_release_2.3.0/images/sha256-652cf25263d05b1de548222970aeb76e60b12de101de66751264709c0d0ff9d8?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `2.3.0 <https://github.com/ROCm/pytorch/tree/release/2.3>`_
+      - 22.04
+      - `3.10 <https://www.python.org/downloads/release/python-31016/>`_
+      - `1.3.0 <https://github.com/ROCm/apex/tree/release/1.3.0>`_
+      - `0.18.0 <https://github.com/pytorch/vision/tree/v0.18.0>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.14.1 <https://github.com/openucx/ucx/tree/v1.14.1>`_
+      - `4.1.5 <https://github.com/open-mpi/ompi/tree/v4.1.5>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu22.04_py3.10_pytorch_release_2.2.1/images/sha256-051976f26beab8f9aa65d999e3ad546c027b39240a0cc3ee81b114a9024f2912?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `2.2.1 <https://github.com/ROCm/pytorch/tree/release/2.2>`_
+      - 22.04
+      - `3.10 <https://www.python.org/downloads/release/python-31016/>`_
+      - `1.2.0 <https://github.com/ROCm/apex/tree/release/1.2.0>`_
+      - `0.17.1 <https://github.com/pytorch/vision/tree/v0.17.1>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.14.1 <https://github.com/openucx/ucx/tree/v1.14.1>`_
+      - `4.1.5 <https://github.com/open-mpi/ompi/tree/v4.1.5>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu20.04_py3.9_pytorch_release_2.2.1/images/sha256-88c839a364d109d3748c100385bfa100d28090d25118cc723fd0406390ab2f7e?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `2.2.1 <https://github.com/ROCm/pytorch/tree/release/2.2>`_
+      - 20.04
+      - `3.9 <https://www.python.org/downloads/release/python-3921/>`_
+      - `1.2.0 <https://github.com/ROCm/apex/tree/release/1.2.0>`_
+      - `0.17.1 <https://github.com/pytorch/vision/tree/v0.17.1>`_
+      - `2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13.0>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+      - `4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu22.04_py3.9_pytorch_release_1.13.1/images/sha256-994424ed07a63113f79dd9aa72159124c00f5fbfe18127151e6658f7d0b6f821?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `1.13.1 <https://github.com/ROCm/pytorch/tree/release/1.13>`_
+      - 22.04
+      - `3.9 <https://www.python.org/downloads/release/python-3921/>`_
+      - `1.0.0 <https://github.com/ROCm/apex/tree/release/1.0.0>`_
+      - `0.14.0 <https://github.com/pytorch/vision/tree/v0.14.0>`_
+      - `2.18.0 <https://github.com/tensorflow/tensorboard/tree/2.18>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.14.1 <https://github.com/openucx/ucx/tree/v1.14.1>`_
+      - `4.1.5 <https://github.com/open-mpi/ompi/tree/v4.1.5>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+    * - .. raw:: html
+
+           <a href="https://hub.docker.com/layers/rocm/pytorch/rocm6.3_ubuntu20.04_py3.9_pytorch_release_1.13.1/images/sha256-7b8139fe40a9aeb4bca3aecd15c22c1fa96e867d93479fa3a24fdeeeeafa1219?context=explore"><i class="fab fa-docker fa-lg"></i></a>
+
+      - `1.13.1 <https://github.com/ROCm/pytorch/tree/release/1.13>`_
+      - 20.04
+      - `3.9 <https://www.python.org/downloads/release/python-3921/>`_
+      - `1.0.0 <https://github.com/ROCm/apex/tree/release/1.0.0>`_
+      - `0.14.0 <https://github.com/pytorch/vision/tree/v0.14.0>`_
+      - `2.18.0 <https://github.com/tensorflow/tensorboard/tree/2.18>`_
+      - `master <https://bitbucket.org/icl/magma/src/master/>`_
+      - `1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+      - `4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
+      - `5.3-1.0.5.0 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+Critical ROCm libraries for PyTorch
+================================================================================
+
+The functionality of PyTorch with ROCm is determined by its underlying library
+dependencies. These critical ROCm components affect the capabilities,
+performance, and feature set available to developers.
+
+.. list-table::
+    :header-rows: 1
+
+    * - ROCm library
+      - Version
+      - Purpose
+      - Used in
+    * - `Composable Kernel <https://github.com/ROCm/composable_kernel>`_
+      - 1.1.0
+      - Enables faster execution of core operations like matrix multiplication
+        (GEMM), convolutions and transformations.
+      - Speeds up ``torch.permute``, ``torch.view``, ``torch.matmul``,
+        ``torch.mm``, ``torch.bmm``, ``torch.nn.Conv2d``, ``torch.nn.Conv3d``
+        and ``torch.nn.MultiheadAttention``. 
+    * - `hipBLAS <https://github.com/ROCm/hipBLAS>`_
+      - 2.3.0
+      - Provides GPU-accelerated Basic Linear Algebra Subprograms (BLAS) for
+        matrix and vector operations.
+      - Supports operations like matrix multiplication, matrix-vector products,
+        and tensor contractions. Utilized in both dense and batched linear
+        algebra operations.
+    * - `hipBLASLt <https://github.com/ROCm/hipBLASLt>`_
+      - 0.10.0
+      - hipBLASLt is an extension of the hipBLAS library, providing additional
+        features like epilogues fused into the matrix multiplication kernel or
+        use of integer tensor cores.
+      - It accelerates operations like ``torch.matmul``, ``torch.mm``, and the
+        matrix multiplications used in convolutional and linear layers.
+    * - `hipCUB <https://github.com/ROCm/hipCUB>`_
+      - 3.3.0
+      - Provides a C++ template library for parallel algorithms for reduction,
+        scan, sort and select.
+      - Supports operations like ``torch.sum``, ``torch.cumsum``, ``torch.sort``
+        and ``torch.topk``. Operations on sparse tensors or tensors with
+        irregular shapes often involve scanning, sorting, and filtering, which
+        hipCUB handles efficiently.
+    * - `hipFFT <https://github.com/ROCm/hipFFT>`_
+      - 1.0.17
+      - Provides GPU-accelerated Fast Fourier Transform (FFT) operations.
+      - Used in functions like the ``torch.fft`` module.
+    * - `hipRAND <https://github.com/ROCm/hipRAND>`_
+      - 2.11.0
+      - Provides fast random number generation for GPUs.
+      - The ``torch.rand``, ``torch.randn`` and stochastic layers like 
+        ``torch.nn.Dropout``.
+    * - `hipSOLVER <https://github.com/ROCm/hipSOLVER>`_
+      - 2.3.0
+      - Provides GPU-accelerated solvers for linear systems, eigenvalues, and
+        singular value decompositions (SVD).
+      - Supports functions like ``torch.linalg.solve``,
+        ``torch.linalg.eig``, and ``torch.linalg.svd``.
+    * - `hipSPARSE <https://github.com/ROCm/hipSPARSE>`_
+      - 3.1.2
+      - Accelerates operations on sparse matrices, such as sparse matrix-vector
+        or matrix-matrix products.
+      - Sparse tensor operations ``torch.sparse``.
+    * - `hipSPARSELt <https://github.com/ROCm/hipSPARSELt>`_
+      - 0.2.2
+      - Accelerates operations on sparse matrices, such as sparse matrix-vector
+        or matrix-matrix products.
+      - Sparse tensor operations ``torch.sparse``.
+    * - `hipTensor <https://github.com/ROCm/hipTensor>`_
+      - 1.4.0
+      - Optimizes for high-performance tensor operations, such as contractions.
+      - Accelerates tensor algebra, especially in deep learning and scientific
+        computing.
+    * - `MIOpen <https://github.com/ROCm/MIOpen>`_
+      - 3.3.0
+      - Optimizes deep learning primitives such as convolutions, pooling,
+        normalization, and activation functions.
+      - Speeds up convolutional neural networks (CNNs), recurrent neural
+        networks (RNNs), and other layers. Used in operations like
+        ``torch.nn.Conv2d``, ``torch.nn.ReLU``, and ``torch.nn.LSTM``.
+    * - `MIGraphX <https://github.com/ROCm/AMDMIGraphX>`_
+      - 2.11.0
+      - Adds graph-level optimizations, ONNX models and mixed precision support
+        and enable Ahead-of-Time (AOT) Compilation.
+      - Speeds up inference models and executes ONNX models for
+        compatibility with other frameworks.
+        ``torch.nn.Conv2d``, ``torch.nn.ReLU``, and ``torch.nn.LSTM``.
+    * - `MIVisionX <https://github.com/ROCm/MIVisionX>`_
+      - 3.1.0
+      - Optimizes acceleration for computer vision and AI workloads like
+        preprocessing, augmentation, and inferencing.
+      - Faster data preprocessing and augmentation pipelines for datasets like
+        ImageNet or COCO and easy to integrate into PyTorch's ``torch.utils.data``
+        and ``torchvision`` workflows.
+    * - `rocAL <https://github.com/ROCm/rocAL>`_
+      - 2.1.0
+      - Accelerates the data pipeline by offloading intensive preprocessing and
+        augmentation tasks. rocAL is part of MIVisionX.
+      - Easy to integrate into PyTorch's ``torch.utils.data`` and
+        ``torchvision`` data load workloads.
+    * - `RCCL <https://github.com/ROCm/rccl>`_
+      - 2.21.5
+      - Optimizes for multi-GPU communication for operations like AllReduce and
+        Broadcast.
+      - Distributed data parallel training (``torch.nn.parallel.DistributedDataParallel``).
+        Handles communication in multi-GPU setups.
+    * - `rocDecode <https://github.com/ROCm/rocDecode>`_
+      - 0.8.0
+      - Provides hardware-accelerated data decoding capabilities, particularly
+        for image, video, and other dataset formats.
+      - Can be integrated in ``torch.utils.data``, ``torchvision.transforms``
+        and ``torch.distributed``.
+    * - `rocJPEG <https://github.com/ROCm/rocJPEG>`_
+      - 0.6.0
+      - Provides hardware-accelerated JPEG image decoding and encoding.
+      - GPU accelerated ``torchvision.io.decode_jpeg`` and
+        ``torchvision.io.encode_jpeg`` and can be integrated in
+        ``torch.utils.data`` and ``torchvision``.
+    * - `RPP <https://github.com/ROCm/RPP>`_
+      - 1.9.1
+      - Speeds up data augmentation, transformation, and other preprocessing steps.
+      - Easy to integrate into PyTorch's ``torch.utils.data`` and
+        ``torchvision`` data load workloads.
+    * - `rocThrust <https://github.com/ROCm/rocThrust>`_
+      - 3.3.0
+      - Provides a C++ template library for parallel algorithms like sorting,
+        reduction, and scanning.
+      - Utilized in backend operations for tensor computations requiring
+        parallel processing.
+    * - `rocWMMA <https://github.com/ROCm/rocWMMA>`_
+      - 1.6.0
+      - Accelerates warp-level matrix-multiply and matrix-accumulate to speed up matrix
+        multiplication (GEMM) and accumulation operations with mixed precision
+        support.
+      - Linear layers (``torch.nn.Linear``), convolutional layers
+        (``torch.nn.Conv2d``), attention layers, general tensor operations that
+        involve matrix products, such as ``torch.matmul``, ``torch.bmm``, and
+        more.
+
+Supported and unsupported features
+================================================================================
+
+The following section maps GPU-accelerated PyTorch features to their supported
+ROCm and PyTorch versions.
+
+torch
+--------------------------------------------------------------------------------
+
+`torch <https://pytorch.org/docs/stable/index.html>`_ is the central module of
+PyTorch, providing data structures for multi-dimensional tensors and
+implementing mathematical operations on them. It also includes utilities for
+efficient serialization of tensors and arbitrary data types, along with various
+other tools.
+
+Tensor data types
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The data type of a tensor is specified using the ``dtype`` attribute or argument, and PyTorch supports a wide range of data types for different use cases.
+
+The following table lists `torch.Tensor <https://pytorch.org/docs/stable/tensors.html>`_'s single data types:
+
+.. list-table::
+    :header-rows: 1
+
+    * - Data type
+      - Description
+      - Since PyTorch
+      - Since ROCm
+    * - ``torch.float8_e4m3fn``
+      - 8-bit floating point, e4m3
+      - 2.3
+      - 5.5
+    * - ``torch.float8_e5m2``
+      - 8-bit floating point, e5m2
+      - 2.3
+      - 5.5
+    * - ``torch.float16`` or ``torch.half``
+      - 16-bit floating point
+      - 0.1.6
+      - 2.0
+    * - ``torch.bfloat16``
+      - 16-bit floating point
+      - 1.6
+      - 2.6
+    * - ``torch.float32`` or ``torch.float``
+      - 32-bit floating point
+      - 0.1.12_2
+      - 2.0
+    * - ``torch.float64`` or ``torch.double``
+      - 64-bit floating point
+      - 0.1.12_2
+      - 2.0
+    * - ``torch.complex32`` or ``torch.chalf``
+      - PyTorch provides native support for 32-bit complex numbers
+      - 1.6
+      - 2.0
+    * - ``torch.complex64`` or ``torch.cfloat``
+      - PyTorch provides native support for 64-bit complex numbers
+      - 1.6
+      - 2.0
+    * - ``torch.complex128`` or ``torch.cdouble``
+      - PyTorch provides native support for 128-bit complex numbers
+      - 1.6
+      - 2.0
+    * - ``torch.uint8``
+      - 8-bit integer (unsigned)
+      - 0.1.12_2
+      - 2.0
+    * - ``torch.uint16``
+      - 16-bit integer (unsigned)
+      - 2.3
+      - Not natively supported
+    * - ``torch.uint32``
+      - 32-bit integer (unsigned)
+      - 2.3
+      - Not natively supported
+    * - ``torch.uint64``
+      - 32-bit integer (unsigned)
+      - 2.3
+      - Not natively supported
+    * - ``torch.int8``
+      - 8-bit integer (signed)
+      - 1.12
+      - 5.0
+    * - ``torch.int16`` or ``torch.short``
+      - 16-bit integer (signed)
+      - 0.1.12_2
+      - 2.0
+    * - ``torch.int32`` or ``torch.int``
+      - 32-bit integer (signed)
+      - 0.1.12_2
+      - 2.0
+    * - ``torch.int64`` or ``torch.long``
+      - 64-bit integer (signed)
+      - 0.1.12_2
+      - 2.0
+    * - ``torch.bool``
+      - Boolean
+      - 1.2
+      - 2.0
+    * - ``torch.quint8``
+      - Quantized 8-bit integer (unsigned)
+      - 1.8
+      - 5.0
+    * - ``torch.qint8``
+      - Quantized 8-bit integer (signed)
+      - 1.8
+      - 5.0
+    * - ``torch.qint32``
+      - Quantized 32-bit integer (signed)
+      - 1.8
+      - 5.0
+    * - ``torch.quint4x2``
+      - Quantized 4-bit integer (unsigned)
+      - 1.8
+      - 5.0
+
+.. note::
+
+  Unsigned types aside from ``uint8`` are currently only have limited support in
+  eager mode (they primarily exist to assist usage with ``torch.compile``).
+
+  The :doc:`ROCm precision support page <rocm:reference/precision-support>`
+  collected the native HW support of different data types.
+
+torch.cuda
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``torch.cuda`` in PyTorch is a module that provides utilities and functions for
+managing and utilizing AMD and NVIDIA GPUs. It enables GPU-accelerated
+computations, memory management, and efficient execution of tensor operations,
+leveraging ROCm and CUDA as the underlying frameworks.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Feature
+      - Description
+      - Since PyTorch
+      - Since ROCm
+    * - Device management
+      - Utilities for managing and interacting with GPUs.
+      - 0.4.0
+      - 3.8
+    * - Tensor operations on GPU
+      - Performs tensor operations such as addition and matrix multiplications on
+        the GPU.
+      - 0.4.0
+      - 3.8
+    * - Streams and events
+      - Streams allow overlapping computation and communication for optimized
+        performance. Events enable synchronization.
+      - 1.6.0
+      - 3.8
+    * - Memory management
+      - Functions to manage and inspect memory usage like
+        ``torch.cuda.memory_allocated()``, ``torch.cuda.max_memory_allocated()``,
+        ``torch.cuda.memory_reserved()`` and ``torch.cuda.empty_cache()``.
+      - 0.3.0
+      - 1.9.2
+    * - Running process lists of memory management
+      - Returns a human-readable printout of the running processes and their GPU
+        memory use for a given device with functions like
+        ``torch.cuda.memory_stats()`` and ``torch.cuda.memory_summary()``.
+      - 1.8.0
+      - 4.0
+    * - Communication collectives
+      - Set of APIs that enable efficient communication between multiple GPUs,
+        allowing for distributed computing and data parallelism.
+      - 1.9.0
+      - 5.0
+    * - ``torch.cuda.CUDAGraph``
+      - Graphs capture sequences of GPU operations to minimize kernel launch
+        overhead and improve performance.
+      - 1.10.0
+      - 5.3
+    * - TunableOp
+      - A mechanism that allows certain operations to be more flexible and
+        optimized for performance. It enables automatic tuning of kernel
+        configurations and other settings to achieve the best possible
+        performance based on the specific hardware (GPU) and workload.
+      - 2.0
+      - 5.4
+    * - NVIDIA Tools Extension (NVTX)
+      - Integration with NVTX for profiling and debugging GPU performance using
+        NVIDIA's Nsight tools.
+      - 1.8.0
+      - ❌
+    * - Lazy loading NVRTC
+      - Delays JIT compilation with NVRTC until the code is explicitly needed.
+      - 1.13.0
+      - ❌
+    * - Jiterator (beta)
+      - Jiterator allows asynchronous data streaming into computation streams
+        during training loops.
+      - 1.13.0
+      - 5.2
+
+.. Need to validate and extend.
+
+torch.backends.cuda
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``torch.backends.cuda`` is a PyTorch module that provides configuration options
+and flags to control the behavior of ROCm or CUDA operations. It is part of the
+PyTorch backend configuration system, which allows users to fine-tune how
+PyTorch interacts with the ROCm or CUDA environment.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Feature
+      - Description
+      - Since PyTorch
+      - Since ROCm
+    * - ``cufft_plan_cache``
+      - Manages caching of GPU FFT plans to optimize repeated FFT computations.
+      - 1.7.0
+      - 5.0
+    * - ``matmul.allow_tf32``
+      - Enables or disables the use of TensorFloat-32 (TF32) precision for
+        faster matrix multiplications on GPUs with Tensor Cores.
+      - 1.10.0
+      - ❌
+    * - ``matmul.allow_fp16_reduced_precision_reduction``
+      - Reduced precision reductions (e.g., with fp16 accumulation type) are
+        allowed with fp16 GEMMs.
+      - 2.0
+      - ❌
+    * - ``matmul.allow_bf16_reduced_precision_reduction``
+      - Reduced precision reductions are allowed with bf16 GEMMs.
+      - 2.0
+      - ❌
+    * - ``enable_cudnn_sdp``
+      - Globally enables cuDNN SDPA's kernels within SDPA.
+      - 2.0
+      - ❌
+    * - ``enable_flash_sdp``
+      - Globally enables or disables FlashAttention for SDPA.
+      - 2.1
+      - ❌
+    * - ``enable_mem_efficient_sdp``
+      - Globally enables or disables Memory-Efficient Attention for SDPA.
+      - 2.1
+      - ❌
+    * - ``enable_math_sdp``
+      - Globally enables or disables the PyTorch C++ implementation within SDPA.
+      - 2.1
+      - ❌
+
+.. Need to validate and extend.
+
+torch.backends.cudnn
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Supported ``torch`` options include:
+
+.. list-table::
+    :header-rows: 1
+
+    * - Option
+      - Description
+      - Since PyTorch
+      - Since ROCm
+    * - ``allow_tf32``
+      - TensorFloat-32 tensor cores may be used in cuDNN convolutions on NVIDIA
+        Ampere or newer GPUs.
+      - 1.12.0
+      - ❌
+    * - ``deterministic``
+      - A bool that, if True, causes cuDNN to only use deterministic
+        convolution algorithms.
+      - 1.12.0
+      - 6.0
+
+Automatic mixed precision: torch.amp
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+PyTorch that automates the process of using both 16-bit (half-precision,
+float16) and 32-bit (single-precision, float32) floating-point types in model
+training and inference.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Feature
+      - Description
+      - Since PyTorch
+      - Since ROCm
+    * - Autocasting
+      - Instances of autocast serve as context managers or decorators that allow
+        regions of your script to run in mixed precision.
+      - 1.9
+      - 2.5
+    * - Gradient scaling
+      - To prevent underflow, “gradient scaling” multiplies the network’s
+        loss(es) by a scale factor and invokes a backward pass on the scaled
+        loss(es). Gradients flowing backward through the network are then
+        scaled by the same factor. In other words, gradient values have a
+        larger magnitude, so they don’t flush to zero.
+      - 1.9
+      - 2.5
+    * - CUDA op-specific behavior
+      - These ops always go through autocasting whether they are invoked as part
+        of a ``torch.nn.Module``, as a function, or as a ``torch.Tensor`` method. If
+        functions are exposed in multiple namespaces, they go through
+        autocasting regardless of the namespace.
+      - 1.9
+      - 2.5
+
+Distributed library features
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The PyTorch distributed library includes a collective of parallelism modules, a
+communications layer, and infrastructure for launching and debugging large
+training jobs. See :ref:`rocm-for-ai-pytorch-distributed` for more information.
+
+The Distributed Library feature in PyTorch provides tools and APIs for building
+and running distributed machine learning workflows. It allows training models
+across multiple processes, GPUs, or nodes in a cluster, enabling efficient use
+of computational resources and scalability for large-scale tasks.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Feature
+      - Description
+      - Since PyTorch
+      - Since ROCm
+    * - TensorPipe
+      - A point-to-point communication library integrated into
+        PyTorch for distributed training. It is designed to handle tensor data
+        transfers efficiently between different processes or devices, including
+        those on separate machines.
+      - 1.8
+      - 5.4
+    * - Gloo
+      - Designed for multi-machine and multi-GPU setups, enabling
+        efficient communication and synchronization between processes. Gloo is
+        one of the default backends for PyTorch's Distributed Data Parallel
+        (DDP) and RPC frameworks, alongside other backends like NCCL and MPI.
+      - 1.0
+      - 2.0
+
+torch.compiler
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. list-table::
+    :header-rows: 1
+
+    * - Feature
+      - Description
+      - Since PyTorch
+      - Since ROCm
+    * - ``torch.compiler`` (AOT Autograd)
+      - Autograd captures not only the user-level code, but also backpropagation,
+        which results in capturing the backwards pass “ahead-of-time”. This
+        enables acceleration of both forwards and backwards pass using
+        ``TorchInductor``.
+      - 2.0
+      - 5.3
+    * - ``torch.compiler`` (TorchInductor)
+      - The default ``torch.compile`` deep learning compiler that generates fast
+        code for multiple accelerators and backends. You need to use a backend
+        compiler to make speedups through ``torch.compile`` possible. For AMD,
+        NVIDIA, and Intel GPUs, it leverages OpenAI Triton as the key building block.
+      - 2.0
+      - 5.3
+
+torchaudio
+--------------------------------------------------------------------------------
+
+The `torchaudio <https://pytorch.org/audio/stable/index.html>`_ library provides
+utilities for processing audio data in PyTorch, such as audio loading,
+transformations, and feature extraction.
+
+To ensure GPU-acceleration with ``torchaudio.transforms``, you need to move audio
+data (waveform tensor) explicitly to GPU using ``.to('cuda')``.
+
+The following ``torchaudio`` features are GPU-accelerated.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Feature
+      - Description
+      - Since torchaudio version
+      - Since ROCm
+    * - ``torchaudio.transforms.Spectrogram``
+      - Generates spectrogram of an input waveform using STFT.
+      - 0.6.0
+      - 4.5
+    * - ``torchaudio.transforms.MelSpectrogram``
+      - Generates the mel-scale spectrogram of raw audio signals.
+      - 0.9.0
+      - 4.5
+    * - ``torchaudio.transforms.MFCC``
+      - Extract of MFCC features.
+      - 0.9.0
+      - 4.5
+    * - ``torchaudio.transforms.Resample``
+      - Resamples a signal from one frequency to another.
+      - 0.9.0
+      - 4.5
+
+torchvision
+--------------------------------------------------------------------------------
+
+The `torchvision <https://pytorch.org/vision/stable/index.html>`_ library
+provide datasets, model architectures, and common image transformations for
+computer vision.
+
+The following ``torchvision`` features are GPU-accelerated.
+
+.. list-table::
+    :header-rows: 1
+
+    * - Feature
+      - Description
+      - Since torchvision version
+      - Since ROCm
+    * - ``torchvision.transforms.functional``
+      - Provides GPU-compatible transformations for image preprocessing like
+        resize, normalize, rotate and crop.
+      - 0.2.0
+      - 4.0
+    * - ``torchvision.ops``
+      - GPU-accelerated operations for object detection and segmentation tasks.
+        ``torchvision.ops.roi_align``, ``torchvision.ops.nms`` and
+        ``box_convert``.
+      - 0.6.0
+      - 3.3
+    * - ``torchvision.models`` with ``.to('cuda')``
+      - ``torchvision`` provides several pre-trained models (ResNet, Faster
+        R-CNN, Mask R-CNN, ...) that can run on CUDA for faster inference and
+        training.
+      - 0.1.6
+      - 2.x
+    * - ``torchvision.io``
+      - Enables video decoding and frame extraction using GPU acceleration with NVIDIA’s
+        NVDEC and nvJPEG (rocJPEG) on CUDA-enabled GPUs.
+      - 0.4.0
+      - 6.3
+
+torchtext
+--------------------------------------------------------------------------------
+
+The `torchtext <https://pytorch.org/text/stable/index.html>`_ library provides
+utilities for processing and working with text data in PyTorch, including
+tokenization, vocabulary management, and text embeddings. torchtext supports
+preprocessing pipelines and integration with PyTorch models, simplifying the
+implementation of natural language processing (NLP) tasks.
+
+To leverage GPU acceleration in torchtext, you need to move tensors
+explicitly to the GPU using ``.to('cuda')``.
+
+* torchtext does not implement its own kernels. ROCm support is enabled by linking against ROCm libraries.
+
+* Only official release exists.
+
+torchtune
+--------------------------------------------------------------------------------
+
+The `torchtune <https://pytorch.org/torchtune/stable/index.html>`_ library for
+authoring, fine-tuning and experimenting with LLMs.
+
+* Usage: It works out-of-the-box, enabling developers to fine-tune ROCm PyTorch solutions.
+
+* Only official release exists.
+
+torchserve
+--------------------------------------------------------------------------------
+
+The `torchserve <https://pytorch.org/torchserve/>`_ is a PyTorch domain library
+for common sparsity and parallelism primitives needed for large-scale recommender
+systems.
+
+* torchtext does not implement its own kernels. ROCm support is enabled by linking against ROCm libraries.
+
+* Only official release exists.
+
+torchrec
+--------------------------------------------------------------------------------
+
+The `torchrec <https://pytorch.org/torchrec/>`_ is a PyTorch domain library for
+common sparsity and parallelism primitives needed for large-scale recommender
+systems.
+
+* torchrec does not implement its own kernels. ROCm support is enabled by linking against ROCm libraries.
+
+* Only official release exists.
+
+Unsupported PyTorch features
+----------------------------
+
+The following are GPU-accelerated PyTorch features not currently supported by ROCm.
+
+.. list-table::
+    :widths: 30, 60, 10
+    :header-rows: 1
+
+    * - Feature
+      - Description
+      - Since PyTorch
+    * - APEX batch norm
+      - Use APEX batch norm instead of PyTorch batch norm.
+      - 1.6.0
+    * - ``torch.backends.cuda`` / ``matmul.allow_tf32``
+      - A bool that controls whether TensorFloat-32 tensor cores may be used in
+        matrix multiplications.
+      - 1.7
+    * - ``torch.cuda`` / NVIDIA Tools Extension (NVTX)
+      - Integration with NVTX for profiling and debugging GPU performance using
+        NVIDIA's Nsight tools.
+      - 1.7.0
+    * - ``torch.cuda`` / Lazy loading NVRTC
+      - Delays JIT compilation with NVRTC until the code is explicitly needed.
+      - 1.8.0
+    * - ``torch-tensorrt``
+      - Integrate TensorRT library for optimizing and deploying PyTorch models.
+        ROCm does not have equialent library for TensorRT.
+      - 1.9.0
+    * - ``torch.backends`` / ``cudnn.allow_tf32``
+      - TensorFloat-32 tensor cores may be used in cuDNN convolutions.
+      - 1.10.0
+    * - ``torch.backends.cuda`` / ``matmul.allow_fp16_reduced_precision_reduction``
+      - Reduced precision reductions with fp16 accumulation type are
+        allowed with fp16 GEMMs.
+      - 2.0
+    * - ``torch.backends.cuda`` / ``matmul.allow_bf16_reduced_precision_reduction``
+      - Reduced precision reductions are allowed with bf16 GEMMs.
+      - 2.0
+    * - ``torch.nn.functional`` / ``scaled_dot_product_attention`` 
+      - Flash attention backend for SDPA to accelerate attention computation in
+        transformer-based models.
+      - 2.0
+    * - ``torch.backends.cuda`` / ``enable_cudnn_sdp``
+      - Globally enables cuDNN SDPA's kernels within SDPA.
+      - 2.0
+    * - ``torch.backends.cuda`` / ``enable_flash_sdp``
+      - Globally enables or disables FlashAttention for SDPA.
+      - 2.1
+    * - ``torch.backends.cuda`` / ``enable_mem_efficient_sdp``
+      - Globally enables or disables Memory-Efficient Attention for SDPA.
+      - 2.1
+    * - ``torch.backends.cuda`` / ``enable_math_sdp``
+      - Globally enables or disables the PyTorch C++ implementation within SDPA.
+      - 2.1
+    * - Dynamic parallelism
+      - PyTorch itself does not directly expose dynamic parallelism as a core
+        feature. Dynamic parallelism allow GPU threads to launch additional
+        threads which can be reached using custom operations via the
+        ``torch.utils.cpp_extension`` module.
+      - Not a core feature
+    * - Unified memory support in PyTorch
+      - Unified Memory is not directly exposed in PyTorch's core API, it can be
+        utilized effectively through custom CUDA extensions or advanced
+        workflows.
+      - Not a core feature
+
+Use cases and recommendations
+================================================================================
+
+* :doc:`Using ROCm for AI: training a model </how-to/rocm-for-ai/training/train-a-model>` provides
+  guidance on how to leverage the ROCm platform for training AI models. It covers the steps, tools, and best practices
+  for optimizing training workflows on AMD GPUs using PyTorch features.
+
+* :doc:`Single-GPU fine-tuning and inference </how-to/rocm-for-ai/fine-tuning/single-gpu-fine-tuning-and-inference>`
+  describes and demonstrates how to use the ROCm platform for the fine-tuning and inference of
+  machine learning models, particularly large language models (LLMs), on systems with a single AMD
+  Instinct MI300X accelerator. This page provides a detailed guide for setting up, optimizing, and
+  executing fine-tuning and inference workflows in such environments.
+
+* :doc:`Multi-GPU fine-tuning and inference optimization </how-to/rocm-for-ai/fine-tuning/multi-gpu-fine-tuning-and-inference>`
+  describes and demonstrates the fine-tuning and inference of machine learning models on systems
+  with multi MI300X accelerators.
+
+* The :doc:`Instinct MI300X workload optimization guide </how-to/rocm-for-ai/inference-optimization/workload>` provides detailed
+  guidance on optimizing workloads for the AMD Instinct MI300X accelerator using ROCm. This guide is aimed at helping
+  users achieve optimal performance for deep learning and other high-performance computing tasks on the MI300X
+  accelerator.
+
+* The :doc:`Inception with PyTorch documentation </conceptual/ai-pytorch-inception>`
+  describes how PyTorch integrates with ROCm for AI workloads It outlines the use of PyTorch on the ROCm platform and
+  focuses on how to efficiently leverage AMD GPU hardware for training and inference tasks in AI applications.
+
+For more use cases and recommendations, see `ROCm PyTorch blog posts <https://rocm.blogs.amd.com/blog/tag/pytorch.html>`_.