WIP

Add quark in model-quantization.rst

.azuredevops/components/AMDMIGraphX.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -27,12 +32,12 @@ parameters:
   type: object
   default:
     - https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
-    - onnx>=1.14.1
-    - numpy>=1.21.6
-    - typing>=3.7.4
-    - pytest>=6.0.1
-    - packaging>=23.0
-    - protobuf>=3.20.2
+    - "onnx>=1.14.1"
+    - "numpy>=1.21.6"
+    - "typing>=3.7.4"
+    - "pytest>=6.0.1"
+    - "packaging>=23.0"
+    - "protobuf>=3.20.2"
 - name: rocmDependencies
   type: object
   default:
@@ -115,6 +120,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -145,7 +151,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/HIP.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -42,7 +47,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -67,6 +72,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependenciesAMD }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
 # compile clr
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -99,7 +105,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -125,6 +131,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependenciesNvidia }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - script: 'ls -1R $(Agent.BuildDirectory)/rocm'
     displayName: 'Artifact listing'
 # compile clr

.azuredevops/components/HIPIFY.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:

.azuredevops/components/MIOpen.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -77,7 +82,7 @@ jobs:
     - template: /.azuredevops/variables-global.yml
     - name: ROCM_PATH
       value: $(Agent.BuildDirectory)/rocm
-    pool: ${{ variables.MEDIUM_BUILD_POOL }}
+    pool: ${{ variables.HIGH_BUILD_POOL }}
     workspace:
       clean: all
     steps:
@@ -97,6 +102,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - task: Bash@3
       displayName: Build and install other dependencies
       inputs:
@@ -107,7 +113,7 @@ jobs:
           mkdir -p $(Agent.BuildDirectory)/miopen-deps
           export CXX=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
           export CC=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang
-          cmake -P install_deps.cmake --prefix $(Agent.BuildDirectory)/miopen-deps
+          cmake -P install_deps.cmake --prefix $(Agent.BuildDirectory)/miopen-deps --generator Ninja
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -142,7 +148,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
@@ -207,7 +214,7 @@ jobs:
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
       parameters:
         componentName: MIOpen
-        testParameters: '--output-on-failure --force-new-ctest-process --output-junit test_output.xml --exclude-regex test_rnn_seq_api'
+        testParameters: '--output-on-failure --force-new-ctest-process --output-junit test_output.xml --exclude-regex "test_rnn_seq_api|GPU_Conv2dTuningAsm_FP32"'
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
       parameters:
         aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/MIVisionX.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -103,6 +108,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -129,7 +135,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/ROCR-Runtime.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -42,7 +47,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -58,6 +63,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -78,7 +84,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/ROCdbgapi.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -24,7 +29,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -40,6 +45,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-

.azuredevops/components/ROCgdb.yml

@@ -5,11 +5,17 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 # reference: https://github.com/ROCm/ROCgdb/blob/amd-staging/README-ROCM.md
 - name: aptPackages
   type: object
   default:
     - bison
+    - cmake
     - dejagnu
     - flex
     - libbabeltrace-dev
@@ -34,19 +40,72 @@ parameters:
 - name: jobMatrix
   type: object
   default:
-    buildTestJobs:
+    testJobs:
       - gfx942:
         target: gfx942
       - gfx90a:
         target: gfx90a
 
 jobs:
-- ${{ each job in parameters.jobMatrix.buildTestJobs }}:
-  - job: ROCgdb_build_test_${{ job.target }}
+- job: ROCgdb
+  variables:
+  - group: common
+  - template: /.azuredevops/variables-global.yml
+  - name: PKG_CONFIG_PATH
+    value: $(Agent.BuildDirectory)/rocm/share/pkgconfig
+  pool:
+    vmImage: ${{ variables.BASE_BUILD_POOL }}
+  workspace:
+    clean: all
+  steps:
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
+    parameters:
+      aptPackages: ${{ parameters.aptPackages }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
+    parameters:
+      checkoutRepo: ${{ parameters.checkoutRepo }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
+    parameters:
+      checkoutRef: ${{ parameters.checkoutRef }}
+      dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-autotools.yml
+    parameters:
+      configureFlags: >-
+        --program-prefix=roc
+        --enable-64-bit-bfd
+        --enable-targets="x86_64-linux-gnu,amdgcn-amd-amdhsa"
+        --disable-ld
+        --disable-gas
+        --disable-gdbserver
+        --disable-sim
+        --enable-tui
+        --disable-gdbtk
+        --disable-shared
+        --disable-gprofng
+        --with-expat
+        --with-system-zlib
+        --without-guile
+        --with-babeltrace
+        --with-lzma
+        --with-python=python3
+        --with-rocm-dbgapi=$(Agent.BuildDirectory)/rocm
+        LDFLAGS="-Wl,--enable-new-dtags,-rpath=$(Agent.BuildDirectory)/rocm/lib"
+      makeCallPrefix: LD_RUN_PATH='${ORIGIN}/../lib'
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
+
+- ${{ each job in parameters.jobMatrix.testJobs }}:
+  - job: ROCgdb_test_${{ job.target }}
+    dependsOn: ROCgdb
     condition:
-      and(
+      and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
@@ -69,6 +128,7 @@ jobs:
       parameters:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-autotools.yml
       parameters:
         configureFlags: >-
@@ -92,8 +152,6 @@ jobs:
           --with-rocm-dbgapi=$(Agent.BuildDirectory)/rocm
           LDFLAGS="-Wl,--enable-new-dtags,-rpath=$(Agent.BuildDirectory)/rocm/lib"
         makeCallPrefix: LD_RUN_PATH='${ORIGIN}/../lib'
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
     - task: Bash@3
       displayName: Setup test environment
       inputs:
@@ -102,7 +160,6 @@ jobs:
           # Assuming that /opt is no longer persistent across runs, test environments are fully ephemeral
           sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm
           echo "##vso[task.prependpath]/opt/rocm/bin"
-    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
     - task: Bash@3
       displayName: check-gdb

.azuredevops/components/ROCmValidationSuite.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -22,6 +27,7 @@ parameters:
   type: object
   default:
     - amdsmi
+    - aomp
     - clr
     - hipBLAS-common
     - hipBLASLt
@@ -38,6 +44,7 @@ parameters:
   type: object
   default:
     - amdsmi
+    - aomp
     - clr
     - hipBLAS-common
     - hipBLASLt
@@ -96,12 +103,14 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
           -DROCM_PATH=$(Agent.BuildDirectory)/rocm
           -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/clang++
           -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
+          -DCMAKE_CXX_FLAGS=-I$(Agent.BuildDirectory)/rocm/llvm/include
           -DCPACK_PACKAGING_INSTALL_PREFIX=$(Build.BinariesDirectory)
           -GNinja
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
@@ -126,7 +135,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/Tensile.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -46,7 +51,7 @@ jobs:
   - template: /.azuredevops/variables-global.yml
   - name: ROCM_PATH
     value: $(Agent.BuildDirectory)/rocm
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -63,6 +68,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - task: Bash@3
     displayName: Create wheel file
@@ -104,7 +110,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/TransferBench.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -67,6 +72,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -92,7 +98,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/amdsmi.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -58,7 +63,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/aomp-extras.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -23,7 +28,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -39,6 +44,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: aomp-extras

.azuredevops/components/aomp.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 # reference:
 # https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL_PREREQUISITE.md
 - name: aptPackages
@@ -108,6 +113,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: extras
@@ -176,7 +182,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/composable_kernel.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -71,6 +76,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - script: |
         mkdir -p $(CCACHE_DIR)
         echo "##vso[task.prependpath]/usr/lib/ccache"
@@ -117,7 +123,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/copyHIP.yml

@@ -26,9 +26,11 @@ jobs:
     parameters:
       componentName: HIP
       pipelineId: $(HIP_PIPELINE_ID)
-  - template:  ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-prepare-package.yml
-    parameters:
-      sourceDir: $(Agent.BuildDirectory)/rocm
+  - task: Bash@3
+    displayName: Copy HIP artifacts
+    inputs:
+      targetType: inline
+      script: cp -a $(Agent.BuildDirectory)/rocm/* $(Build.BinariesDirectory)/
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml

.azuredevops/components/half.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -25,7 +30,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -41,6 +46,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-

.azuredevops/components/hip-tests.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -75,6 +80,7 @@ jobs:
       parameters:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     # compile hip-tests
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
@@ -109,7 +115,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipBLAS-common.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -29,7 +34,7 @@ jobs:
   - name: ROCM_PATH
     value: $(Agent.BuildDirectory)/rocm
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -45,6 +50,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-

.azuredevops/components/hipBLAS.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -89,6 +94,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -121,13 +127,12 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
-    - name: AMD_COMGR_CACHE
-      value: 0
     pool: ${{ job.target }}_test_pool
     workspace:
       clean: all

.azuredevops/components/hipBLASLt.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -22,7 +27,7 @@ parameters:
   type: object
   default:
     - joblib
-    - packaging>=22.0
+    - "packaging>=22.0"
     - --upgrade
 - name: rocmDependencies
   type: object
@@ -98,6 +103,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - task: Bash@3
       displayName: Add ROCm binaries to PATH
       inputs:
@@ -179,15 +185,14 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
     - name: ROCM_PATH
       value: $(Agent.BuildDirectory)/rocm
-    - name: AMD_COMGR_CACHE
-      value: 0
     pool: ${{ job.target }}_test_pool
     workspace:
       clean: all

.azuredevops/components/hipCUB.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -67,6 +72,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -94,7 +100,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipFFT.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -79,6 +84,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -112,7 +118,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipRAND.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -70,6 +75,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -101,7 +107,8 @@ jobs:
     condition:
         and(succeeded(),
           eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-          not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+          not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+          eq(${{ parameters.aggregatePipeline }}, False)
         )
     variables:
     - group: common

.azuredevops/components/hipSOLVER.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -82,6 +87,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
   # build external gtest and lapack
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
@@ -122,13 +128,12 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
-    - name: AMD_COMGR_CACHE
-      value: 0
     pool: ${{ job.target }}_test_pool
     workspace:
       clean: all

.azuredevops/components/hipSPARSE.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -77,6 +82,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -116,13 +122,12 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
-    - name: AMD_COMGR_CACHE
-      value: 0
     pool: ${{ job.target }}_test_pool
     workspace:
       clean: all

.azuredevops/components/hipSPARSELt.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -91,6 +96,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
   # Build and install gtest and lapack
   # $(Pipeline.Workspace)/deps is a temporary folder for the build process
   # $(Pipeline.Workspace)/s/deps is part of the hipSPARSELt repo
@@ -150,7 +156,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipTensor.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -66,6 +71,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -95,13 +101,12 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
-    - name: AMD_COMGR_CACHE
-      value: 0
     pool: ${{ job.target }}_test_pool
     workspace:
       clean: all

.azuredevops/components/hipfort.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -76,6 +81,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -111,7 +117,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/llvm-project.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -45,6 +50,7 @@ jobs:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       skipLlvmSymlink: true
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: rocm-llvm

.azuredevops/components/rccl.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -72,7 +77,7 @@ jobs:
     - template: /.azuredevops/variables-global.yml
     - name: HIP_ROCCLR_HOME
       value: $(Build.BinariesDirectory)/rocm
-    pool: ${{ variables.MEDIUM_BUILD_POOL }}
+    pool: ${{ variables.HIGH_BUILD_POOL }}
     workspace:
       clean: all
     steps:
@@ -90,6 +95,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -124,7 +130,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rdc.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -89,6 +94,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
   # Build grpc
     - task: Bash@3
       displayName: 'git clone grpc'
@@ -135,7 +141,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocAL.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -151,6 +156,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -182,7 +188,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocALUTION.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -85,6 +90,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -117,7 +123,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocBLAS.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -101,6 +106,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -145,13 +151,12 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
-    - name: AMD_COMGR_CACHE
-      value: 0
     pool: ${{ job.target }}_test_pool
     workspace:
       clean: all

.azuredevops/components/rocDecode.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -73,6 +78,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -92,7 +98,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocFFT.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -79,6 +84,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -113,7 +119,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocJPEG.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -75,6 +80,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -83,6 +89,8 @@ jobs:
           -GNinja
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
     # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
     #   parameters:
@@ -96,7 +104,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
@@ -115,6 +124,8 @@ jobs:
         registerROCmPackages: true
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
+      parameters:
+        gpuTarget: ${{ job.target }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
       parameters:
@@ -140,4 +151,3 @@ jobs:
         environment: test
         gpuTarget: ${{ job.target }}
         registerROCmPackages: true
-        optSymLink: true

.azuredevops/components/rocMLIR.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -62,6 +67,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -87,7 +93,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocPRIM.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -66,6 +71,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -93,7 +99,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocPyDecode.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -77,6 +82,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - task: Bash@3
       displayName: 'Save Python Package Paths'
       inputs:
@@ -152,7 +158,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocRAND.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -70,6 +75,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -98,7 +104,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocSOLVER.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -88,6 +93,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         componentName: lapack
@@ -131,13 +137,12 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
-    - name: AMD_COMGR_CACHE
-      value: 0
     pool: ${{ job.target }}_test_pool
     workspace:
       clean: all

.azuredevops/components/rocSPARSE.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -29,6 +34,7 @@ parameters:
     - rocminfo
     - rocPRIM
     - rocprofiler-register
+    - roctracer
 - name: rocmTestDependencies
   type: object
   default:
@@ -80,6 +86,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -126,13 +133,12 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
-    - name: AMD_COMGR_CACHE
-      value: 0
     pool: ${{ job.target }}_test_pool
     workspace:
       clean: all

.azuredevops/components/rocThrust.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -71,6 +76,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -98,7 +104,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocWMMA.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -65,7 +70,7 @@ jobs:
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
-    pool: ${{ variables.MEDIUM_BUILD_POOL }}
+    pool: ${{ variables.HIGH_BUILD_POOL }}
     workspace:
       clean: all
     steps:
@@ -81,6 +86,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -113,13 +119,12 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
-    - name: AMD_COMGR_CACHE
-      value: 0
     pool: ${{ job.target }}_test_pool
     workspace:
       clean: all

.azuredevops/components/rocm-cmake.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -26,7 +31,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/components/rocm-core.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -17,7 +22,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/components/rocm-examples.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -100,6 +105,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         # https://github.com/ROCm/HIP/issues/2203
@@ -136,7 +142,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
@@ -176,6 +183,7 @@ jobs:
       parameters:
         componentName: rocm-examples
         testDir: $(Build.SourcesDirectory)/build
+        testParameters: '--output-on-failure --force-new-ctest-process --output-junit test_output.xml --exclude-regex "rocfft_callback"'
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
       parameters:
         aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/rocm_bandwidth_test.yml

@@ -5,6 +5,12 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
+
 - name: aptPackages
   type: object
   default:
@@ -49,7 +55,7 @@ jobs:
     value: $(Agent.BuildDirectory)/rocm
   - name: ROCR_LIB_DIR
     value: $(Agent.BuildDirectory)/rocm
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -66,6 +72,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -90,7 +97,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocm_smi_lib.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -28,7 +33,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -59,7 +64,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocminfo.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -38,7 +43,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -55,6 +60,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
       skipLlvmSymlink: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -72,7 +78,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocprofiler-compute.yml

@@ -5,10 +5,17 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
     - cmake
+    - libdw-dev
+    - libtbb-dev
     - locales
     - ninja-build
     - python3-pip
@@ -17,10 +24,10 @@ parameters:
   default:
     - astunparse==1.6.2
     - colorlover
-    - dash>=1.12.0
+    - "dash>=1.12.0"
     - matplotlib
-    - numpy>=1.17.5
-    - pandas>=1.4.3
+    - "numpy>=1.17.5"
+    - "pandas>=1.4.3"
     - pymongo
     - pyyaml
     - tabulate
@@ -114,6 +121,7 @@ jobs:
         dependencyList: ${{ parameters.rocmDependencies }}
         dependencySource: ${{ job.dependencySource }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -140,7 +148,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
@@ -182,12 +191,9 @@ jobs:
       displayName: Add ROCm binaries to PATH
       inputs:
         targetType: inline
-        script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
-    - task: Bash@3
-      displayName: Add ROCm compilers to PATH
-      inputs:
-        targetType: inline
-        script: echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
+        script: |
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/bin"
+          echo "##vso[task.prependpath]$(Agent.BuildDirectory)/rocm/llvm/bin"
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -206,18 +212,6 @@ jobs:
         componentName: rocprofiler-compute
         testDir: $(Build.BinariesDirectory)/libexec/rocprofiler-compute
         testExecutable: ROCM_PATH=$(Agent.BuildDirectory)/rocm ctest
-    - task: Bash@3
-      displayName: Remove ROCm binaries from PATH
-      condition: always()
-      inputs:
-        targetType: inline
-        script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/bin;;' -e 's;^/;;' -e 's;/$;;')"
-    - task: Bash@3
-      displayName: Remove ROCm compilers from PATH
-      condition: always()
-      inputs:
-        targetType: inline
-        script: echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$(Agent.BuildDirectory)/rocm/llvm/bin;;' -e 's;^/;;' -e 's;/$;;')"
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
       parameters:
         aptPackages: ${{ parameters.aptPackages }}

.azuredevops/components/rocprofiler-register.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -17,7 +22,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -32,20 +37,14 @@ jobs:
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: rocprofiler-register
-      extraBuildFlags: >-
-        -GNinja
-  - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
-    parameters:
-      componentName: rocprofiler-register-tests
       extraBuildFlags: >-
         -DCMAKE_PREFIX_PATH=$(Build.BinariesDirectory)
+        -DROCPROFILER_REGISTER_BUILD_TESTS=ON
+        -DROCPROFILER_REGISTER_BUILD_SAMPLES=ON
         -GNinja
-      cmakeBuildDir: 'tests/build'
-      installEnabled: false
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
     parameters:
       componentName: rocprofiler-register
-      testDir: 'tests/build'
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml

.azuredevops/components/rocprofiler-sdk.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -31,7 +36,7 @@ parameters:
     - pandas
     - perfetto
     - pycobertura
-    - pytest>=6.2.5
+    - "pytest>=6.2.5"
     - pyyaml
 - name: rocmDependencies
   type: object
@@ -89,6 +94,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - task: Bash@3
       displayName: Add Python site-packages binaries to path
       inputs:
@@ -125,7 +131,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocprofiler-systems.yml

@@ -6,6 +6,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -16,6 +21,7 @@ parameters:
     - bzip2
     - clang
     - cmake
+    - chrpath
     - environment-modules
     - ffmpeg
     - g++-12
@@ -61,6 +67,7 @@ parameters:
     - rocprofiler-register
     - rocprofiler-sdk
     - ROCR-Runtime
+    - roctracer
 
 - name: jobMatrix
   type: object
@@ -103,6 +110,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - task: Bash@3
       displayName: Add ROCm binaries to PATH
       inputs:
@@ -124,6 +132,7 @@ jobs:
           -DDYNINST_BUILD_BOOST=ON
           -DROCPROFSYS_USE_PAPI=ON
           -DROCPROFSYS_USE_MPI=ON
+          -DCMAKE_CXX_FLAGS=-I$(Agent.BuildDirectory)/rocm/include/rocjpeg
           -DGPU_TARGETS=${{ job.target }}
           -GNinja
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
@@ -147,7 +156,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     timeoutInMinutes: 180
     variables:
@@ -197,6 +207,7 @@ jobs:
           -DDYNINST_BUILD_BOOST=ON
           -DROCPROFSYS_USE_PAPI=ON
           -DROCPROFSYS_USE_MPI=ON
+          -DCMAKE_CXX_FLAGS=-I$(Agent.BuildDirectory)/rocm/include/rocjpeg
           -DGPU_TARGETS=${{ job.target }}
           -GNinja
     - task: Bash@3

.azuredevops/components/rocprofiler.yml

@@ -5,6 +5,12 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
+
 - name: aptPackages
   type: object
   default:
@@ -83,6 +89,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -115,7 +122,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocr_debug_agent.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -49,7 +54,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -65,6 +70,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -86,7 +92,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/roctracer.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -77,6 +82,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -107,7 +113,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rpp.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -84,6 +89,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -113,7 +119,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/nightly/pytorch.yml

@@ -68,7 +68,7 @@ parameters:
   default:
     - cmake
     - astunparse
-    - expecttest>=0.2.1
+    - "expecttest>=0.2.1"
     - hypothesis
     - numpy
     - psutil
@@ -76,8 +76,8 @@ parameters:
     - requests
     - setuptools==75.8.0
     - types-dataclasses
-    - typing-extensions>=4.8.0
-    - sympy>=1.13.0
+    - "typing-extensions>=4.8.0"
+    - "sympy>=1.13.0"
     - filelock
     - networkx
     - jinja2
@@ -85,8 +85,8 @@ parameters:
     - lintrunner
     - ninja
     - packaging
-    - optree>=0.13.0
-    - click>=8.0.3
+    - "optree>=0.13.0"
+    - "click>=8.0.3"
   # list for vision
     - auditwheel
     - future

.azuredevops/scripts/download-manifest-artifacts/download-manifest-artifacts.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+import json
+import requests
+import argparse
+from pathlib import Path
+
+def get_builds(entries, gpu_target, output):
+    already_downloaded = {}
+    for entry in entries:
+        already_downloaded = _get_builds(entry, gpu_target, already_downloaded, output)
+
+def _get_builds(entry, gpu_target, already_downloaded, output):
+    print()
+    print(f"{entry['buildNumber']} - {entry['buildId']} - {entry['repoName']}")
+    if already_downloaded.get(entry['buildId']):
+        print('Skipping, already downloaded from build ' + entry['buildId'])
+        return already_downloaded
+
+    artifacts_url = f"https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds/{entry['buildId']}/artifacts?api-version=7.1"
+    artifacts = requests.get(artifacts_url).json()
+    for artifact in artifacts['value']:
+        if 'gfx' in artifact['name'] and gpu_target not in artifact['name']:
+            continue
+
+        print('Artifact name: ' + artifact['name'])
+        print('File size: ~' +
+              str(round(int(artifact['resource']['properties']['artifactsize'])/1000000, 2)) + ' MB')
+        download_url = f"{artifact['resource']['downloadUrl']}"
+        download = requests.get(download_url)
+
+        zip_file = Path(output) / f"{artifact['name']}.zip"
+        with open(zip_file, 'wb') as f:
+            f.write(download.content)
+        already_downloaded[entry['buildId']] = True
+
+    return already_downloaded
+
+def main():
+    parser = argparse.ArgumentParser(description="Command line tool for downloading external ci artifacts")
+    parser.add_argument('--target', type=str, dest="target", choices=["gfx90a", "gfx942"], help="Target gfx")
+    parser.add_argument('--manifest', type=str, dest="manifest", help='JSON manifest url or path to local manifest')
+    parser.add_argument('--output_dir', type=str, dest="output", help='Path to download directory')
+    args = parser.parse_args()
+
+    manifest = args.manifest
+    gpu_target = args.target
+
+    if not gpu_target:
+        print("Enter the GPU target (gfx942, gfx90a)")
+        gpu_target = input()
+
+    if not manifest:
+        print("Enter the manifest file (URL or local path)")
+        manifest = input()
+
+    if 'http' in manifest:
+        data = requests.get(manifest).json()
+    else:
+        with open(manifest, 'r') as f:
+            data = json.load(f)
+
+    entries = [e for e in data['current']]
+    entries.extend([e for e in data['dependencies']])
+    get_builds(entries, gpu_target, args.output)
+
+if __name__ == "__main__":
+    main()

.azuredevops/scripts/download-manifest-artifacts/requirements.txt

@@ -0,0 +1 @@
+requests

.azuredevops/templates/steps/artifact-download.yml

@@ -12,6 +12,11 @@ parameters:
 - name: fileFilter
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 
 steps:
 - task: Bash@3
@@ -27,17 +32,23 @@ steps:
 - task: DownloadPipelineArtifact@2
   displayName: Download ${{ parameters.componentName }}
   inputs:
-    buildType: 'specific'
-    project: ROCm-CI
-    definition: ${{ parameters.pipelineId }}
-    specificBuildWithTriggering: true
-    itemPattern: '**/*${{ parameters.fileFilter }}*'
-    # aomp is a special case, since the trigger file is under ROCm/ROCm instead of the component repo
-    ${{ if notIn(parameters.componentName, 'aomp') }}:
-      buildVersionToDownload: latestFromBranch # default is 'latest'
-    branchName: refs/heads/${{ parameters.branchName }}
-    allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
-    targetPath: '$(Pipeline.Workspace)/d'
+    ${{ if eq(parameters.aggregatePipeline, false) }}:
+      buildType: 'specific'
+      project: ROCm-CI
+      definition: ${{ parameters.pipelineId }}
+      specificBuildWithTriggering: true
+      itemPattern: '**/*${{ parameters.fileFilter }}*'
+      # aomp is a special case, since the trigger file is under ROCm/ROCm instead of the component repo
+      ${{ if notIn(parameters.componentName, 'aomp') }}:
+        buildVersionToDownload: latestFromBranch # default is 'latest'
+      branchName: refs/heads/${{ parameters.branchName }}
+      allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
+      targetPath: '$(Pipeline.Workspace)/d'
+    ${{ else }}:
+      buildType: 'current'
+      itemPattern: '**/${{ parameters.componentName }}*${{ parameters.fileFilter }}*'
+      allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
+      targetPath: '$(Pipeline.Workspace)/d'
 - task: ExtractFiles@1
   displayName: Extract ${{ parameters.componentName }}
   inputs:

.azuredevops/templates/steps/dependencies-rocm.yml

@@ -31,6 +31,11 @@ parameters:
 - name: setupHIPLibrarySymlinks
   type: boolean
   default: false
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 
 - name: componentVarList
   type: object
@@ -354,6 +359,7 @@ steps:
       parameters:
         componentName: ${{ split(dependency, ':')[0] }}
         pipelineId: ${{ parameters.componentVarList[split(dependency, ':')[0]].pipelineId }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
         ${{ if parameters.componentVarList[split(dependency, ':')[0]].hasGpuTarget }}:
           fileFilter: "${{ split(dependency, ':')[1] }}*${{ parameters.gpuTarget }}"
         # dependencySource = staging
@@ -383,6 +389,7 @@ steps:
       parameters:
         componentName: ${{ dependency }}
         pipelineId: ${{ parameters.componentVarList[dependency].pipelineId }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
         ${{ if parameters.componentVarList[dependency].hasGpuTarget }}:
           fileFilter: ${{ parameters.gpuTarget }}
         # dependencySource = staging
@@ -456,7 +463,7 @@ steps:
   displayName: 'List downloaded ROCm files'
   inputs:
     targetType: inline
-    script: ls -1R $(Agent.BuildDirectory)/rocm
+    script: ls -la1R $(Agent.BuildDirectory)/rocm
 - ${{ if eq(parameters.skipLibraryLinking, false) }}:
   - task: Bash@3
     displayName: 'Link ROCm shared libraries'

.azuredevops/templates/steps/docker-container-commit.yml

@@ -0,0 +1,82 @@
+# This template creates and uploads a Docker image from the current environment
+# It uses `docker commit` to do so, which can provide more accurate images than the standard template
+# It requires the following conditions:
+# - Job must be run inside a Docker container
+# - The container's external name must be the same as the container's internal hostname
+# - Docker must be installed inside said container and given sufficient permissions
+# Currently, it is only usable for test jobs run on our self-managed systems
+# Jobs run on Azure VMs will not be able to use this template (most if not all build jobs)
+
+parameters:
+- name: gpuTarget
+  type: string
+  default: ''
+- name: environment
+  type: string
+  default: build
+  values:
+    - build
+    - test
+    - combined
+    - amd
+    - nvidia
+- name: extraPaths
+  type: string
+  default: ''
+- name: extraEnvVars
+  type: object
+  default: []
+- name: forceDockerCreation
+  type: boolean
+  default: false
+
+steps:
+  - task: Bash@3
+    displayName: Commit container and initialize Dockerfile
+    condition: or(and(failed(), not(contains(variables['DOCKER_SKIP_GFX'], variables['JOB_GPU_TARGET']))), ${{ eq(parameters.forceDockerCreation, true) }})
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: |
+        docker commit $(hostname) az-ci-temp-image:latest
+        echo "FROM az-ci-temp-image:latest" > Dockerfile
+        echo "RUN sudo groupmod -g $(getent group render | awk -F: '{print $3}') render" >> Dockerfile
+        echo "RUN sudo groupmod -g $(getent group docker | awk -F: '{print $3}') docker" >> Dockerfile
+        echo "ENV PATH='$PATH:${{ parameters.extraPaths }}'" >> Dockerfile
+        echo "ENTRYPOINT [\"/bin/bash\"]" >> Dockerfile
+  - ${{ each extraEnvVar in parameters.extraEnvVars }}:
+    - task: Bash@3
+      displayName: Add extra environment variables
+      condition: or(and(failed(), not(contains(variables['DOCKER_SKIP_GFX'], variables['JOB_GPU_TARGET']))), ${{ eq(parameters.forceDockerCreation, true) }})
+      inputs:
+        workingDirectory: $(Pipeline.Workspace)
+        targetType: inline
+        script: echo "ENV ${{ split(extraEnvVar, ':::')[0] }}='${{ split(extraEnvVar, ':::')[1] }}'" >> Dockerfile
+  - task: Bash@3
+    displayName: Print Dockerfile
+    condition: or(and(failed(), not(contains(variables['DOCKER_SKIP_GFX'], variables['JOB_GPU_TARGET']))), ${{ eq(parameters.forceDockerCreation, true) }})
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      script: cat Dockerfile
+  - task: Docker@2
+    displayName: Build and upload Docker image
+    condition: or(and(failed(), not(contains(variables['DOCKER_SKIP_GFX'], variables['JOB_GPU_TARGET']))), ${{ eq(parameters.forceDockerCreation, true) }})
+    inputs:
+      containerRegistry: 'ContainerService3'
+      ${{ if ne(parameters.gpuTarget, '') }}:
+        repository: '$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}'
+      ${{ else }}:
+        repository: '$(Build.DefinitionName)-${{ parameters.environment }}'
+      Dockerfile: '$(Pipeline.Workspace)/Dockerfile'
+      buildContext: '$(Pipeline.Workspace)'
+  - task: Bash@3
+    condition: or(and(failed(), not(contains(variables['DOCKER_SKIP_GFX'], variables['JOB_GPU_TARGET']))), ${{ eq(parameters.forceDockerCreation, true) }})
+    displayName: "!! Docker Image URL !!"
+    inputs:
+      workingDirectory: $(Pipeline.Workspace)
+      targetType: inline
+      ${{ if ne(parameters.gpuTarget, '') }}:
+        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}:$(Build.BuildId)" | tr '[:upper:]' '[:lower:]'
+      ${{ else }}:
+        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}:$(Build.BuildId)" | tr '[:upper:]' '[:lower:]'

.azuredevops/templates/steps/docker-container.yml

@@ -334,7 +334,7 @@ steps:
   - task: Docker@2
     condition: or(and(failed(), ${{ not(containsValue(parameters.dockerSkipGfx, parameters.gpuTarget)) }}), ${{ eq(parameters.forceDockerCreation, true) }})
     inputs:
-      containerRegistry: 'ContainerService'
+      containerRegistry: 'ContainerService3'
       ${{ if ne(parameters.gpuTarget, '') }}:
         repository: '$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}'
       ${{ else }}:
@@ -348,6 +348,6 @@ steps:
       workingDirectory: $(Pipeline.Workspace)
       targetType: inline
       ${{ if ne(parameters.gpuTarget, '') }}:
-        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}:$(Build.BuildId)"
+        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}-${{ parameters.gpuTarget }}:$(Build.BuildId)" | tr '[:upper:]' '[:lower:]'
       ${{ else }}:
-        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}:$(Build.BuildId)"
+        script: echo "rocmexternalcicd.azurecr.io/$(Build.DefinitionName)-${{ parameters.environment }}:$(Build.BuildId)" | tr '[:upper:]' '[:lower:]'

.azuredevops/templates/steps/miopen-get-ck-build.yml

@@ -54,7 +54,7 @@ steps:
       fi
 
       echo "Downloading CK artifact from $ARTIFACT_URL"
-      wget -nv $ARTIFACT_URL -O $(System.ArtifactsDirectory)/ck.zip
+      wget --tries=5 --waitretry=10 --retry-connrefused -nv $ARTIFACT_URL -O $(System.ArtifactsDirectory)/ck.zip
       unzip $(System.ArtifactsDirectory)/ck.zip -d $(System.ArtifactsDirectory)
       mkdir -p $(Agent.BuildDirectory)/rocm
       tar -zxvf $(System.ArtifactsDirectory)/$ARTIFACT_NAME/*.tar.gz -C $(Agent.BuildDirectory)/rocm

.azuredevops/variables-global.yml

@@ -30,13 +30,13 @@ variables:
 - name: GFX90A_TEST_POOL
   value: gfx90a_test_pool
 - name: LATEST_RELEASE_VERSION
-  value: 6.3.4
-- name: REPO_RADEON_VERSION
-  value: 6.3.4
-- name: NEXT_RELEASE_VERSION
   value: 6.4.0
+- name: REPO_RADEON_VERSION
+  value: 6.4
+- name: NEXT_RELEASE_VERSION
+  value: 6.5.0
 - name: LATEST_RELEASE_TAG
-  value: rocm-6.3.4
+  value: rocm-6.4.0
 - name: DOCKER_SKIP_GFX
   value: gfx90a
 - name: AMDMIGRAPHX_GFX942_TEST_PIPELINE_ID

.wordlist.txt

@@ -34,7 +34,6 @@ Autocast
 BARs
 BLAS
 BMC
-BabelStream
 Blit
 Blockwise
 Bluefield
@@ -139,7 +138,6 @@ GDR
 GDS
 GEMM
 GEMMs
-GFLOPS
 GFortran
 GFXIP
 Gemma
@@ -643,7 +641,6 @@ hipSPARSELt
 hipTensor
 hipamd
 hipblas
-hipcc
 hipcub
 hipfft
 hipfort

CHANGELOG.md

@@ -125,8 +125,7 @@ Some workaround options are as follows:
 - The `pasid` field in struct `amdsmi_process_info_t` will be deprecated in a future ROCm release.
 
 ```{note}
-See the full [AMD SMI changelog](https://github.com/ROCm/amdsmi/blob/release/rocm-rel-6.4/CHANGELOG.md) for details, examples,
-and in-depth descriptions.
+See the full [AMD SMI changelog](https://github.com/ROCm/amdsmi/blob/release/rocm-rel-6.4/CHANGELOG.md) for details, examples, and in-depth descriptions.
 ```
 
 ### **AMDMIGraphX** (2.12.0)
@@ -736,8 +735,7 @@ The following lists the backward incompatible changes planned for upcoming major
 - Fixed `rsmi_dev_target_graphics_version_get`, `rocm-smi --showhw`, and `rocm-smi --showprod` not displaying graphics version correctly for Instinct MI200 series, MI100 series, and RDNA3-based GPUs. 
 
 ```{note}
-See the full [ROCm SMI changelog](https://github.com/ROCm/rocm_smi_lib/blob/release/rocm-rel-6.4/CHANGELOG.md) for details, examples,
-and in-depth descriptions.
+See the full [ROCm SMI changelog](https://github.com/ROCm/rocm_smi_lib/blob/release/rocm-rel-6.4/CHANGELOG.md) for details, examples, and in-depth descriptions.
 ```
 
 ### **ROCm Systems Profiler** (1.0.0)

RELEASE.md

@@ -634,8 +634,7 @@ Some workaround options are as follows:
 - The `pasid` field in struct `amdsmi_process_info_t` will be deprecated in a future ROCm release.
 
 ```{note}
-See the full [AMD SMI changelog](https://github.com/ROCm/amdsmi/blob/release/rocm-rel-6.4/CHANGELOG.md) for details, examples,
-and in-depth descriptions.
+See the full [AMD SMI changelog](https://github.com/ROCm/amdsmi/blob/release/rocm-rel-6.4/CHANGELOG.md) for details, examples, and in-depth descriptions.
 ```
 
 ### **AMDMIGraphX** (2.12.0)
@@ -767,17 +766,17 @@ and in-depth descriptions.
 
 #### Changed
 
-* The `roc-obj` tools have been deprecated and will be removed in a future release.
+* `roc-obj` tools is deprecated and will be removed in an upcoming release.
 
-   -	`llvm-objdump`, `llvm-objcopy`, and `llvm-readobj` will be enhanced to provide similar functionality as that provided by the `roc-obj` tools . The LLVM tools are available in the `rocm-llvm` pkg.
-   -	While not related to the deprecation, also note that the `roc-obj` tools’ package dependency on Perl has been changed to recommended. It is the user’s responsibility to install Perl to use these tools.
+    - Perl package installation is not required, and users will need to install this themselves if they want to.
+    - Support for ROCm Object tooling has moved into `llvm-objdump` provided by package `rocm-llvm`.
 
 * SDMA retainer logic is removed for engine selection in operation of runtime buffer copy.
 
 #### Optimized
 
 * `hipGraphLaunch` parallelism is improved for complex data-parallel graphs.
-* Round-robin queue mechanism is updated for command scheduling. For multi-streams execution, HSA queue from null stream lock is freed and won't occupy the queue ID after the kernel in the stream is finished.
+* Make the round-robin queue selection in command scheduling. For multi-streams execution, HSA queue from null stream lock is freed and won't occupy the queue ID after the kernel in the stream is finished.
 * The HIP runtime doesn't free bitcode object before code generation. It adds a cache, which allows compiled code objects to be reused instead of recompiling. This improves performance on multi-GPU systems.
 * Runtime now uses unified copy approach:
 
@@ -786,11 +785,6 @@ and in-depth descriptions.
     - The default environment variable `GPU_FORCE_BLIT_COPY_SIZE` is set to `16`, which limits the kernel copy to sizes less than 16 KB, while copies larger than that would be handled by `SDMA` engine.
     - Blit code is refactored, and ASAN instrumentation is cleaned up.
 
-* HIP runtime uses signals without interrupts:
-
-    - In active wait mode, uses signals without interrupts by default.
-    - Only when a callback is required, switches to the interrupts.
-
 #### Resolved issues
 
 * Out-of-memory error on Microsoft Windows. When the user calls `hipMalloc` for device memory allocation while specifying a size larger than the available device memory, the HIP runtime fixes the error in the API implementation, allocating the available device memory plus system memory (shared virtual memory).
@@ -801,15 +795,13 @@ and in-depth descriptions.
 
 The following lists the backward incompatible changes planned for upcoming major ROCm releases.
 
-* Signature changes in APIs to match corresponding CUDA APIs,
+* Signature changes in APIs to correspond with NVIDIA CUDA APIs,
 
     - `hiprtcCreateProgram`
     - `hiprtcCompileProgram`
     - `hipCtxGetApiVersion`
 
 * Behavior of `hipPointerGetAttributes` is changed to match corresponding CUDA API in version 11 and later releases.
-* Behavior of `hipFree` is changed to match corresponding CUDA API `cudaFree`.
-* HIP vector constructor changes for `hipComplex`.
 * Return error/value code updates in the following hip APIs to match the corresponding CUDA APIs,
 
     - `hipModuleLaunchKernel`
@@ -1252,8 +1244,7 @@ The following lists the backward incompatible changes planned for upcoming major
 - Fixed `rsmi_dev_target_graphics_version_get`, `rocm-smi --showhw`, and `rocm-smi --showprod` not displaying graphics version correctly for Instinct MI200 series, MI100 series, and RDNA3-based GPUs. 
 
 ```{note}
-See the full [ROCm SMI changelog](https://github.com/ROCm/rocm_smi_lib/blob/release/rocm-rel-6.4/CHANGELOG.md) for details, examples,
-and in-depth descriptions.
+See the full [ROCm SMI changelog](https://github.com/ROCm/rocm_smi_lib/blob/release/rocm-rel-6.4/CHANGELOG.md) for details, examples, and in-depth descriptions.
 ```
 
 ### **ROCm Systems Profiler** (1.0.0)
@@ -1732,7 +1723,7 @@ and will be disabled in a future release.
 
 * The `__AMDGCN_WAVEFRONT_SIZE__` macro and `__AMDGCN_WAVEFRONT_SIZE` alias will be removed in an upcoming release.
   It is recommended to remove any use of this macro. For more information, see
-  [AMDGPU support](https://rocm.docs.amd.com/projects/llvm-project/en/docs-6.4.0/LLVM/clang/html/AMDGPUSupport.html).
+  [AMDGPU support](https://rocm.docs.amd.com/projects/llvm-project/en/docs-6.3.2/LLVM/clang/html/AMDGPUSupport.html).
 * `warpSize` will only be available as a non-`constexpr` variable. Where required,
   the wavefront size should be queried via the `warpSize` variable in device code,
   or via `hipGetDeviceProperties` in host code. Neither of these will result in a compile-time constant. 

docs/conf.py

@@ -51,8 +51,6 @@ article_pages = [
     {"file": "how-to/deep-learning-rocm", "os": ["linux"]},
 
     {"file": "how-to/rocm-for-ai/index", "os": ["linux"]},
-    {"file": "how-to/rocm-for-ai/install", "os": ["linux"]},
-    {"file": "how-to/rocm-for-ai/system-health-check", "os": ["linux"]},
 
     {"file": "how-to/rocm-for-ai/training/index", "os": ["linux"]},
     {"file": "how-to/rocm-for-ai/training/train-a-model", "os": ["linux"]},
@@ -69,6 +67,7 @@ article_pages = [
     {"file": "how-to/rocm-for-ai/fine-tuning/multi-gpu-fine-tuning-and-inference", "os": ["linux"]},
 
     {"file": "how-to/rocm-for-ai/inference/index", "os": ["linux"]},
+    {"file": "how-to/rocm-for-ai/inference/install", "os": ["linux"]},
     {"file": "how-to/rocm-for-ai/inference/hugging-face-models", "os": ["linux"]},
     {"file": "how-to/rocm-for-ai/inference/llm-inference-frameworks", "os": ["linux"]},
     {"file": "how-to/rocm-for-ai/inference/vllm-benchmark", "os": ["linux"]},

docs/how-to/rocm-for-ai/inference/install.rst

@@ -30,7 +30,7 @@ ROCm supports multiple :doc:`installation methods <rocm-install-on-linux:install
 
 * :doc:`Using the AMDGPU installer <rocm-install-on-linux:install/amdgpu-install>`
 
-* :ref:`Multi-version installation <rocm-install-on-linux:installation-types>`
+* :ref:`Multi-version installation <rocm-install-on-linux:installation-types>`.
 
 .. grid:: 1
 
@@ -59,8 +59,4 @@ images with the framework pre-installed.
 
 * :doc:`JAX for ROCm <rocm-install-on-linux:install/3rd-party/jax-install>`
 
-Next steps
-==========
-
-After installing ROCm and your desired ML libraries -- and before running AI workloads -- conduct system health benchmarks
-to test the optimal performance of your AMD hardware. See :doc:`system-health-check` to get started.
+The sections that follow in :doc:`Training a model <../training/train-a-model>` are geared for a ROCm with PyTorch installation.

docs/how-to/rocm-for-ai/inference/pytorch-inference-benchmark.rst

@@ -62,52 +62,47 @@ PyTorch inference performance testing
       {% endfor %}
    {% endfor %}
 
-   System validation
-   =================
+   Getting started
+   ===============
 
-   Before running AI workloads, it's important to validate that your AMD hardware is configured
-   correctly and performing optimally.
+   Use the following procedures to reproduce the benchmark results on an
+   MI300X series accelerator with the prebuilt PyTorch Docker image.
 
-   To optimize performance, disable automatic NUMA balancing. Otherwise, the GPU
-   might hang until the periodic balancing is finalized. For more information,
-   see the :ref:`system validation steps <rocm-for-ai-system-optimization>`.
+   .. _pytorch-benchmark-get-started:
 
-   .. code-block:: shell
+   1. Disable NUMA auto-balancing.
 
-      # disable automatic NUMA balancing
-      sh -c 'echo 0 > /proc/sys/kernel/numa_balancing'
-      # check if NUMA balancing is disabled (returns 0 if disabled)
-      cat /proc/sys/kernel/numa_balancing
-      0
+      To optimize performance, disable automatic NUMA balancing. Otherwise, the GPU
+      might hang until the periodic balancing is finalized. For more information,
+      see :ref:`AMD Instinct MI300X system optimization <mi300x-disable-numa>`.
 
-   To test for optimal performance, consult the recommended :ref:`System health benchmarks
-   <rocm-for-ai-system-health-bench>`. This suite of tests will help you verify and fine-tune your
-   system's configuration.
+      .. code-block:: shell
 
-   Pull the Docker image
-   =====================
+         # disable automatic NUMA balancing
+         sh -c 'echo 0 > /proc/sys/kernel/numa_balancing'
+         # check if NUMA balancing is disabled (returns 0 if disabled)
+         cat /proc/sys/kernel/numa_balancing
+         0
 
    .. container:: model-doc pyt_chai1_inference
 
-      Use the following command to pull the `ROCm PyTorch Docker image <https://hub.docker.com/layers/rocm/pytorch/rocm6.2.3_ubuntu22.04_py3.10_pytorch_release_2.3.0_triton_llvm_reg_issue/images/sha256-b736a4239ab38a9d0e448af6d4adca83b117debed00bfbe33846f99c4540f79b>`_ from Docker Hub.
+      2. Use the following command to pull the `ROCm PyTorch Docker image <https://hub.docker.com/layers/rocm/pytorch/rocm6.2.3_ubuntu22.04_py3.10_pytorch_release_2.3.0_triton_llvm_reg_issue/images/sha256-b736a4239ab38a9d0e448af6d4adca83b117debed00bfbe33846f99c4540f79b>`_ from Docker Hub.
 
-      .. code-block:: shell
+         .. code-block:: shell
 
-         docker pull rocm/pytorch:rocm6.2.3_ubuntu22.04_py3.10_pytorch_release_2.3.0_triton_llvm_reg_issue
+            docker pull rocm/pytorch:rocm6.2.3_ubuntu22.04_py3.10_pytorch_release_2.3.0_triton_llvm_reg_issue
 
-      .. note::
+         .. note::
 
-         The Chai-1 benchmark uses a specifically selected Docker image using ROCm 6.2.3 and PyTorch 2.3.0 to address an accuracy issue.
+            The Chai-1 benchmark uses a specifically selected Docker image using ROCm 6.2.3 and PyTorch 2.3.0 to address an accuracy issue.
 
    .. container:: model-doc pyt_clip_inference
 
-      Use the following command to pull the `ROCm PyTorch Docker image <https://hub.docker.com/layers/rocm/pytorch/latest/images/sha256-05b55983e5154f46e7441897d0908d79877370adca4d1fff4899d9539d6c4969>`_ from Docker Hub.
+      2. Use the following command to pull the `ROCm PyTorch Docker image <https://hub.docker.com/layers/rocm/pytorch/latest/images/sha256-05b55983e5154f46e7441897d0908d79877370adca4d1fff4899d9539d6c4969>`_ from Docker Hub.
 
-      .. code-block:: shell
+         .. code-block:: shell
 
-         docker pull rocm/pytorch:latest
-
-   .. _pytorch-benchmark-get-started:
+            docker pull rocm/pytorch:latest
 
    Benchmarking
    ============

docs/how-to/rocm-for-ai/inference/vllm-benchmark.rst

@@ -111,37 +111,35 @@ vLLM inference performance testing
    For information on experimental features and known issues related to ROCm optimization efforts on vLLM,
    see the developer's guide at `<https://github.com/ROCm/vllm/blob/main/docs/dev-docker/README.md>`__.
 
-   System validation
-   =================
+   Getting started
+   ===============
 
-   Before running AI workloads, it's important to validate that your AMD hardware is configured
-   correctly and performing optimally.
+   Use the following procedures to reproduce the benchmark results on an
+   MI300X accelerator with the prebuilt vLLM Docker image.
 
-   To optimize performance, disable automatic NUMA balancing. Otherwise, the GPU
-   might hang until the periodic balancing is finalized. For more information,
-   see the :ref:`system validation steps <rocm-for-ai-system-optimization>`.
+   .. _vllm-benchmark-get-started:
 
-   .. code-block:: shell
+   1. Disable NUMA auto-balancing.
 
-      # disable automatic NUMA balancing
-      sh -c 'echo 0 > /proc/sys/kernel/numa_balancing'
-      # check if NUMA balancing is disabled (returns 0 if disabled)
-      cat /proc/sys/kernel/numa_balancing
-      0
+      To optimize performance, disable automatic NUMA balancing. Otherwise, the GPU
+      might hang until the periodic balancing is finalized. For more information,
+      see :ref:`AMD Instinct MI300X system optimization <mi300x-disable-numa>`.
 
-   To test for optimal performance, consult the recommended :ref:`System health benchmarks
-   <rocm-for-ai-system-health-bench>`. This suite of tests will help you verify and fine-tune your
-   system's configuration.
+      .. code-block:: shell
 
-   Pull the Docker image
-   =====================
+         # disable automatic NUMA balancing
+         sh -c 'echo 0 > /proc/sys/kernel/numa_balancing'
+         # check if NUMA balancing is disabled (returns 0 if disabled)
+         cat /proc/sys/kernel/numa_balancing
+         0
 
-   Download the `ROCm vLLM Docker image <{{ unified_docker.docker_hub_url }}>`_.
-   Use the following command to pull the Docker image from Docker Hub.
+   2. Download the `ROCm vLLM Docker image <{{ unified_docker.docker_hub_url }}>`_.
 
-   .. code-block:: shell
+      Use the following command to pull the Docker image from Docker Hub.
 
-      docker pull {{ unified_docker.pull_tag }}
+      .. code-block:: shell
+
+         docker pull {{ unified_docker.pull_tag }}
 
    Benchmarking
    ============

docs/how-to/rocm-for-ai/system-health-check.rst

@@ -1,104 +0,0 @@
-.. meta::
-   :description: System health checks with RVS, RCCL tests, BabelStream, and TransferBench to validate AMD hardware performance running AI workloads.
-   :keywords: gpu, accelerator, system, health, validation, bench, perf, performance, rvs, rccl, babel, mi300x, mi325x, flops, bandwidth, rbt, training, inference
-
-.. _rocm-for-ai-system-health-bench:
-
-************************
-System health benchmarks
-************************
-
-Before running AI workloads, it is important to validate that your AMD hardware is configured correctly and is performing optimally. This topic outlines several system health benchmarks you can use to test key aspects like GPU compute capabilities (FLOPS), memory bandwidth, and interconnect performance. Many of these tests are part of the ROCm Validation Suite (RVS).
-
-ROCm Validation Suite (RVS) tests
-=================================
-
-RVS provides a collection of tests, benchmarks, and qualification tools, each
-targeting a specific subsystem of the system under test. It includes tests for
-GPU stress and memory bandwidth.
-
-.. _healthcheck-install-rvs:
-
-Install ROCm Validation Suite
------------------------------
-
-To get started, install RVS. For example, on an Ubuntu system with ROCm already
-installed, run the following command:
-
-.. code-block:: shell
-
-   sudo apt update
-   sudo apt install rocm-validation-suite
-
-See the `ROCm Validation Suite installation instructions <https://rocm.docs.amd.com/projects/ROCmValidationSuite/en/latest/install/installation.html>`_,
-and `System validation tests <https://instinct.docs.amd.com/projects/system-acceptance/en/latest/mi300x/system-validation.html#system-validation-tests>`_
-in the Instinct documentation for more detailed instructions.
-
-Benchmark, stress, and qualification tests
-------------------------------------------
-
-The GPU stress test runs various GEMM computations as workloads to stress the GPU FLOPS performance and check whether it
-meets the configured target GFLOPS.
-
-Run the benchmark, stress, and qualification tests included with RVS. See the `Benchmark, stress, qualification
-<https://instinct.docs.amd.com/projects/system-acceptance/en/latest/mi300x/system-validation.html#benchmark-stress-qualification>`_
-section of the Instinct documentation for usage instructions.
-
-BabelStream test
-----------------
-
-BabelStream is a synthetic GPU benchmark based on the STREAM benchmark for
-CPUs, measuring memory transfer rates to and from global device memory.
-BabelStream tests are included with the RVS package as part of the `BABEL module
-<https://rocm.docs.amd.com/projects/ROCmValidationSuite/en/latest/conceptual/rvs-modules.html#babel-benchmark-test-babel-module>`_.
-
-For more information, see `Performance benchmarking
-<https://instinct.docs.amd.com/projects/system-acceptance/en/latest/mi300x/performance-bench.html#babelstream-benchmarking-results>`_
-in the Instinct documentation.
-
-RCCL tests
-==========
-
-The ROCm Communication Collectives Library (RCCL) enables efficient multi-GPU
-communication. The `<https://github.com/ROCm/rccl-tests>`__ suite benchmarks
-the performance and verifies the correctness of these collective operations.
-This helps ensure optimal scaling for multi-accelerator tasks.
-
-1. To get started, build RCCL-tests using the official instructions in the README at
-   `<https://github.com/ROCm/rccl-tests?tab=readme-ov-file#build>`__ or use the
-   following commands:
-
-   .. code-block:: shell
-
-      git clone https://github.com/ROCm/rccl-tests.git
-      cd rccl-tests
-      make
-
-2. Run the suggested RCCL tests -- see `RCCL benchmarking
-   <https://instinct.docs.amd.com/projects/system-acceptance/en/latest/mi300x/performance-bench.html#rccl-benchmarking-results>`_
-   in the Instinct performance benchmarking documentation for instructions.
-
-TransferBench test
-==================
-
-TransferBench is a standalone utility for benchmarking simultaneous data
-transfer performance between various devices in the system, including
-CPU-to-GPU and GPU-to-GPU (peer-to-peer). This helps identify potential
-bottlenecks in data movement between the host system and the GPUs, or between
-GPUs, which can impact end-to-end latency.
-
-.. _healthcheck-install-transferbench:
-
-1. To get started, use the instructions in the `TransferBench documentation
-   <https://rocm.docs.amd.com/projects/TransferBench/en/latest/install/install.html#install-transferbench>`_
-   or use the following commands:
-
-   .. code:: shell
-
-      git clone https://github.com/ROCm/TransferBench.git
-      cd TransferBench
-      CC=hipcc make
-
-2. Run the suggested TransferBench tests -- see `TransferBench benchmarking
-   <https://instinct.docs.amd.com/projects/system-acceptance/en/latest/mi300x/performance-bench.html#transferbench-benchmarking-results>`_
-   in the Instinct performance benchmarking documentation for instructions.

docs/how-to/rocm-for-ai/training/benchmark-docker/jax-maxtext.rst

@@ -79,18 +79,11 @@ across different input sequences. Support for packed input format is planned for
 System validation
 =================
 
-Before running AI workloads, it's important to validate that your AMD hardware is configured
-correctly and performing optimally.
-
-If you have already validated your system settings, including aspects like NUMA auto-balancing, you
-can skip this step. Otherwise, complete the procedures in the :ref:`System validation and
-optimization <rocm-for-ai-system-optimization>` guide to properly configure your system settings
+If you have already validated your system settings, including NUMA
+auto-balancing, skip this step. Otherwise, complete the :ref:`system validation
+and optimization steps <train-a-model-system-validation>` to set up your system
 before starting training.
 
-To test for optimal performance, consult the recommended :ref:`System health benchmarks
-<rocm-for-ai-system-health-bench>`. This suite of tests will help you verify and fine-tune your
-system's configuration.
-
 Environment setup
 =================
 
@@ -182,8 +175,8 @@ with RDMA, skip ahead to :ref:`amd-maxtext-download-docker`.
 
 .. _amd-maxtext-download-docker:
 
-Pull the Docker image
----------------------
+Download the Docker image
+-------------------------
 
 1. Use the following command to pull the Docker image from Docker Hub.
 

docs/how-to/rocm-for-ai/training/benchmark-docker/megatron-lm.rst

@@ -103,18 +103,11 @@ popular AI models.
 System validation
 =================
 
-Before running AI workloads, it's important to validate that your AMD hardware is configured
-correctly and performing optimally.
-
-If you have already validated your system settings, including aspects like NUMA auto-balancing, you
-can skip this step. Otherwise, complete the procedures in the :ref:`System validation and
-optimization <rocm-for-ai-system-optimization>` guide to properly configure your system settings
+If you have already validated your system settings, including NUMA
+auto-balancing, skip this step. Otherwise, complete the :ref:`system validation
+and optimization steps <train-a-model-system-validation>` to set up your system
 before starting training.
 
-To test for optimal performance, consult the recommended :ref:`System health benchmarks
-<rocm-for-ai-system-health-bench>`. This suite of tests will help you verify and fine-tune your
-system's configuration.
-
 .. _mi300x-amd-megatron-lm-training:
 
 Environment setup

docs/how-to/rocm-for-ai/training/benchmark-docker/mpt-llm-foundry.rst

@@ -34,18 +34,11 @@ for MPT-30B with access to detailed logs and performance metrics.
 System validation
 =================
 
-Before running AI workloads, it's important to validate that your AMD hardware is configured
-correctly and performing optimally.
-
-If you have already validated your system settings, including aspects like NUMA auto-balancing, you
-can skip this step. Otherwise, complete the procedures in the :ref:`System validation and
-optimization <rocm-for-ai-system-optimization>` guide to properly configure your system settings
+If you have already validated your system settings, including NUMA
+auto-balancing, skip this step. Otherwise, complete the :ref:`system validation
+and optimization steps <train-a-model-system-validation>` to set up your system
 before starting training.
 
-To test for optimal performance, consult the recommended :ref:`System health benchmarks
-<rocm-for-ai-system-health-bench>`. This suite of tests will help you verify and fine-tune your
-system's configuration.
-
 Getting started
 ===============
 

docs/how-to/rocm-for-ai/training/benchmark-docker/pytorch-training.rst

@@ -77,18 +77,11 @@ popular AI models.
 System validation
 =================
 
-Before running AI workloads, it's important to validate that your AMD hardware is configured
-correctly and performing optimally.
-
-If you have already validated your system settings, including aspects like NUMA auto-balancing, you
-can skip this step. Otherwise, complete the procedures in the :ref:`System validation and
-optimization <rocm-for-ai-system-optimization>` guide to properly configure your system settings
+If you have already validated your system settings, including NUMA
+auto-balancing, skip this step. Otherwise, complete the :ref:`system validation
+and optimization steps <train-a-model-system-validation>` to set up your system
 before starting training.
 
-To test for optimal performance, consult the recommended :ref:`System health benchmarks
-<rocm-for-ai-system-health-bench>`. This suite of tests will help you verify and fine-tune your
-system's configuration.
-
 This Docker image is optimized for specific model configurations outlined
 below. Performance can vary for other training workloads, as AMD 
 doesn’t validate configurations and run conditions outside those described.

docs/how-to/rocm-for-ai/training/index.rst

@@ -21,12 +21,8 @@ In this guide, you'll learn about:
 
 - Training a model
 
-  - :doc:`With Megatron-LM <benchmark-docker/megatron-lm>`
+  - :doc:`Train a model with Megatron-LM <benchmark-docker/megatron-lm>`
 
-  - :doc:`With PyTorch <benchmark-docker/pytorch-training>`
-
-  - :doc:`With JAX MaxText <benchmark-docker/jax-maxtext>`
-
-  - :doc:`With LLM Foundry <benchmark-docker/mpt-llm-foundry>`
+  - :doc:`Train a model with PyTorch <benchmark-docker/pytorch-training>`
 
 - :doc:`Scaling model training <scale-model-training>`

docs/how-to/rocm-for-ai/training/prerequisite-system-validation.rst

@@ -5,13 +5,12 @@
    :keywords: ROCm, AI, LLM, train, megatron, Llama, tutorial, docker, torch, pytorch, jax
 
 .. _train-a-model-system-validation:
-.. _rocm-for-ai-system-optimization:
 
-**********************************************************
-Prerequisite system validation before running AI workloads
-**********************************************************
+**********************************************
+Prerequisite system validation before training
+**********************************************
 
-Complete the following system validation and optimization steps to set up your system before starting training and inference.
+Complete the following system validation and optimization steps to set up your system before starting training.
 
 Disable NUMA auto-balancing
 ---------------------------
@@ -27,8 +26,7 @@ the output is ``1``, run the following command to disable NUMA auto-balancing.
 
    sudo sh -c 'echo 0 > /proc/sys/kernel/numa_balancing'
 
-See `Disable NUMA auto-balancing <https://instinct.docs.amd.com/projects/amdgpu-docs/en/latest/system-optimization/mi300x.html#disable-numa-auto-balancing>`_
-in the Instinct documentation for more information.
+See :ref:`mi300x-disable-numa` for more information.
 
 Hardware verification with ROCm
 -------------------------------
@@ -44,8 +42,7 @@ Run the command:
 
    rocm-smi --setperfdeterminism 1900
 
-See `Hardware verfication for ROCm <https://instinct.docs.amd.com/projects/amdgpu-docs/en/latest/system-optimization/mi300x.html#hardware-verification-with-rocm>`_
-in the Instinct documentation for more information.
+See :ref:`mi300x-hardware-verification-with-rocm` for more information.
 
 RCCL Bandwidth Test for multi-node setups
 -----------------------------------------

docs/sphinx/_toc.yml.in

@@ -12,14 +12,14 @@ subtrees:
   - file: compatibility/compatibility-matrix.rst
     title: Compatibility matrix
     entries:
-    - url: https://rocm.docs.amd.com/projects/install-on-linux/en/${branch}/reference/system-requirements.html
+    - url: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html
       title: Linux system requirements
     - url: https://rocm.docs.amd.com/projects/install-on-windows/en/${branch}/reference/system-requirements.html
       title: Windows system requirements
 
 - caption: Install
   entries:
-  - url: https://rocm.docs.amd.com/projects/install-on-linux/en/${branch}/
+  - url: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/
     title: ROCm on Linux
   - url: https://rocm.docs.amd.com/projects/install-on-windows/en/${branch}/
     title: HIP SDK on Windows
@@ -36,10 +36,6 @@ subtrees:
     title: Use ROCm for AI
     subtrees:
     - entries:
-      - file: how-to/rocm-for-ai/install.rst
-        title: Installation
-      - file: how-to/rocm-for-ai/system-health-check.rst
-        title: System health benchmarks
       - file: how-to/rocm-for-ai/training/index.rst
         title: Training
         subtrees:
@@ -74,6 +70,8 @@ subtrees:
         title: Inference
         subtrees:
         - entries:
+          - file: how-to/rocm-for-ai/inference/install.rst
+            title: Installation
           - file: how-to/rocm-for-ai/inference/hugging-face-models.rst
             title: Run models from Hugging Face
           - file: how-to/rocm-for-ai/inference/llm-inference-frameworks.rst

docs/sphinx/requirements.txt

@@ -2,55 +2,54 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile requirements.in
+#    pip-compile docs/sphinx/requirements.in
 #
 accessible-pygments==0.0.5
     # via pydata-sphinx-theme
 alabaster==1.0.0
     # via sphinx
-appnope==0.1.4
-    # via ipykernel
 asttokens==3.0.0
     # via stack-data
-attrs==25.3.0
+attrs==25.1.0
     # via
     #   jsonschema
     #   jupyter-cache
     #   referencing
-babel==2.17.0
+babel==2.16.0
     # via
     #   pydata-sphinx-theme
     #   sphinx
-beautifulsoup4==4.13.3
+beautifulsoup4==4.12.3
     # via pydata-sphinx-theme
-breathe==4.36.0
+breathe==4.35.0
     # via rocm-docs-core
-certifi==2025.1.31
+certifi==2024.8.30
     # via requests
 cffi==1.17.1
     # via
     #   cryptography
     #   pynacl
-charset-normalizer==3.4.1
+charset-normalizer==3.4.0
     # via requests
-click==8.1.8
+click==8.1.7
     # via
     #   jupyter-cache
     #   sphinx-external-toc
 comm==0.2.2
     # via ipykernel
-cryptography==44.0.2
+cryptography==44.0.1
     # via pyjwt
-debugpy==1.8.14
+debugpy==1.8.12
     # via ipykernel
-decorator==5.2.1
+decorator==5.1.1
     # via ipython
 defusedxml==0.7.1
     # via sphinxcontrib-datatemplates
-deprecated==1.2.18
+deprecated==1.2.15
     # via pygithub
 docutils==0.21.2
     # via
+    #   breathe
     #   myst-parser
     #   pydata-sphinx-theme
     #   sphinx
@@ -58,14 +57,16 @@ exceptiongroup==1.2.2
     # via ipython
 executing==2.2.0
     # via stack-data
-fastjsonschema==2.21.1
+fastjsonschema==2.20.0
     # via
     #   nbformat
     #   rocm-docs-core
-gitdb==4.0.12
+gitdb==4.0.11
     # via gitpython
-gitpython==3.1.44
+gitpython==3.1.43
     # via rocm-docs-core
+greenlet==3.1.1
+    # via sqlalchemy
 idna==3.10
     # via requests
 imagesize==1.4.1
@@ -76,7 +77,7 @@ importlib-metadata==8.6.1
     #   myst-nb
 ipykernel==6.29.5
     # via myst-nb
-ipython==8.35.0
+ipython==8.31.0
     # via
     #   ipykernel
     #   myst-nb
@@ -116,9 +117,9 @@ mdit-py-plugins==0.4.2
     # via myst-parser
 mdurl==0.1.2
     # via markdown-it-py
-myst-nb==1.2.0
+myst-nb==1.1.2
     # via rocm-docs-core
-myst-parser==4.0.1
+myst-parser==4.0.0
     # via myst-nb
 nbclient==0.10.2
     # via
@@ -134,17 +135,16 @@ nest-asyncio==1.6.0
 packaging==24.2
     # via
     #   ipykernel
-    #   pydata-sphinx-theme
     #   sphinx
 parso==0.8.4
     # via jedi
 pexpect==4.9.0
     # via ipython
-platformdirs==4.3.7
+platformdirs==4.3.6
     # via jupyter-core
 prompt-toolkit==3.0.50
     # via ipython
-psutil==7.0.0
+psutil==6.1.1
     # via ipykernel
 ptyprocess==0.7.0
     # via pexpect
@@ -152,19 +152,19 @@ pure-eval==0.2.3
     # via stack-data
 pycparser==2.22
     # via cffi
-pydata-sphinx-theme==0.15.4
+pydata-sphinx-theme==0.16.0
     # via
     #   rocm-docs-core
     #   sphinx-book-theme
-pygithub==2.6.1
+pygithub==2.5.0
     # via rocm-docs-core
-pygments==2.19.1
+pygments==2.18.0
     # via
     #   accessible-pygments
     #   ipython
     #   pydata-sphinx-theme
     #   sphinx
-pyjwt[crypto]==2.10.1
+pyjwt[crypto]==2.10.0
     # via pygithub
 pynacl==1.5.0
     # via pygithub
@@ -178,7 +178,7 @@ pyyaml==6.0.2
     #   rocm-docs-core
     #   sphinx-external-toc
     #   sphinxcontrib-datatemplates
-pyzmq==26.4.0
+pyzmq==26.2.0
     # via
     #   ipykernel
     #   jupyter-client
@@ -192,13 +192,13 @@ requests==2.32.3
     #   sphinx
 rocm-docs-core==1.18.2
     # via -r requirements.in
-rpds-py==0.24.0
+rpds-py==0.22.3
     # via
     #   jsonschema
     #   referencing
 six==1.17.0
     # via python-dateutil
-smmap==5.0.2
+smmap==5.0.1
     # via gitdb
 snowballstemmer==2.2.0
     # via sphinx
@@ -220,7 +220,7 @@ sphinx==8.1.3
     #   sphinx-sitemap
     #   sphinxcontrib-datatemplates
     #   sphinxcontrib-runcmd
-sphinx-book-theme==1.1.4
+sphinx-book-theme==1.1.3
     # via rocm-docs-core
 sphinx-copybutton==0.5.2
     # via rocm-docs-core
@@ -228,7 +228,7 @@ sphinx-design==0.6.1
     # via rocm-docs-core
 sphinx-external-toc==1.0.1
     # via rocm-docs-core
-sphinx-notfound-page==1.1.0
+sphinx-notfound-page==1.0.4
     # via rocm-docs-core
 sphinx-reredirects==0.1.6
     # via -r requirements.in
@@ -250,13 +250,13 @@ sphinxcontrib-runcmd==0.2.0
     # via sphinxcontrib-datatemplates
 sphinxcontrib-serializinghtml==2.0.0
     # via sphinx
-sqlalchemy==2.0.40
+sqlalchemy==2.0.37
     # via jupyter-cache
 stack-data==0.6.3
     # via ipython
 tabulate==0.9.0
     # via jupyter-cache
-tomli==2.2.1
+tomli==2.1.0
     # via sphinx
 tornado==6.4.2
     # via
@@ -272,22 +272,21 @@ traitlets==5.14.3
     #   matplotlib-inline
     #   nbclient
     #   nbformat
-typing-extensions==4.13.2
+typing-extensions==4.12.2
     # via
-    #   beautifulsoup4
     #   ipython
     #   myst-nb
     #   pydata-sphinx-theme
     #   pygithub
     #   referencing
     #   sqlalchemy
-urllib3==2.4.0
+urllib3==2.2.3
     # via
     #   pygithub
     #   requests
 wcwidth==0.2.13
     # via prompt-toolkit
-wrapt==1.17.2
+wrapt==1.17.0
     # via deprecated
 zipp==3.21.0
     # via importlib-metadata

tools/rocm-build/ROCm.mk

@@ -12,17 +12,36 @@ RELEASE_FLAG?=-r
 ASAN_DEP:=
 ifeq (${ENABLE_ADDRESS_SANITIZER},true)
 	ASAN_DEP=lightning
-	SANITIZER_FLAG=-a
+	SANITIZER_FLAG=--address_sanitizer
+endif
+
+# Set STATIC_FLAG for static builds
+ifeq (${ENABLE_STATIC_BUILDS},true)
+	STATIC_FLAG=-s
 endif
 
 export INFRA_REPO:=ROCm/tools/rocm-build
 
+# # commannds to be run at makefile read time
+# # should only output $$OUT_DIR to stdout
+# # In an ideal world this would be another target
+# define INITBUILD
+# source ${INFRA_REPO}/envsetup.sh >/dev/null 2>&1;
+# [ -w "$${ROCM_INSTALL_PATH}" ] || sudo mkdir -p -m 775 "$${ROCM_INSTALL_PATH}" ;
+# sudo chown "$$(id -u):$$(id -g)" "$${ROCM_INSTALL_PATH}" "/home/$$(id -un)" ;
+# mkdir -p ${HOME}/.ccache ;
+# echo $${OUT_DIR} ;
+# endef
+
 OUT_DIR:=$(shell . ${INFRA_REPO}/envsetup.sh >/dev/null 2>&1 ; echo $${OUT_DIR})
 ROCM_INSTALL_PATH:=$(shell . ${INFRA_REPO}/envsetup.sh >/dev/null 2>&1 ; echo $${ROCM_INSTALL_PATH})
 
 $(info OUT_DIR=${OUT_DIR})
 $(info ROCM_INSTALL_PATH=${ROCM_INSTALL_PATH})
 
+# define SILENT to be empty to see the runner invocation
+SILENT?= @
+
 # -------------------------------------------------------------------------
 # Internal stuff. Could be put in a different file to hide it.
 # Internal macros, they need to be defined before being used.
@@ -47,7 +66,8 @@ endif
 # It should not be needed.
 define adddep =
 $(strip $(call peval,components+= $(1) $(2))
-$(foreach comp,$(strip $2),$(call peval,${OUT_DIR}/logs/${1}: ${OUT_DIR}/logs/${comp}))
+$(call peval,$(1)_DEPS += $(2))
+$(foreach comp,$(strip $2),$(call peval,${OUT_DIR}/logs/${1}.txt: ${OUT_DIR}/logs/${comp}.txt))
 )
 endef
 # End of internal stuff that is needed at the start of the file
@@ -67,6 +87,7 @@ endef
 
 $(call adddep,amd_smi_lib,${ASAN_DEP})
 $(call adddep,aqlprofile,${ASAN_DEP} rocr)
+$(call adddep,aqlprofiletest,lightning rocminfo aqlprofile opencl_on_rocclr hip_on_rocclr)
 $(call adddep,comgr,lightning devicelibs)
 $(call adddep,dbgapi,rocr comgr)
 $(call adddep,devicelibs,lightning)
@@ -83,27 +104,29 @@ $(call adddep,rocm-core,${ASAN_DEP})
 $(call adddep,rocm-gdb,dbgapi)
 $(call adddep,rocminfo,${ASAN_DEP} rocr)
 $(call adddep,rocprofiler-register,${ASAN_DEP})
-$(call adddep,rocprofiler-sdk,${ASAN_DEP} rocr aqlprofile opencl_on_rocclr hip_on_rocclr comgr)
+$(call adddep,rocprofiler-sdk,${ASAN_DEP} rocr aqlprofile opencl_on_rocclr hip_on_rocclr comgr rccl rocdecode)
 $(call adddep,rocprofiler-systems,${ASAN_DEP} hipcc rocr hip_on_rocclr rocm_smi_lib rocprofiler roctracer rocprofiler-sdk)
 $(call adddep,rocprofiler,${ASAN_DEP} rocr roctracer aqlprofile opencl_on_rocclr hip_on_rocclr comgr)
 $(call adddep,rocprofiler-compute,${ASAN_DEP})
 $(call adddep,rocr,${ASAN_DEP} lightning rocm_smi_lib devicelibs rocprofiler-register)
 $(call adddep,rocr_debug_agent,${ASAN_DEP} hip_on_rocclr rocr dbgapi)
+$(call adddep,rocrsamples,lightning devicelibs rocr )
 $(call adddep,roctracer,${ASAN_DEP} rocr hip_on_rocclr)
 
+
 # rocm-dev points to all possible last finish components of Stage1 build.
-rocm-dev-components :=amd_smi_lib aqlprofile comgr dbgapi devicelibs hip_on_rocclr hipcc hipify_clang \
+rocm-dev-components :=amd_smi_lib aqlprofile aqlprofiletest comgr dbgapi devicelibs hip_on_rocclr hipcc hipify_clang \
 	lightning rocprofiler-compute opencl_on_rocclr openmp_extras rocm_bandwidth_test rocm_smi_lib \
 	rocm-cmake rocm-core rocm-gdb rocminfo rocprofiler-register rocprofiler-sdk rocprofiler-systems \
-	rocprofiler rocr rocr_debug_agent roctracer
-$(call adddep,rocm-dev,$(filter-out ${NOBUILD} kernel_ubuntu,${rocm-dev-components}))
+	rocprofiler rocr rocr_debug_agent rocrsamples roctracer
+$(call adddep,rocm-dev,$(filter-out ${NOBUILD},${rocm-dev-components}))
 
 $(call adddep,amdmigraphx,hip_on_rocclr half rocblas miopen-hip lightning hipcc hiptensor)
 $(call adddep,composable_kernel,lightning hipcc hip_on_rocclr rocm-cmake)
 $(call adddep,half,rocm-cmake)
 $(call adddep,hipblas-common,lightning)
 $(call adddep,hipblas,hip_on_rocclr rocblas rocsolver lightning hipcc)
-$(call adddep,hipblaslt,hip_on_rocclr openmp_extras lightning hipcc hipblas-common rocm-dev)
+$(call adddep,hipblaslt,hip_on_rocclr openmp_extras lightning hipcc hipblas-common roctracer)
 $(call adddep,hipcub,hip_on_rocclr rocprim lightning hipcc)
 $(call adddep,hipfft,hip_on_rocclr openmp_extras rocfft rocrand hiprand lightning hipcc)
 $(call adddep,hipfort,rocblas hipblas rocsparse hipsparse rocfft hipfft rocrand hiprand rocsolver hipsolver lightning hipcc)
@@ -113,26 +136,34 @@ $(call adddep,hipsparse,hip_on_rocclr rocsparse lightning hipcc)
 $(call adddep,hipsparselt,hip_on_rocclr hipsparse lightning hipcc openmp_extras)
 $(call adddep,hiptensor,hip_on_rocclr composable_kernel lightning hipcc)
 $(call adddep,miopen-deps,lightning hipcc)
-$(call adddep,miopen-hip,composable_kernel half hip_on_rocclr miopen-deps hipblas hipblaslt rocrand roctracer lightning hipcc)
+$(call adddep,miopen-hip,rocm-core composable_kernel half hip_on_rocclr miopen-deps hipblas hipblaslt rocrand roctracer lightning hipcc)
 $(call adddep,mivisionx,amdmigraphx miopen-hip rpp lightning hipcc)
 $(call adddep,rccl,rocm-core hip_on_rocclr rocr lightning hipcc rocm_smi_lib hipify_clang)
-$(call adddep,rdc,rocm_smi_lib rocprofiler rocmvalidationsuite)
+$(call adddep,rdc,amd_smi_lib rocprofiler-sdk rocm_smi_lib rocprofiler  rocmvalidationsuite)
 $(call adddep,rocalution,rocblas rocsparse rocrand lightning hipcc)
-$(call adddep,rocblas,hip_on_rocclr openmp_extras lightning hipcc hipblaslt)
+$(call adddep,rocblas,rocminfo hip_on_rocclr openmp_extras lightning hipcc hipblaslt)
 $(call adddep,rocal,mivisionx)
-$(call adddep,rocdecode,hip_on_rocclr lightning hipcc amdmigraphx)
+$(call adddep,rocdecode,hip_on_rocclr lightning hipcc)
 $(call adddep,rocfft,hip_on_rocclr rocrand hiprand lightning hipcc openmp_extras)
-$(call adddep,rocjpeg,hip_on_rocclr lightning hipcc rocm-dev)
+$(call adddep,rocjpeg,hip_on_rocclr lightning hipcc)
 $(call adddep,rocmvalidationsuite,hip_on_rocclr rocr hipblas hiprand hipblaslt rocm-core lightning hipcc rocm_smi_lib)
 $(call adddep,rocprim,hip_on_rocclr lightning hipcc)
 $(call adddep,rocrand,hip_on_rocclr lightning hipcc)
+$(call adddep,rocshmem,rccl )
 $(call adddep,rocsolver,hip_on_rocclr rocblas rocsparse rocprim lightning hipcc)
 $(call adddep,rocsparse,hip_on_rocclr rocprim lightning hipcc)
 $(call adddep,rocthrust,hip_on_rocclr rocprim lightning hipcc)
 $(call adddep,rocwmma,hip_on_rocclr rocblas lightning hipcc rocm-cmake rocm_smi_lib)
 $(call adddep,rpp,half lightning hipcc openmp_extras)
 $(call adddep,transferbench,hip_on_rocclr lightning hipcc)
+ifneq ($(filter rocm-dev upload-rocm-dev, ${MAKECMDGOALS}),)
+	components = $(rocm-dev-components)
+endif
+$(call adddep,rocm,$(filter-out ${NOBUILD} rocm,${components}))
 
+ifeq ($(DISTRO_NAME),rhel)
+    WHL_GEN :=
+endif
 
 # -------------------------------------------------------------------------
 # The rest of the file is internal
@@ -165,7 +196,7 @@ ifeq (${toplevel},)
 define toplevel =
 
 # The "target" make, this builds the package if it is out of date
-T_$1: ${OUT_DIR}/logs/$1 FRC
+T_$1: ${OUT_DIR}/logs/$1.txt FRC
 	:              $1 built
 
 # The "upload" for $1, it uploads the packages for $1 to the central storage
@@ -176,25 +207,26 @@ U_$1: T_$1 FRC
 # The "clean" for $1, it just marks the target as not existing so it will be built
 # in the future.
 C_$1: FRC
-	rm -f ${OUT_DIR}/logs/$1 ${OUT_DIR}/logs/$1.repackaged
+	rm -f ${OUT_DIR}/logs/$1.txt ${OUT_DIR}/logs/$1.repackaged
 
-# parallel build {
-${OUT_DIR}/logs/$1: | ${OUT_DIR}/logs
-ifneq ($(wildcard ${OUT_DIR}/logs/$1.repackaged),)
+# parallel build
+${OUT_DIR}/logs/$1.txt: | ${OUT_DIR}/logs
+ifneq ($(wildcard ${OUT_DIR}/logs/$1.repackaged),) # {
 	@echo  Skipping build of $1 as it has already been repackaged
-	cat $$@.repackaged > $$@
-	rm -f $$@.repackaged
+	cat $${@:.txt=.repackaged} > $$@
+	rm -f $${@:.txt=.repackaged}
 else # } {
 	@echo  $1 started due to $$? | sed "s:${OUT_DIR}/logs/::g"
 # Build in a subshell so we get the time output
 # Pass in jobserver info using the RMAKE variable
-	${RMAKE}@( if set -x && source $${INFRA_REPO}/envsetup.sh && \
-	rm -f $$@.errors $$@ $$@.repackaged && \
-	$${INFRA_REPO}/build_$1.sh -c && \
-	time bash -x $${INFRA_REPO}/build_$1.sh $${RELEASE_FLAG} $${SANITIZER_FLAG} && $${INFRA_REPO}/post_inst_pkg.sh "$1" ; \
-	then mv $$@.inprogress $$@ ; \
-	else mv $$@.inprogress $$@.errors ; echo Error in $1 >&2 ; exit 1 ;\
-	fi ) > $$@.inprogress 2>&1
+# Allow project specific flags e..g. ROCMBUILD_lightning.
+	${RMAKE}${SILENT}( if set -x && source $${INFRA_REPO}/envsetup.sh && \
+	rm -f $${@:$1.txt=1.Errors.$1.txt} $$@ $${@:.txt=.repackaged} && \
+	$${INFRA_REPO}/runner $1 $${RELEASE_FLAG} $${SANITIZER_FLAG} $${STATIC_FLAG} ${ROCMBUILD_$1}; \
+	then mv $${@:$1.txt=2.Inprogress.$1.txt} $$@ ; \
+	else mv $${@:$1.txt=2.Inprogress.$1.txt} $${@:$1.txt=1.Errors.$1.txt} ;\
+		echo Error in $1 >&2 ; exit 1 ;\
+	fi ) > $${@:$1.txt=2.Inprogress.$1.txt} 2>&1
 endif # }
 
 # end of toplevel macro
@@ -227,22 +259,45 @@ upload-rocm-dev: $(addprefix U_,$(filter-out ${NOBUILD},${components}))
 rocm-dev: $(addprefix T_,$(filter-out ${NOBUILD},${components}))
 	@echo rocm-dev built
 
+ifeq ($(DISTRO_NAME),almalinux)
+	@sudo chmod -R 777 "/home/builder"
+endif
+
+# This code is broken. It stops us exiting a container and
+# starting a new one and continueing the build. The attempt
+# is to have run-once code.
 ${OUT_DIR}/logs:
 	sudo mkdir -p -m 775 "${ROCM_INSTALL_PATH}" && \
 	sudo chown -R "$(shell id -u):$(shell id -g)" "/opt"
-	sudo chown -R "$(shell id -u):$(shell id -g)" "/home/$(shell id -un)"
+	sudo chown "$(shell id -u):$(shell id -g)" "/home/$(shell id -un)"
 	mkdir -p "${@}"
 	mkdir -p ${HOME}/.ccache
 
 ##help clean: remove the output directory and recreate it
 clean:
 	[ -n "${OUT_DIR}" ] && rm -rf "${OUT_DIR}"
-	mkdir -p ${OUT_DIR}/logs
+#	mkdir -p ${OUT_DIR}/logs
 
 .SECONDARY: ${components:%=${OUT_DIR}/logs/%}
 
 .PHONY: all clean repack help list_components
 
+# get_all_deps: Recursively get all dependencies for a given component.
+# Usage: $(call get_all_deps,component_name,)
+# - component_name: The name of the component to get dependencies for.
+# - The second parameter is an internal parameter used to track already
+#   processed components to avoid circular dependencies.
+define get_all_deps
+$(if $(filter $(1),$(2)),,\
+	$(sort $(1) $(foreach d,$($(1)_DEPS),$(call get_all_deps,$d,$(1) $(2))))
+)
+endef
+
+##help deps_<component>: output the dependencies for <component>
+deps_%:
+	@echo "=== Dependencies for [$*] ==="
+	@echo "$(filter-out $*,$(call get_all_deps,$*,))"
+
 ##help list_components: output the list of components
 ##help : Hint make list_components | paste - - - | column -t
 list_components:

tools/rocm-build/build_amd_smi_lib.sh

@@ -1,4 +1,3 @@
-#!/bin/bash
 
 source "$(dirname "${BASH_SOURCE}")/compute_utils.sh"
 
@@ -10,7 +9,9 @@ printUsage() {
     echo "  -c,  --clean              Removes all amd_smi build artifacts"
     echo "  -r,  --release            Build non-debug version amd_smi (default is debug)"
     echo "  -a,  --address_sanitizer  Enable address sanitizer"
-    echo "  -s,  --static             Build static lib (.a).  build instead of dynamic/shared(.so) "
+    echo "  -s,  --static             Component/Build does not support static builds just accepting this param & ignore. No effect of the param on this build"
+    echo "  -w,  --wheel              Creates python wheel package of amd-smi. 
+                                      It needs to be used along with -r option"
     echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of type referred to by pkg_type"
     echo "  -p,  --package <type>     Specify packaging format"
     echo "  -h,  --help               Prints this help"
@@ -25,7 +26,6 @@ printUsage() {
 PROJ_NAME="amdsmi"
 PACKAGE_ROOT="$(getPackageRoot)"
 TARGET="build"
-
 PACKAGE_LIB=$(getLibPath)
 PACKAGE_INCLUDE="$(getIncludePath)"
 AMDSMI_BUILD_DIR=$(getBuildPath $PROJ_NAME)
@@ -42,7 +42,7 @@ SHARED_LIBS="ON"
 CLEAN_OR_OUT=0;
 PKGTYPE="deb"
 
-VALID_STR=`getopt -o hcraso:p: --long help,clean,release,static,address_sanitizer,outdir:,package: -- "$@"`
+VALID_STR=`getopt -o hcraswo:p: --long help,clean,release,static,wheel,address_sanitizer,outdir:,package: -- "$@"`
 eval set -- "$VALID_STR"
 
 while true ;
@@ -60,6 +60,8 @@ do
                 ADDRESS_SANITIZER=true ; shift ;;
         (-s | --static)
                 ack_and_skip_static ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ; shift ;;
         (-o | --outdir)
                 TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 2 ;;
         (-p | --package)
@@ -135,7 +137,7 @@ verifyEnvSetup
 
 case $TARGET in
     (clean) clean_amdsmi ;;
-    (build) build_amdsmi ;;
+    (build) build_amdsmi; build_wheel "$AMDSMI_BUILD_DIR" "$PROJ_NAME" ;;
     (outdir) print_output_directory ;;
     (*) die "Invalid target $TARGET" ;;
 esac

tools/rocm-build/build_amdmigraphx.sh

@@ -1,41 +1,42 @@
 #!/bin/bash
 
 set -ex
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src AMDMIGraphX
 
 build_amdmigraphx() {
     echo "Start build"
 
+    if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
+        ack_and_skip_static
+    fi
+
     cd $COMPONENT_SRC
 
     if ! command -v rbuild &> /dev/null; then
         pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
     fi
 
+    # Remove CK
+    xargs -d '\n' -a ${OUT_DIR}/ck.files rm -- || true
+
     if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
          set_asan_env_vars
          set_address_sanitizer_on
     fi
 
-    if [ -n "$GPU_ARCHS" ]; then
-        GPU_TARGETS="$GPU_ARCHS"
-    else
-        GPU_TARGETS="gfx900;gfx906;gfx908;gfx90a;gfx1030;gfx1100;gfx1101;gfx1102;gfx942;gfx1200;gfx1201"
-    fi
     init_rocm_common_cmake_params
 
     mkdir -p ${BUILD_DIR} && rm -rf ${BUILD_DIR}/* && mkdir -p ${HOME}/amdmigraphx && rm -rf ${HOME}/amdmigraphx/*
     rbuild package -d "${HOME}/amdmigraphx" -B "${BUILD_DIR}" \
-        --cxx="${ROCM_PATH}/llvm/bin/clang++" \
-        --cc="${ROCM_PATH}/llvm/bin/clang" \
+        --cxx="$(set_build_variables __CLANG++__)" \
+        --cc="$(set_build_variables __CLANG__)" \
         "${rocm_math_common_cmake_params[@]}" \
         -DCMAKE_MODULE_LINKER_FLAGS="-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ROCM_LIB_RPATH" \
-	    -DGPU_TARGETS="${GPU_TARGETS}" \
         -DCMAKE_INSTALL_RPATH=""
 
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
     cd $BUILD_DIR && cmake --build . -- install -j${PROC}
 
     show_build_cache_stats
@@ -50,7 +51,7 @@ clean_amdmigraphx() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_amdmigraphx ;;
+    build) build_amdmigraphx; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_amdmigraphx ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_clang-ocl.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+
+source "$(dirname "${BASH_SOURCE}")/compute_utils.sh"
+
+printUsage() {
+    echo
+    echo "Usage: $(basename "${BASH_SOURCE}") [-c|-r|-h] [makeopts]"
+    echo
+    echo "Options:"
+    echo "  -c,  --clean              Removes all clang-ocl build artifacts"
+    echo "  -r,  --release            Build non-debug version clang-ocl (default is debug)"
+    echo "  -a,  --address_sanitizer  Enable address sanitizer"
+    echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of
+                                      type referred to by pkg_type"
+    echo "  -h,  --help               Prints this help"
+    echo "  -s,  --static             Supports static CI by accepting this param & not bailing out. No effect of the param though"
+    echo
+
+    return 0
+}
+
+
+TARGET="build"
+CLANG_OCL_DEST="$(getBinPath)"
+CLANG_OCL_SRC_ROOT="$CLANG_OCL_ROOT"
+CLANG_OCL_BUILD_DIR="$(getBuildPath clang-ocl)"
+
+MAKEARG="$DASH_JAY"
+PACKAGE_ROOT="$(getPackageRoot)"
+PACKAGE_UTILS="$(getUtilsPath)"
+CLANG_OCL_PACKAGE_DEB="$PACKAGE_ROOT/deb/clang-ocl"
+CLANG_OCL_PACKAGE_RPM="$PACKAGE_ROOT/rpm/clang-ocl"
+BUILD_TYPE="Debug"
+SHARED_LIBS="ON"
+CLEAN_OR_OUT=0;
+MAKETARGET="deb"
+PKGTYPE="deb"
+
+
+VALID_STR=`getopt -o hcraso:g: --long help,clean,release,clean,static,address_sanitizer,outdir:,gpu_list: -- "$@"`
+eval set -- "$VALID_STR"
+
+while true ;
+do
+    case "$1" in
+        (-h | --help)
+                printUsage ; exit 0;;
+        (-c | --clean)
+                TARGET="clean" ; ((CLEAN_OR_OUT|=1)) ; shift ;;
+        (-r | --release)
+                MAKEARG="$MAKEARG BUILD_TYPE=rel" ;  BUILD_TYPE="Release" ; shift ;;
+        (-a | --address_sanitizer)
+                set_asan_env_vars
+                set_address_sanitizer_on ; shift ;;
+        (-s | --static)
+                SHARED_LIBS="OFF" ; shift ;;
+        (-o | --outdir)
+                TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 2 ;;
+        (-g | --gpu_list )
+                GPU_LIST=$2; shift 2 ;;
+        --)     shift; break;;
+        (*)
+                echo " This should never come but just incase : UNEXPECTED ERROR Parm : [$1] ">&2 ; exit 20;;
+    esac
+
+done
+
+RET_CONFLICT=1
+check_conflicting_options $CLEAN_OR_OUT $PKGTYPE $MAKETARGET
+if [ $RET_CONFLICT -ge 30 ]; then
+   print_vars $API_NAME $TARGET $BUILD_TYPE $SHARED_LIBS $CLEAN_OR_OUT $PKGTYPE $MAKETARGET
+   exit $RET_CONFLICT
+fi
+
+clean_clang-ocl() {
+    echo "Removing clang-ocl"
+    rm -rf $CLANG_OCL_DEST/clang-ocl
+    rm -rf $CLANG_OCL_BUILD_DIR
+    rm -rf $CLANG_OCL_PACKAGE_DEB
+    rm -rf $CLANG_OCL_PACKAGE_RPM
+}
+
+build_clang-ocl() {
+    if [ ! -d "$CLANG_OCL_BUILD_DIR" ]; then
+        mkdir -p $CLANG_OCL_BUILD_DIR
+        pushd $CLANG_OCL_BUILD_DIR
+
+        if [ -e $PACKAGE_ROOT/lib/bitcode/opencl.amdgcn.bc ]; then
+            BC_DIR="$ROCM_INSTALL_PATH/lib"
+        else
+            BC_DIR="$ROCM_INSTALL_PATH/amdgcn/bitcode"
+        fi
+
+        cmake \
+            $(rocm_cmake_params) \
+            -DDISABLE_CHECKS="ON" \
+            -DCLANG_BIN="$ROCM_INSTALL_PATH/llvm/bin" \
+            -DBITCODE_DIR="$BC_DIR" \
+	    $(rocm_common_cmake_params) \
+            -DCPACK_SET_DESTDIR="OFF" \
+            $CLANG_OCL_SRC_ROOT
+
+        echo "Making clang-ocl:"
+        cmake --build . -- $MAKEARG
+        cmake --build . -- $MAKEARG install
+        cmake --build . -- $MAKEARG package
+        popd
+    fi
+
+    copy_if DEB "${CPACKGEN:-"DEB;RPM"}" "$CLANG_OCL_PACKAGE_DEB" $CLANG_OCL_BUILD_DIR/rocm-clang-ocl*.deb
+    copy_if RPM "${CPACKGEN:-"DEB;RPM"}" "$CLANG_OCL_PACKAGE_RPM" $CLANG_OCL_BUILD_DIR/rocm-clang-ocl*.rpm
+}
+
+
+print_output_directory() {
+     case ${PKGTYPE} in
+         ("deb")
+             echo ${CLANG_OCL_PACKAGE_DEB};;
+         ("rpm")
+             echo ${CLANG_OCL_PACKAGE_RPM};;
+         (*)
+             echo "Invalid package type \"${PKGTYPE}\" provided for -o" >&2; exit 1;;
+     esac
+     exit
+}
+
+case $TARGET in
+    (clean) clean_clang-ocl ;;
+    (build) build_clang-ocl ;;
+   (outdir) print_output_directory ;;
+        (*) die "Invalid target $TARGET" ;;
+esac
+
+echo "Operation complete"
+exit 0
+

tools/rocm-build/build_comgr.sh

@@ -14,6 +14,7 @@ printUsage() {
         type referred to by pkg_type"
     echo "  -h,  --help               Prints this help"
     echo "  -s,  --static             Build static lib (.a).  build instead of dynamic/shared(.so) "
+    echo "  -w,  --wheel              Creates python wheel package of comgr. It needs to be used along with -r option"
     echo "  -l,  --link_llvm_static   Link to LLVM statically.  Default is to dynamically link to LLVM; requires that LLVM dylibs are created."
     echo
     echo "Possible values for <type>:"
@@ -24,6 +25,7 @@ printUsage() {
     return 0
 }
 
+## Build environment variables
 API_NAME=amd_comgr
 PROJ_NAME=$API_NAME
 LIB_NAME=lib${API_NAME}
@@ -33,8 +35,8 @@ PACKAGE_ROOT=$(getPackageRoot)
 PACKAGE_LIB=$(getLibPath)
 PACKAGE_INCLUDE=$(getIncludePath)
 BUILD_DIR=$(getBuildPath $API_NAME)
-PACKAGE_DEB=$(getPackageRoot)/deb/$API_NAME
-PACKAGE_RPM=$(getPackageRoot)/rpm/$API_NAME
+PACKAGE_DEB=$PACKAGE_ROOT/deb/$PROJ_NAME
+PACKAGE_RPM=$PACKAGE_ROOT/rpm/$PROJ_NAME
 PACKAGE_PREFIX=$ROCM_INSTALL_PATH
 BUILD_TYPE=Debug
 MAKE_OPTS="$DASH_JAY CTEST_OUTPUT_ON_FAILURE=1 -C $BUILD_DIR"
@@ -44,13 +46,17 @@ CLEAN_OR_OUT=0;
 PKGTYPE="deb"
 MAKETARGET="deb"
 
+# link to LLVM dynamicaly.  Default is to link to llvm dynamically.
+# temporarily set this default to OFF until we resolve Issues
 LINK_LLVM_DYLIB="OFF"
 
-VALID_STR=`getopt -o hcraslo:p: --long help,clean,release,address_sanitizer,static,link_llvm_static,outdir:,package: -- "$@"`
+#parse the arguments
+VALID_STR=`getopt -o hcraswlo:p: --long help,clean,release,address_sanitizer,static,wheel,link_llvm_static,outdir:,package: -- "$@"`
 eval set -- "$VALID_STR"
 
 while true ;
 do
+    #echo "parocessing $1"
     case "$1" in
         (-h | --help)
                 printUsage ; exit 0;;
@@ -63,6 +69,8 @@ do
                 set_address_sanitizer_on ; shift ;;
         (-s | --static)
                 SHARED_LIBS="OFF" ; shift ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ; shift ;;
         (-l | --link_llvm_static)
                 LINK_LLVM_DYLIB="OFF"; shift ;;
         (-o | --outdir)
@@ -105,6 +113,7 @@ build() {
        echo " Building Shared Object "
     fi
 
+# Remove CTEST var once SWDEV-381396 is fixed
     cmake \
         $(rocm_cmake_params) \
         -DBUILD_SHARED_LIBS=$SHARED_LIBS \
@@ -143,7 +152,7 @@ verifyEnvSetup
 
 case $TARGET in
     (clean) clean ;;
-    (build) build ;;
+    (build) build; build_wheel "$BUILD_DIR" "$PROJ_NAME" ;;
     (outdir) print_output_directory ;;
     (*) die "Invalid target $TARGET" ;;
 esac

tools/rocm-build/build_composable_kernel.sh

@@ -2,10 +2,14 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
+
+# Temporarily disable Address Sanitizer
+ENABLE_ADDRESS_SANITIZER=false
 
 set_component_src composable_kernel
-
+disable_debug_package_generation
+# Set the GPU_ARCH_LIST to the supported GPUs needed after https://github.com/ROCm/composable_kernel/pull/1536/
 GPU_ARCH_LIST="gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201"
 
 build_miopen_ck() {
@@ -23,10 +27,17 @@ build_miopen_ck() {
         GPU_ARCH_LIST="gfx942"
         ack_and_skip_static
     fi
-
+    # if ENABLE_GPU_ARCH is set in env by Job parameter ENABLE_GPU_ARCH, then set GPU_ARCH_LIST to that value
+    # This needs to be removed when CK aligns with other component and uses -DAMDGPU_TARGET or _DGPO_TARGET
+    # then we can use set_gpu_arch from compure_helper.sh and get rid of all if clauses
+    if [ -n "$ENABLE_GPU_ARCH" ]; then
+        GPU_ARCH_LIST="$ENABLE_GPU_ARCH"
+    fi
+    # Latest CK requiring Python 3.8 as the minimum.
+    # Point CMake to that explicit location and adjust LD_LIBRARY_PATH.
     PYTHON_VERSION_WORKAROUND=''
     echo "DISTRO_ID: ${DISTRO_ID}"
-    if [ "$DISTRO_ID" = "rhel-8.8" ] || [ "$DISTRO_ID" = "sles-15.5" ] ; then
+    if [ "$DISTRO_ID" = "rhel-8.8" ] || [ "$DISTRO_NAME" == "sles" ] || [ "$DISTRO_ID" = "debian-10" ]; then
         EXTRA_PYTHON_PATH=/opt/Python-3.8.13
         PYTHON_VERSION_WORKAROUND="-DCK_USE_ALTERNATIVE_PYTHON=${EXTRA_PYTHON_PATH}/bin/python3.8"
         # For the python interpreter we need to export LD_LIBRARY_PATH.
@@ -42,19 +53,20 @@ build_miopen_ck() {
         "${rocm_math_common_cmake_params[@]}" \
         ${PYTHON_VERSION_WORKAROUND} \
         -DCPACK_GENERATOR="${PKGTYPE^^}" \
-        -DCMAKE_CXX_COMPILER="${ROCM_PATH}/llvm/bin/clang++" \
-        -DCMAKE_C_COMPILER="${ROCM_PATH}/llvm/bin/clang" \
+        -DCMAKE_CXX_COMPILER=$(set_build_variables __CLANG++__) \
+        -DCMAKE_C_COMPILER=$(set_build_variables __CLANG__) \
         ${LAUNCHER_FLAGS} \
         -DGPU_ARCHS="${GPU_ARCH_LIST}" \
-        -DCMAKE_BUILD_TYPE=Release \
-        -DCMAKE_CXX_FLAGS=" -O3 " \
         "$COMPONENT_SRC"
 
     cmake --build . -- -j${PROC} package
     cmake --build "$BUILD_DIR" -- install
-    mkdir -p $PACKAGE_DIR && cp ./*.${PKGTYPE} $PACKAGE_DIR
+
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 }
 
+# Use the function to unset the LDFLAGS and CXXFLAGS
+# specifically set for ASAN
 unset_asan_env_vars() {
     ASAN_CMAKE_PARAMS="false"
     export ADDRESS_SANITIZER="OFF"
@@ -68,85 +80,6 @@ set_address_sanitizer_off() {
     export LDFLAGS=""
 }
 
-build_miopen_ckProf() {
-    ENABLE_ADDRESS_SANITIZER=false
-    echo "Start Building Composable Kernel Profiler"
-    if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
-       set_asan_env_vars
-       set_address_sanitizer_on
-    else
-       unset_asan_env_vars
-       set_address_sanitizer_off
-    fi
-
-    cd $COMPONENT_SRC
-    cd "$BUILD_DIR"
-    rm -rf *
-
-    architectures='gfx10 gfx11 gfx90 gfx94'
-    if [ -n "$GPU_ARCHS" ]; then
-        architectures=$(echo ${GPU_ARCHS} | awk -F';' '{for(i=1;i<=NF;i++) a[substr($i,1,5)]} END{for(i in a) printf i" "}')
-    fi
-
-    for arch in ${architectures}
-        do
-            if [ "${ASAN_CMAKE_PARAMS}" == "true" ] ; then
-                cmake -DBUILD_DEV=OFF \
-                    -DCMAKE_PREFIX_PATH="${ROCM_PATH%-*}/lib/cmake;${ROCM_PATH%-*}/$ASAN_LIBDIR;${ROCM_PATH%-*}/llvm;${ROCM_PATH%-*}" \
-                    -DCMAKE_BUILD_TYPE=${BUILD_TYPE:-'RelWithDebInfo'} \
-                    -DCMAKE_SHARED_LINKER_FLAGS_INIT="-Wl,--enable-new-dtags,--rpath,$ROCM_ASAN_LIB_RPATH" \
-                    -DCMAKE_EXE_LINKER_FLAGS_INIT="-Wl,--enable-new-dtags,--rpath,$ROCM_ASAN_EXE_RPATH" \
-                    -DCMAKE_VERBOSE_MAKEFILE=1 \
-                    -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE \
-                    -DCMAKE_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DCMAKE_PACKAGING_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF \
-                    -DROCM_SYMLINK_LIBS=OFF \
-                    -DCPACK_PACKAGING_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DROCM_DISABLE_LDCONFIG=ON \
-                    -DROCM_PATH="${ROCM_PATH}" \
-                    -DCPACK_GENERATOR="${PKGTYPE^^}" \
-                    -DCMAKE_CXX_COMPILER="${ROCM_PATH}/llvm/bin/clang++" \
-                    -DCMAKE_C_COMPILER="${ROCM_PATH}/llvm/bin/clang" \
-                    ${LAUNCHER_FLAGS} \
-                    -DPROFILER_ONLY=ON \
-                    -DENABLE_ASAN_PACKAGING=true \
-                    -DGPU_ARCH="${arch}" \
-                    "$COMPONENT_SRC"
-            else
-                cmake -DBUILD_DEV=OFF \
-                    -DCMAKE_PREFIX_PATH="${ROCM_PATH%-*}" \
-                    -DCMAKE_BUILD_TYPE=Release \
-                    -DCMAKE_SHARED_LINKER_FLAGS_INIT='-Wl,--enable-new-dtags,--rpath,$ORIGIN' \
-                    -DCMAKE_EXE_LINKER_FLAGS_INIT='-Wl,--enable-new-dtags,--rpath,$ORIGIN/../lib' \
-                    -DCMAKE_VERBOSE_MAKEFILE=1 \
-                    -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE \
-                    -DCMAKE_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DCMAKE_PACKAGING_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF \
-                    -DROCM_SYMLINK_LIBS=OFF \
-                    -DCPACK_PACKAGING_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DROCM_DISABLE_LDCONFIG=ON \
-                    -DROCM_PATH="${ROCM_PATH}" \
-                    -DCPACK_GENERATOR="${PKGTYPE^^}" \
-                    -DCMAKE_CXX_COMPILER="${ROCM_PATH}/llvm/bin/clang++" \
-                    -DCMAKE_C_COMPILER="${ROCM_PATH}/llvm/bin/clang" \
-                    ${LAUNCHER_FLAGS} \
-                    -DPROFILER_ONLY=ON \
-                    -DGPU_ARCH="${arch}" \
-                    "$COMPONENT_SRC"
-            fi
-
-            cmake --build . -- -j${PROC} package
-            cp ./*ckprofiler*.${PKGTYPE} $PACKAGE_DIR
-            rm -rf *
-        done
-    rm -rf _CPack_Packages/ && find -name '*.o' -delete
-
-    echo "Finished building Composable Kernel"
-    show_build_cache_stats
-}
-
 clean_miopen_ck() {
     echo "Cleaning MIOpen-CK build directory: ${BUILD_DIR} ${PACKAGE_DIR}"
     rm -rf "$BUILD_DIR" "$PACKAGE_DIR"
@@ -156,7 +89,7 @@ clean_miopen_ck() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_miopen_ck ;;
+    build) build_miopen_ck; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_miopen_ck ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_dbgapi.sh

@@ -11,6 +11,8 @@ printUsage() {
     echo "  -r,  --release            Make a release build instead of a debug build"
     echo "       --enable-assertions  Enable assertions"
     echo "  -a,  --address_sanitizer  Enable address sanitizer"
+    echo "  -w,  --wheel              Creates python wheel package of dbgapi. 
+                                      It needs to be used along with -r option"
     echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of
             type referred to by pkg_type"
     echo "  -h,  --help               Prints this help"
@@ -25,6 +27,7 @@ printUsage() {
     return 0
 }
 
+## Build environment variables
 API_NAME=rocm-dbgapi
 AMD_DBGAPI_NAME=amd-dbgapi
 MAKEINSTALL_MANIFEST=makeinstall_manifest.txt
@@ -36,21 +39,24 @@ PACKAGE_ROOT=$(getPackageRoot)
 PACKAGE_LIB=$(getLibPath)
 PACKAGE_INCLUDE=$(getIncludePath)
 BUILD_DIR=$(getBuildPath $API_NAME)
-PACKAGE_DEB=$(getPackageRoot)/deb/$API_NAME
-PACKAGE_RPM=$(getPackageRoot)/rpm/$API_NAME
+PACKAGE_DEB=$(getPackageRoot)/deb/$PROJ_NAME
+PACKAGE_RPM=$(getPackageRoot)/rpm/$PROJ_NAME
+#PACKAGE_PREFIX=$ROCM_INSTALL_PATH
 BUILD_TYPE=Debug
-MAKE_OPTS=($DASH_JAY -C "$BUILD_DIR")
+MAKE_OPTS=($DASH_JAY -C "$BUILD_DIR") # Note that DASH_JAY might have a space after the -j
 SHARED_LIBS="ON"
 CLEAN_OR_OUT=0;
 MAKETARGET="deb"
 PKGTYPE="deb"
 DODOCSBUILD=true
 
-VALID_STR=$(getopt -o hcraso:p:M --long help,clean,release,enable-assertions,static,address_sanitizer,outdir:,package:skip_man_pages -- "$@")
+#parse the arguments
+VALID_STR=$(getopt -o hcraswo:p:M --long help,clean,release,enable-assertions,static,wheel,address_sanitizer,outdir:,package:skip_man_pages -- "$@")
 eval set -- "$VALID_STR"
 
 while true ;
 do
+    #echo "parocessing $1"
     case "$1" in
         (-h | --help)
                 printUsage ; exit 0;;
@@ -66,12 +72,14 @@ do
                 set_address_sanitizer_on ;;
         (-s | --static)
                 ack_and_skip_static ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ;;
         (-o | --outdir)
                 TARGET="outdir"; PKGTYPE=$2 ; ((CLEAN_OR_OUT|=2)) ; shift 1 ;;
         (-M | --skip_man_pages) DODOCSBUILD=false;;
         (-p | --package)
                 MAKETARGET="$2" ; shift 1;;
-        --)     shift; break;;
+        --)     shift; break;; # end delimiter
         (*)
                 echo " This should never come but just incase : UNEXPECTED ERROR Parm : [$1] ">&2 ; exit 20;;
     esac
@@ -104,7 +112,7 @@ build() {
     then
            echo " No $ROCM_DBGAPI_ROOT/CMakeLists.txt file, skipping rocm-dbgapi" >&2
            echo " No $ROCM_DBGAPI_ROOT/CMakeLists.txt file, skipping rocm-dbgapi"
-       exit 0
+       exit 0 # THis is not an error
     fi
     echo "Building $PROJ_NAME"
 
@@ -121,11 +129,14 @@ build() {
     cmake --build "$BUILD_DIR" -- "${MAKE_OPTS[@]}"
     "$DODOCSBUILD" && cmake --build "$BUILD_DIR" -- "${MAKE_OPTS[@]}" doc
     cmake --build "$BUILD_DIR" -- "${MAKE_OPTS[@]}" install
-
+    #install_manifest.txt is created by make install and make package with same name unless
+    #component packaging is enabled. To avoid overwriting by make package,move the manifest file
+    #to a different name and can be used for build clean up
     mv "$BUILD_DIR/install_manifest.txt" "$BUILD_DIR/$MAKEINSTALL_MANIFEST"
     cmake --build "$BUILD_DIR" -- "${MAKE_OPTS[@]}" package
 
     mkdir -p "$PACKAGE_LIB"
+    # handle the library being in more than one place to avoid breaking the build.
     (
         shopt -s nullglob
         cp -R "$BUILD_DIR/lib/${LIB_NAME}"* "$BUILD_DIR/${LIB_NAME}"* "$PACKAGE_LIB"
@@ -151,7 +162,7 @@ verifyEnvSetup
 
 case $TARGET in
     (clean) clean ;;
-    (build) build ;;
+    (build) build; build_wheel "$BUILD_DIR" "$PROJ_NAME";;
     (outdir) print_output_directory ;;
     (*) die "Invalid target $TARGET" ;;
 esac

tools/rocm-build/build_devicelibs.sh

@@ -10,7 +10,9 @@ printUsage() {
     echo "  -c,  --clean              Clean output and delete all intermediate work"
     echo "  -r,  --release            Build a release version of the package"
     echo "  -a,  --address_sanitizer  Enable address sanitizer"
-    echo "  -s,  --static             Supports static CI by accepting this param & not bailing out. No effect of the param though"
+    echo "  -s,  --static             Build static lib (.a).  build instead of dynamic/shared(.so) "
+    echo "  -w,  --wheel              Creates python wheel package of devicelibs.
+                                      It needs to be used along with -r option"
     echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of
     type referred to by pkg_type"
     echo "  -h,  --help             Prints this help"
@@ -19,10 +21,11 @@ printUsage() {
 
     return 0
 }
+PROJ_NAME="devicelibs"
 PACKAGE_ROOT="$(getPackageRoot)"
 PACKAGE_BIN="$(getBinPath)"
 PACKAGE_LIB="$(getLibPath)"
-BUILD_PATH="$(getBuildPath devicelibs)"
+BUILD_PATH="$(getBuildPath $PROJ_NAME)"
 INSTALL_PATH="$(getPackageRoot)"
 LIGHTNING_BUILD_PATH="$(getBuildPath lightning)"
 DEB_PATH="$(getDebPath devicelibs)"
@@ -36,11 +39,13 @@ CLEAN_OR_OUT=0;
 PKGTYPE="deb"
 MAKETARGET="deb"
 
-VALID_STR=`getopt -o hcraso: --long help,clean,release,static,address_sanitizer,outdir: -- "$@"`
+#parse the arguments
+VALID_STR=`getopt -o hcraswo: --long help,clean,release,static,wheel,address_sanitizer,outdir: -- "$@"`
 eval set -- "$VALID_STR"
 
 while true ;
 do
+    #echo "parocessing $1"
     case "$1" in
         (-h | --help)
                 printUsage ; exit 0;;
@@ -53,6 +58,8 @@ do
                 ack_and_ignore_asan ; shift ;;
         (-s | --static)
                 SHARED_LIBS="OFF" ; shift ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ; shift ;;
         (-o | --outdir)
                 TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 2 ;;
         --)     shift; break;; # end delimiter
@@ -71,7 +78,9 @@ fi
 
 
 clean_devicelibs() {
+    # Delete cmake output directory
     rm -rf "$BUILD_PATH"
+    # Delete the *.bc files from the PACKAGE_LIB directory
     rm -f $PACKAGE_LIB/hc*.bc
     rm -f $PACKAGE_LIB/irif*.bc
     rm -f $PACKAGE_LIB/ockl*.bc
@@ -80,6 +89,7 @@ clean_devicelibs() {
     rm -f $PACKAGE_LIB/opencl*.bc
     rm -f $PACKAGE_LIB/openmp*.bc
     rm -rf $PACKAGE_ROOT/amdgcn
+    # Delete any packages generated
     rm -rf "$DEB_PATH"
     rm -rf "$RPM_PATH"
 }
@@ -96,6 +106,7 @@ build_devicelibs() {
     if [ ! -e Makefile ]; then
         cmake $(rocm_cmake_params) \
               $(rocm_common_cmake_params) \
+              ${GEN_NINJA} \
               -DBUILD_SHARED_LIBS=$SHARED_LIBS \
               -DROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_NEW="$bitcodeInstallLoc/amdgcn" \
               -DROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_OLD="amdgcn" \
@@ -133,7 +144,7 @@ print_output_directory() {
 }
 case $TARGET in
     (clean) clean_devicelibs ;;
-    (build) build_devicelibs; package_devicelibs ;;
+    (build) build_devicelibs; package_devicelibs; build_wheel "$BUILD_PATH" "$PROJ_NAME" ;;
     (outdir) print_output_directory ;;
     (*) die "Invalid target $TARGET" ;;
 esac

tools/rocm-build/build_half.sh

@@ -1,22 +1,31 @@
 #!/bin/bash
 
 set -ex
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src half
 
 build_half() {
     echo "Start build"
 
+    if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
+        ack_and_skip_static
+    fi
+
     if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
          set_asan_env_vars
          set_address_sanitizer_on
+         # ASAN packaging is not required for HALF, since its header only package
+         # Setting the asan_cmake_params to false will disable ASAN packaging
          ASAN_CMAKE_PARAMS="false"
     fi
     mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
 
     cmake \
         -DCMAKE_INSTALL_PREFIX="$ROCM_PATH" \
+        -DCPACK_PACKAGING_INSTALL_PREFIX=${ROCM_PATH}\
+        -DCPACK_SET_DESTDIR="OFF" \
+        -DCPACK_RPM_PACKAGE_RELOCATABLE="ON" \
         -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF \
         "$COMPONENT_SRC"
 
@@ -25,7 +34,7 @@ build_half() {
     cmake --build "$BUILD_DIR" -- install
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -39,7 +48,7 @@ clean_half() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_half ;;
+    build) build_half; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_half ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hip_on_rocclr.sh

@@ -10,6 +10,8 @@ printUsage() {
     echo "  -r,  --release              Make a release build instead of a debug build"
     echo "  -a,  --address_sanitizer    Enable address sanitizer"
     echo "  -s,  --static               Build static lib (.a).  build instead of dynamic/shared(.so) "
+    echo "  -w,  --wheel                Creates python wheel package of hip-on-rocclr.
+                                        It needs to be used along with -r option"
     echo "  -o,  --outdir <pkg_type>    Print path of output directory containing packages of type referred to by pkg_type"
     echo "  -t,  --offload-arch=<arch>  Specify arch for catch tests ex: --offload-arch=gfx1030 --offload-arch=gfx1100"
     echo "  -p,  --package <type>       Specify packaging format"
@@ -25,7 +27,6 @@ printUsage() {
 source "$(dirname "${BASH_SOURCE}")/compute_utils.sh"
 MAKEOPTS="$DASH_JAY"
 PROJ_NAME="hip-on-rocclr"
-
 BUILD_PATH="$(getBuildPath $PROJ_NAME)"
 
 TARGET="build"
@@ -53,9 +54,10 @@ MAKETARGET="deb"
 PKGTYPE="deb"
 OFFLOAD_ARCH=()
 
-DEFAULT_OFFLOAD_ARCH=(gfx900 gfx906 gfx908 gfx90a gfx940 gfx941 gfx942 gfx1030 gfx1031 gfx1033 gfx1034 gfx1035 gfx1100 gfx1101 gfx1102 gfx1200 gfx1201)
+DEFAULT_OFFLOAD_ARCH=(gfx900 gfx906 gfx908 gfx90a gfx940 gfx941 gfx942 gfx1030 gfx1031 gfx1033 gfx1034 gfx1035 gfx1100 gfx1101 gfx1102 gfx1103 gfx1150 gfx1151 gfx1200 gfx1201)
 
-VALID_STR=`getopt -o hcrast:o: --long help,clean,release,address_sanitizer,static,offload-arch=:,outdir: -- "$@"`
+#parse the arguments
+VALID_STR=`getopt -o hcraswt:o: --long help,clean,release,address_sanitizer,static,wheel,offload-arch=:,outdir: -- "$@"`
 eval set -- "$VALID_STR"
 
 while true ;
@@ -72,11 +74,13 @@ do
                 set_address_sanitizer_on ; shift ;;
         (-s | --static)
                 SHARED_LIBS="OFF" ; shift ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ; shift ;;
         (-t | --offload-arch=)
                 OFFLOAD_ARCH+=( "$2" ); ((CLEAN_OR_OUT|=2)); shift 2 ;;
         (-o | --outdir)
                 TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 2 ;;
-        --)     shift; break;;
+        --)     shift; break;; # end delimiter
 
         (*)
                 echo " This should never come but just incase : UNEXPECTED ERROR Parm : [$1] ">&2 ; exit 20;;
@@ -98,7 +102,10 @@ if [ $RET_CONFLICT -ge 30 ]; then
    exit $RET_CONFLICT
 fi
 
+
+
 clean_hip_on_rocclr() {
+    # Delete cmake output directory
     rm -rf "$BUILD_PATH"
     rm -rf "$PACKAGE_DEB"
     rm -rf "$PACKAGE_RPM"
@@ -106,17 +113,23 @@ clean_hip_on_rocclr() {
 }
 
 build_hip_on_rocclr() {
+    # TODO This if condition is a temporary workaround so that mainline builds dont error out
+    # until build migrated from hipamd to clr repo
     if [  -e "$CLR_ROOT/CMakeLists.txt" ]; then
+        # We are in a branch that has migrated to clr repo
         _HIP_CMAKELIST_DIR="$CLR_ROOT"
         _HIP_CMAKELIST_OPT="-DCLR_BUILD_HIP=ON -DCLR_BUILD_OCL=OFF"
         if [ -e "$HIPOTHER_ROOT/hipnv" ]; then
+            # We are in a branch that has hipnv headers migrated
             _HIP_CMAKELIST_OPT="$_HIP_CMAKELIST_OPT -DHIPNV_DIR=$HIPOTHER_ROOT/hipnv"
         fi
     elif [ ! -e "$HIPAMD_ROOT/CMakeLists.txt" ]; then
+        # We seem to have hit a branch in which both the old and the new repo don't exist
         echo "No $HIPAMD_ROOT/CMakeLists.txt file, skipping hip on rocclr" >&2
         echo "No $HIPAMD_ROOT/CMakeLists.txt file, skipping hip on rocclr"
-        exit 0
+        exit 0 # This is not an error
     else
+        # We are in a branch that has not yet migrated to clr repo yet
         _HIP_CMAKELIST_DIR="$HIPAMD_ROOT"
         _HIP_CMAKELIST_OPT=""
     fi
@@ -125,6 +138,7 @@ build_hip_on_rocclr() {
     mkdir -p "$BUILD_PATH"
     pushd "$BUILD_PATH"
 
+    # FIXME: Remove -DROCclr_DIR/LIBROCclr_STATIC_DIR
     if [ ! -e Makefile ]; then
         echo "Building HIP-On-ROCclr CMake environment"
         print_lib_type $SHARED_LIBS
@@ -165,8 +179,13 @@ build_catch_tests() {
    rm -rf "$CATCH_BUILD_DIR"
    mkdir -p "$CATCH_BUILD_DIR"
    pushd "$CATCH_BUILD_DIR"
+   # use the newly built hip as HIP for catch
    export HIP_PATH="$ROCM_INSTALL_PATH"
    export ROCM_PATH="$ROCM_INSTALL_PATH"
+   # Note: The rocm_common_cmake_params will provide CMAKE_EXE_LINKER_FLAGS_INIT
+   # for binaries located in /opt/rocm/bin
+   # hip-catch binaries are located in each test folder under /opt/rocm/share/hip/catch_test/
+   # Append the EXE LINKER flags with ROCm library path
    cmake \
          -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
          -DBUILD_SHARED_LIBS=$SHARED_LIBS \
@@ -177,6 +196,7 @@ build_catch_tests() {
          $(rocm_common_cmake_params) \
          -DCPACK_RPM_DEBUGINFO_PACKAGE=FALSE \
          -DCPACK_DEBIAN_DEBUGINFO_PACKAGE=FALSE \
+         -DCMAKE_EXE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ROCM_EXE_RPATH:$ROCM_INSTALL_PATH/lib:/opt/rocm/lib \
          -DCPACK_INSTALL_PREFIX="$ROCM_INSTALL_PATH" \
          "$CATCH_SRC"
 
@@ -190,6 +210,7 @@ build_catch_tests() {
 }
 
 package_samples() {
+   # TODO: To be removed once the issue related to ASAN builds are fixed
    if [ "$ASAN_CMAKE_PARAMS" == "true" ] ; then
       echo "Disable the packaging of HIP samples" >&2
       return
@@ -198,13 +219,17 @@ package_samples() {
    if [ ! -e "$SAMPLES_SRC/CMakeLists.txt" ]; then
       echo "HIP samples source not found at: $SAMPLES_SRC" >&2
       echo "Using samples package from hip project: $BUILD_PATH" >&2
+      # TODO: change to return failure after hip-tests samples change is available in mainline
       return
    fi
-
+   # package samples
    rm -rf "$SAMPLES_BUILD_DIR"
    mkdir -p "$SAMPLES_BUILD_DIR"
    pushd "$SAMPLES_BUILD_DIR"
+   # The cmake path is different for asan and non-asan builds.
+   # Fetch after getting build type. Default will be non-asan build
    local CMAKE_PATH="$(getCmakePath)"
+   # use the newly built hip as HIP for samples
    export HIP_PATH="$ROCM_INSTALL_PATH"
    export ROCM_PATH="$ROCM_INSTALL_PATH"
    cmake \
@@ -291,10 +316,24 @@ print_output_directory() {
 }
 
 case $TARGET in
-    (clean) clean_hip_on_rocclr; clean_hip_tests ;;
-    (build) build_hip_on_rocclr; build_catch_tests; package_hip_on_rocclr; package_samples; copy_hip_tests;;
-   (outdir) print_output_directory ;;
-        (*) die "Invalid target $TARGET" ;;
+    (clean)
+        clean_hip_on_rocclr
+        clean_hip_tests
+        ;;
+    (build)
+        build_hip_on_rocclr
+        build_catch_tests
+        package_hip_on_rocclr
+        package_samples
+        build_wheel "$BUILD_PATH" "$PROJ_NAME"
+        copy_hip_tests
+        ;;
+    (outdir)
+        print_output_directory
+        ;;
+    (*)
+        die "Invalid target $TARGET"
+        ;;
 esac
 
 echo "Operation complete"

tools/rocm-build/build_hipblas-common.sh

@@ -2,16 +2,19 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipBLAS-common
 
 build_hipblas-common() {
     echo "Start build"
 
+    CXX=$(set_build_variables __C_++__)
     cd $COMPONENT_SRC
     mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
 
+    echo "C compiler: $CC"
+    echo "CXX compiler: $CXX"
     init_rocm_common_cmake_params
     cmake \
         "${rocm_math_common_cmake_params[@]}" \
@@ -20,8 +23,7 @@ build_hipblas-common() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
-
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
     show_build_cache_stats
 }
 
@@ -34,7 +36,7 @@ clean_hipblas-common() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipblas-common ;;
+    build) build_hipblas-common; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipblas-common ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipblas.sh

@@ -2,28 +2,32 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipBLAS
 
 build_hipblas() {
     echo "Start build"
 
-    CXX="g++"
+    CXX=$(set_build_variables __G_++__)
     CXX_FLAG=
     if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
-        CXX="amdclang++"
-        CXX_FLAG="-DCMAKE_CXX_COMPILER=${ROCM_PATH}/llvm/bin/clang++"
+        CXX=$(set_build_variables __AMD_CLANG_++__)
+        CXX_FLAG=$(set_build_variables __CMAKE_CXX_PARAMS__)
     fi
 
     CLIENTS_SAMPLES="ON"
     if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
        set_asan_env_vars
        set_address_sanitizer_on
+       # fixme: remove CLIENTS_SAMPLES=OFF once SWDEV-417076 is fixed
        CLIENTS_SAMPLES="OFF"
     fi
 
     SHARED_LIBS="ON"
+    if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
+        SHARED_LIBS="OFF"
+    fi
 
     echo "C compiler: $CC"
     echo "CXX compiler: $CXX"
@@ -39,10 +43,10 @@ build_hipblas() {
     init_rocm_common_cmake_params
     cmake \
         ${LAUNCHER_FLAGS} \
-        "${rocm_math_common_cmake_params[@]}" \
+	    "${rocm_math_common_cmake_params[@]}" \
         -DUSE_CUDA=OFF \
         -DBUILD_SHARED_LIBS=$SHARED_LIBS \
-        -DBUILD_CLIENTS_TESTS=ON \
+	    -DBUILD_CLIENTS_TESTS=ON \
         -DBUILD_CLIENTS_BENCHMARKS=ON \
         -DBUILD_CLIENTS_SAMPLES="${CLIENTS_SAMPLES}" \
         -DBUILD_ADDRESS_SANITIZER="${ADDRESS_SANITIZER}" \
@@ -54,7 +58,7 @@ build_hipblas() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -68,7 +72,7 @@ clean_hipblas() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipblas ;;
+    build) build_hipblas; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipblas ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipblaslt.sh

@@ -1,10 +1,20 @@
 #!/bin/bash
 
 set -ex
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipBLASLt
 
+#not enabling the debug files by defalut
+disable_debug_package_generation
+# if ENABLE_GPU_ARCH is set in env by Job parameter ENABLE_GPU_ARCH, then set GFX_ARCH to that value
+if [ -n "$ENABLE_GPU_ARCH" ]; then
+    set_gpu_arch "$ENABLE_GPU_ARCH"
+else
+    # gfx90a:xnack+;gfx90a:xnack-;gfx940;gfx941;gfx942
+    set_gpu_arch "all"
+fi
+
 build_hipblaslt() {
     echo "Start build"
 
@@ -20,31 +30,31 @@ build_hipblaslt() {
     cd $COMPONENT_SRC
     mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
 
-    if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
-       rebuild_lapack
-    fi
-
-    if [ -n "$GPU_ARCHS" ]; then
-        GPU_TARGETS="$GPU_ARCHS"
-    else
-        # gfx90a:xnack+;gfx90a:xnack-;gfx940;gfx941;gfx942
-        GPU_TARGETS=all
+    # hipblaslt requiring Python 3.12.
+    # Point CMake to that explicit location and adjust LD_LIBRARY_PATH.
+    if [ "$DISTRO_ID" = "rhel-8.8" ] || [ "$DISTRO_NAME" == "sles" ] || \
+        [ "$DISTRO_ID" = "rhel-9.1" ] || [ "$DISTRO_ID" = "almalinux-8.10" ] || \
+        [ "$DISTRO_ID" = "debian-10" ]; then
+        EXTRA_PYTHON_PATH=/opt/Python-3.12.7
+        EXTRA_CMAKE_OPTIONS="-DPython_ROOT=/opt/Python-3.12.7"
+        export LD_LIBRARY_PATH=${EXTRA_PYTHON_PATH}/lib
     fi
 
     init_rocm_common_cmake_params
-    CXX=$(set_build_variables CXX)\
+    CXX=$(set_build_variables __CXX__)\
     cmake \
-        -DAMDGPU_TARGETS=${GPU_TARGETS} \
         ${LAUNCHER_FLAGS} \
-        "${rocm_math_common_cmake_params[@]}" \
+         "${rocm_math_common_cmake_params[@]}" \
         -DTensile_LOGIC= \
         -DTensile_CODE_OBJECT_VERSION=default \
-        -DTensile_CPU_THREADS= \
+        -DTensile_CPU_THREADS=$((PROC / 4)) \
         -DTensile_LIBRARY_FORMAT=msgpack \
         -DBUILD_CLIENTS_SAMPLES=ON \
         -DBUILD_CLIENTS_TESTS=ON \
+        -DLINK_BLIS=ON \
         -DBUILD_CLIENTS_BENCHMARKS=ON \
         -DBUILD_ADDRESS_SANITIZER="${ADDRESS_SANITIZER}" \
+        ${EXTRA_CMAKE_OPTIONS} \
         "$COMPONENT_SRC"
 
     cmake --build "$BUILD_DIR" -- -j${PROC}
@@ -52,8 +62,7 @@ build_hipblaslt() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
-
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
     show_build_cache_stats
 }
 
@@ -66,7 +75,7 @@ clean_hipblaslt() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipblaslt ;;
+    build) build_hipblaslt; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipblaslt ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipcc.sh

@@ -9,6 +9,8 @@ printUsage() {
     echo "Options:"
     echo "  -a,  --address_sanitizer  Enable address sanitizer"
     echo "  -c,  --clean              Clean output and delete all intermediate work"
+    echo "  -w,  --wheel              Creates python wheel package of hipcc. 
+                                      It needs to be used along with -r option"
     echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of
                             type referred to by pkg_type"
     echo "  -r,  --release            Makes a release build"
@@ -21,6 +23,7 @@ printUsage() {
 }
 
 
+## Build environment variables
 API_NAME=hipcc
 PROJ_NAME=$API_NAME
 
@@ -29,10 +32,11 @@ MAKEOPTS="$DASH_JAY"
 BUILD_TYPE="Debug"
 SHARED_LIBS="ON"
 BUILD_DIR=$(getBuildPath $API_NAME)
-PACKAGE_DEB=$(getPackageRoot)/deb/$API_NAME
-PACKAGE_RPM=$(getPackageRoot)/rpm/$API_NAME
+PACKAGE_DEB=$(getPackageRoot)/deb/$PROJ_NAME
+PACKAGE_RPM=$(getPackageRoot)/rpm/$PROJ_NAME
 PACKAGE_SRC="$(getSrcPath)"
 
+#parse the arguments
 VALID_STR=`getopt -o hcraswo:p: --long help,clean,release,address_sanitizer,static,outdir,wheel:,package: -- "$@"`
 eval set -- "$VALID_STR"
 
@@ -45,13 +49,15 @@ do
             TARGET="clean" ;;
         (-o  | --outdir)
             TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 1 ;;
+        (-w  | --wheel)
+            WHEEL_PACKAGE=true ;;
         (-r  | --release)
             BUILD_TYPE="RelWithDebInfo" ;;
         (-s | --static)
             SHARED_LIBS="OFF" ;;
         (-h  | --help)
             printUsage ; exit 0 ;;
-        --)     shift; break;;
+        --)     shift; break;; # end delimiter
         (*)
             echo "Invalid option [$1]" >&2; printUsage; exit 1 ;;
     esac
@@ -91,7 +97,7 @@ build() {
         $(rocm_cmake_params) \
         $(rocm_common_cmake_params) \
         -DHIPCC_BACKWARD_COMPATIBILITY=OFF \
-        -DCMAKE_INSTALL_PREFIX="$OUT_DIR" \
+        -DCMAKE_INSTALL_PREFIX="$ROCM_PATH" \
         $HIPCC_ROOT
     popd
 
@@ -118,10 +124,20 @@ print_output_directory() {
 }
 
 case $TARGET in
-    (clean) clean ;;
-    (build) build ; copy_hipcc_sources ;;
-    (outdir) print_output_directory ;;
-    (*) die "Invalid target $TARGET" ;;
+    (clean)
+        clean
+        ;;
+    (build)
+        build
+        build_wheel "$BUILD_DIR" "$PROJ_NAME"
+        copy_hipcc_sources
+        ;;
+    (outdir)
+        print_output_directory
+        ;;
+    (*)
+        die "Invalid target $TARGET"
+        ;;
 esac
 
 echo "Operation complete"

tools/rocm-build/build_hipcub.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipCUB
 
@@ -17,26 +17,21 @@ build_hipcub() {
     if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
          set_asan_env_vars
          set_address_sanitizer_on
+         # ASAN packaging is not required for HIPCUB, since its header only package
+         # Setting the asan_cmake_params to false will disable ASAN packaging
          ASAN_CMAKE_PARAMS="false"
     fi
 
     mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
     init_rocm_common_cmake_params
 
-    if [ -n "$GPU_ARCHS" ]; then
-        GPU_TARGETS="$GPU_ARCHS"
-    else
-        GPU_TARGETS="gfx908:xnack-;gfx90a:xnack-;gfx90a:xnack+;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201"
-    fi
-
-    CXX=$(set_build_variables CXX)\
+    CXX=$(set_build_variables __CXX__)\
     cmake \
         ${LAUNCHER_FLAGS} \
-        "${rocm_math_common_cmake_params[@]}" \
+	    "${rocm_math_common_cmake_params[@]}" \
         -DCMAKE_MODULE_PATH="${ROCM_PATH}/lib/cmake/hip;${ROCM_PATH}/hip/cmake" \
         -Drocprim_DIR="${ROCM_PATH}/rocprim"  \
         -DBUILD_TEST=ON \
-        -DAMDGPU_TARGETS=${GPU_TARGETS} \
         "$COMPONENT_SRC"
 
     cmake --build "$BUILD_DIR" -- -j${PROC}
@@ -44,7 +39,7 @@ build_hipcub() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -58,7 +53,7 @@ clean_hipcub() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipcub ;;
+    build) build_hipcub; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipcub ;;
     *) die "Invalid target $TARGET" ;;