Update packages - remove broken packages

aqlprofiletest

.azuredevops/components/AMDMIGraphX.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -115,6 +120,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -145,7 +151,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/HIP.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -42,7 +47,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -67,6 +72,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependenciesAMD }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
 # compile clr
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -99,7 +105,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -125,6 +131,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependenciesNvidia }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - script: 'ls -1R $(Agent.BuildDirectory)/rocm'
     displayName: 'Artifact listing'
 # compile clr

.azuredevops/components/HIPIFY.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:

.azuredevops/components/MIOpen.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -77,7 +82,7 @@ jobs:
     - template: /.azuredevops/variables-global.yml
     - name: ROCM_PATH
       value: $(Agent.BuildDirectory)/rocm
-    pool: ${{ variables.MEDIUM_BUILD_POOL }}
+    pool: ${{ variables.HIGH_BUILD_POOL }}
     workspace:
       clean: all
     steps:
@@ -97,6 +102,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - task: Bash@3
       displayName: Build and install other dependencies
       inputs:
@@ -113,7 +119,7 @@ jobs:
         extraBuildFlags: >-
           -DMIOPEN_BACKEND=HIP
           -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++
-          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/miopen-deps
+          -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/miopen-deps --generator Ninja
           -DGPU_TARGETS=${{ job.target }}
           -DMIOPEN_ENABLE_AI_KERNEL_TUNING=OFF
           -DMIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK=OFF
@@ -142,7 +148,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/MIVisionX.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -103,6 +108,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -129,7 +135,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/ROCR-Runtime.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -42,7 +47,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -58,6 +63,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -78,7 +84,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/ROCdbgapi.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -24,7 +29,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -40,6 +45,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-

.azuredevops/components/ROCgdb.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 # reference: https://github.com/ROCm/ROCgdb/blob/amd-staging/README-ROCM.md
 - name: aptPackages
   type: object
@@ -46,7 +51,8 @@ jobs:
     condition:
       and(
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common
@@ -69,6 +75,7 @@ jobs:
       parameters:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-autotools.yml
       parameters:
         configureFlags: >-

.azuredevops/components/ROCmValidationSuite.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -96,6 +101,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -126,7 +132,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/Tensile.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -46,7 +51,7 @@ jobs:
   - template: /.azuredevops/variables-global.yml
   - name: ROCM_PATH
     value: $(Agent.BuildDirectory)/rocm
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -63,6 +68,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
   - task: Bash@3
     displayName: Create wheel file
@@ -104,7 +110,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/TransferBench.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -67,6 +72,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -92,7 +98,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/amdsmi.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -58,7 +63,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/aomp-extras.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -23,7 +28,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -39,6 +44,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: aomp-extras

.azuredevops/components/aomp.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 # reference:
 # https://github.com/ROCm/aomp/blob/aomp-dev/docs/SOURCEINSTALL_PREREQUISITE.md
 - name: aptPackages
@@ -108,6 +113,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: extras
@@ -176,7 +182,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/composable_kernel.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -71,6 +76,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - script: |
         mkdir -p $(CCACHE_DIR)
         echo "##vso[task.prependpath]/usr/lib/ccache"
@@ -117,7 +123,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/half.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -25,7 +30,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -41,6 +46,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-

.azuredevops/components/hip-tests.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -75,6 +80,7 @@ jobs:
       parameters:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     # compile hip-tests
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
@@ -109,7 +115,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipBLAS-common.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -29,7 +34,7 @@ jobs:
   - name: ROCM_PATH
     value: $(Agent.BuildDirectory)/rocm
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -45,6 +50,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-

.azuredevops/components/hipBLAS.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -89,6 +94,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -121,7 +127,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipBLASLt.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -98,6 +103,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - task: Bash@3
       displayName: Add ROCm binaries to PATH
       inputs:
@@ -179,7 +185,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipCUB.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -67,6 +72,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -94,7 +100,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipFFT.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -79,6 +84,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -112,7 +118,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipRAND.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -70,6 +75,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -101,7 +107,8 @@ jobs:
     condition:
         and(succeeded(),
           eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-          not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+          not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+          eq(${{ parameters.aggregatePipeline }}, False)
         )
     variables:
     - group: common

.azuredevops/components/hipSOLVER.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -82,6 +87,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
   # build external gtest and lapack
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
@@ -122,7 +128,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipSPARSE.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -77,6 +82,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -116,7 +122,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipSPARSELt.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -91,6 +96,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
   # Build and install gtest and lapack
   # $(Pipeline.Workspace)/deps is a temporary folder for the build process
   # $(Pipeline.Workspace)/s/deps is part of the hipSPARSELt repo
@@ -150,7 +156,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipTensor.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -66,6 +71,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -95,7 +101,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/hipfort.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -76,6 +81,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -111,7 +117,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/llvm-project.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -45,6 +50,7 @@ jobs:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
       skipLlvmSymlink: true
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       componentName: rocm-llvm

.azuredevops/components/rccl.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -72,7 +77,7 @@ jobs:
     - template: /.azuredevops/variables-global.yml
     - name: HIP_ROCCLR_HOME
       value: $(Build.BinariesDirectory)/rocm
-    pool: ${{ variables.MEDIUM_BUILD_POOL }}
+    pool: ${{ variables.HIGH_BUILD_POOL }}
     workspace:
       clean: all
     steps:
@@ -90,6 +95,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -124,7 +130,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rdc.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -89,6 +94,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
   # Build grpc
     - task: Bash@3
       displayName: 'git clone grpc'
@@ -135,7 +141,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocAL.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -151,6 +156,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -182,7 +188,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocALUTION.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -85,6 +90,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -117,7 +123,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocBLAS.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -101,6 +106,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -145,7 +151,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocDecode.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -73,6 +78,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -92,7 +98,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocFFT.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -79,6 +84,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -113,7 +119,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocJPEG.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -75,6 +80,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -96,7 +102,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocMLIR.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -62,6 +67,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -87,7 +93,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocPRIM.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -66,6 +71,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -93,7 +99,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocPyDecode.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -77,6 +82,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - task: Bash@3
       displayName: 'Save Python Package Paths'
       inputs:
@@ -152,7 +158,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocRAND.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -70,6 +75,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -98,7 +104,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocSOLVER.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -88,6 +93,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         componentName: lapack
@@ -131,7 +137,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocSPARSE.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -80,6 +85,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -126,7 +132,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocThrust.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -71,6 +76,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -98,7 +104,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocWMMA.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -65,7 +70,7 @@ jobs:
     variables:
     - group: common
     - template: /.azuredevops/variables-global.yml
-    pool: ${{ variables.MEDIUM_BUILD_POOL }}
+    pool: ${{ variables.HIGH_BUILD_POOL }}
     workspace:
       clean: all
     steps:
@@ -81,6 +86,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -113,7 +119,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocm-cmake.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -26,7 +31,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/components/rocm-core.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -17,7 +22,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/components/rocm-examples.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -100,6 +105,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         # https://github.com/ROCm/HIP/issues/2203
@@ -136,7 +142,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocm_bandwidth_test.yml

@@ -5,6 +5,12 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
+
 - name: aptPackages
   type: object
   default:
@@ -49,7 +55,7 @@ jobs:
     value: $(Agent.BuildDirectory)/rocm
   - name: ROCR_LIB_DIR
     value: $(Agent.BuildDirectory)/rocm
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -66,6 +72,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -90,7 +97,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocm_smi_lib.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -28,7 +33,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -59,7 +64,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocminfo.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -38,7 +43,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -55,6 +60,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
       skipLlvmSymlink: true
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
@@ -72,7 +78,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocprofiler-compute.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -114,6 +119,7 @@ jobs:
         dependencyList: ${{ parameters.rocmDependencies }}
         dependencySource: ${{ job.dependencySource }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -140,7 +146,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocprofiler-register.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -17,7 +22,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all

.azuredevops/components/rocprofiler-sdk.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -89,6 +94,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - task: Bash@3
       displayName: Add Python site-packages binaries to path
       inputs:
@@ -125,7 +131,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocprofiler-systems.yml

@@ -6,6 +6,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -103,6 +108,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - task: Bash@3
       displayName: Add ROCm binaries to PATH
       inputs:
@@ -147,7 +153,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     timeoutInMinutes: 180
     variables:

.azuredevops/components/rocprofiler.yml

@@ -5,6 +5,12 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
+
 - name: aptPackages
   type: object
   default:
@@ -83,6 +89,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -115,7 +122,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rocr_debug_agent.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -49,7 +54,7 @@ jobs:
   variables:
   - group: common
   - template: /.azuredevops/variables-global.yml
-  pool: 
+  pool:
     vmImage: ${{ variables.BASE_BUILD_POOL }}
   workspace:
     clean: all
@@ -65,6 +70,7 @@ jobs:
     parameters:
       checkoutRef: ${{ parameters.checkoutRef }}
       dependencyList: ${{ parameters.rocmDependencies }}
+      aggregatePipeline: ${{ parameters.aggregatePipeline }}
   - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
     parameters:
       extraBuildFlags: >-
@@ -86,7 +92,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/roctracer.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -77,6 +82,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -107,7 +113,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/components/rpp.yml

@@ -5,6 +5,11 @@ parameters:
 - name: checkoutRef
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 - name: aptPackages
   type: object
   default:
@@ -84,6 +89,7 @@ jobs:
         checkoutRef: ${{ parameters.checkoutRef }}
         dependencyList: ${{ parameters.rocmDependencies }}
         gpuTarget: ${{ job.target }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
       parameters:
         extraBuildFlags: >-
@@ -113,7 +119,8 @@ jobs:
     condition:
       and(succeeded(),
         eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
-        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName']))
+        not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])),
+        eq(${{ parameters.aggregatePipeline }}, False)
       )
     variables:
     - group: common

.azuredevops/templates/steps/artifact-download.yml

@@ -12,6 +12,11 @@ parameters:
 - name: fileFilter
   type: string
   default: ''
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 
 steps:
 - task: Bash@3
@@ -27,17 +32,23 @@ steps:
 - task: DownloadPipelineArtifact@2
   displayName: Download ${{ parameters.componentName }}
   inputs:
-    buildType: 'specific'
-    project: ROCm-CI
-    definition: ${{ parameters.pipelineId }}
-    specificBuildWithTriggering: true
-    itemPattern: '**/*${{ parameters.fileFilter }}*'
-    # aomp is a special case, since the trigger file is under ROCm/ROCm instead of the component repo
-    ${{ if notIn(parameters.componentName, 'aomp') }}:
-      buildVersionToDownload: latestFromBranch # default is 'latest'
-    branchName: refs/heads/${{ parameters.branchName }}
-    allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
-    targetPath: '$(Pipeline.Workspace)/d'
+    ${{ if eq(parameters.aggregatePipeline, false) }}:
+      buildType: 'specific'
+      project: ROCm-CI
+      definition: ${{ parameters.pipelineId }}
+      specificBuildWithTriggering: true
+      itemPattern: '**/*${{ parameters.fileFilter }}*'
+      # aomp is a special case, since the trigger file is under ROCm/ROCm instead of the component repo
+      ${{ if notIn(parameters.componentName, 'aomp') }}:
+        buildVersionToDownload: latestFromBranch # default is 'latest'
+      branchName: refs/heads/${{ parameters.branchName }}
+      allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
+      targetPath: '$(Pipeline.Workspace)/d'
+    ${{ else }}:
+      buildType: 'current'
+      itemPattern: '**/${{ parameters.componentName }}*${{ parameters.fileFilter }}*'
+      allowPartiallySucceededBuilds: $(allowPartiallySucceededBuilds)
+      targetPath: '$(Pipeline.Workspace)/d'
 - task: ExtractFiles@1
   displayName: Extract ${{ parameters.componentName }}
   inputs:

.azuredevops/templates/steps/dependencies-rocm.yml

@@ -31,6 +31,11 @@ parameters:
 - name: setupHIPLibrarySymlinks
   type: boolean
   default: false
+# set to true if doing full build of ROCm stack
+# and dependencies are pulled from same pipeline
+- name: aggregatePipeline
+  type: boolean
+  default: false
 
 - name: componentVarList
   type: object
@@ -354,6 +359,7 @@ steps:
       parameters:
         componentName: ${{ split(dependency, ':')[0] }}
         pipelineId: ${{ parameters.componentVarList[split(dependency, ':')[0]].pipelineId }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
         ${{ if parameters.componentVarList[split(dependency, ':')[0]].hasGpuTarget }}:
           fileFilter: "${{ split(dependency, ':')[1] }}*${{ parameters.gpuTarget }}"
         # dependencySource = staging
@@ -383,6 +389,7 @@ steps:
       parameters:
         componentName: ${{ dependency }}
         pipelineId: ${{ parameters.componentVarList[dependency].pipelineId }}
+        aggregatePipeline: ${{ parameters.aggregatePipeline }}
         ${{ if parameters.componentVarList[dependency].hasGpuTarget }}:
           fileFilter: ${{ parameters.gpuTarget }}
         # dependencySource = staging

.azuredevops/templates/steps/miopen-get-ck-build.yml

@@ -54,7 +54,7 @@ steps:
       fi
 
       echo "Downloading CK artifact from $ARTIFACT_URL"
-      wget -nv $ARTIFACT_URL -O $(System.ArtifactsDirectory)/ck.zip
+      wget --tries=5 --waitretry=10 --retry-connrefused -nv $ARTIFACT_URL -O $(System.ArtifactsDirectory)/ck.zip
       unzip $(System.ArtifactsDirectory)/ck.zip -d $(System.ArtifactsDirectory)
       mkdir -p $(Agent.BuildDirectory)/rocm
       tar -zxvf $(System.ArtifactsDirectory)/$ARTIFACT_NAME/*.tar.gz -C $(Agent.BuildDirectory)/rocm

.azuredevops/variables-global.yml

@@ -30,13 +30,13 @@ variables:
 - name: GFX90A_TEST_POOL
   value: gfx90a_test_pool
 - name: LATEST_RELEASE_VERSION
-  value: 6.3.4
-- name: REPO_RADEON_VERSION
-  value: 6.3.4
-- name: NEXT_RELEASE_VERSION
   value: 6.4.0
+- name: REPO_RADEON_VERSION
+  value: 6.4
+- name: NEXT_RELEASE_VERSION
+  value: 6.5.0
 - name: LATEST_RELEASE_TAG
-  value: rocm-6.3.4
+  value: rocm-6.4.0
 - name: DOCKER_SKIP_GFX
   value: gfx90a
 - name: AMDMIGRAPHX_GFX942_TEST_PIPELINE_ID

README.md

@@ -127,6 +127,7 @@ bash install-prerequisites.sh
 export GPU_ARCHS="gfx942"               # Example
 export GPU_ARCHS="gfx940;gfx941;gfx942" # Example
 
+cd ~/WORKSPACE/
 # Pick and run build commands in the docker container:
 # Build rocm-dev packages
 make -f ROCm/tools/rocm-build/ROCm.mk -j ${NPROC:-$(nproc)} rocm-dev

RELEASE.md

@@ -64,7 +64,7 @@ ROCm 6.4.0 has been tested to allow you to choose a combination of AMD Kernel-mo
 
 ### Separation of user space and driver space components documentation
 
-As of ROCm 6.4.0, the driver space components documentation has moved from [AMD ROCm documentation](https://rocmdocs.amd.com/) to its own documentation site, [AMD Instinct Data Center GPU Driver](instinct.docs.amd.com). The goal is to make the software for AMD Instinct GPUs more modular. This helps in having a clear understanding of the options for installation combinations of Instinct driver and multiple supported ROCm user space versions.
+As of ROCm 6.4.0, the driver space components documentation has moved from [AMD ROCm documentation](https://rocmdocs.amd.com/) to its own documentation site, [AMD Instinct Data Center GPU Driver](https://instinct.docs.amd.com/latest/). The goal is to make the software for AMD Instinct GPUs more modular. This helps in having a clear understanding of the options for installation combinations of Instinct driver and multiple supported ROCm user space versions.
 
 Information about the variant of the `amdgpu` driver built for Instinct GPUs is available on [AMD Instinct Data Center GPU Driver](https://instinct.docs.amd.com/projects/amdgpu-docs/en/latest/). See [ROCm/ROCK-Kernel-Driver](https://github.com/ROCm/ROCK-Kernel-Driver) GitHub repository for source code, which is planned to be renamed to **instinct-driver** in a future ROCm release. For ROCm 6.4.0, the versioning scheme for the Instinct driver is parallel to the ROCm versioning; that is, 6.4.0. In future ROCm releases, the Instinct driver version is planned to be separate from the ROCm versioning.
 
@@ -164,9 +164,10 @@ ROCm documentation continues to be updated to provide clearer and more comprehen
     - The new [HIP complex math API](https://rocm.docs.amd.com/projects/HIP/en/latest/reference/complex_math_api.html) topic describes HIP complex number types and usage of these types with example code.
     - The new [HIP error codes](https://rocm.docs.amd.com/projects/HIP/en/latest/reference/error_codes.html) topic list notes all HIP runtime error codes and their descriptions. HIP API functions return these error codes to indicate various runtime conditions and errors.
     - The [Introduction to the HIP programming model](https://rocm.docs.amd.com/projects/HIP/en/latest/understand/programming_model.html) topic has been updated, providing a more robust introduction to HIP.
-    - The [Math API](https://rocm.docs.amd.com/projects/HIP/en/latest/understand/programming_model.html) topic has been reorganized, and the ULP difference of maximum absolute error information has been added.
-    - The new [Low precision floating point types](https://rocm.docs.amd.com/projects/HIP/en/latest/understand/programming_model.html) topic includes information about FP8 (Quarter Precision) and FP16 (Half Precision).
+    - The [Math API](https://rocm.docs.amd.com/projects/HIP/en/latest/reference/math_api.html) topic has been reorganized, and the ULP difference of maximum absolute error information has been added.
+    - The new [Low precision floating point types](https://rocm.docs.amd.com/projects/HIP/en/latest/reference/low_fp_types.html) topic includes information about FP8 (Quarter Precision) and FP16 (Half Precision).
 
+* In addition to these Release Notes, see the blog [Breaking Barriers in AI, HPC, and Modular GPU Software](https://rocm.blogs.amd.com/ecosystems-and-partners/rocm-6.4-blog/README.html) for a wide-ranging discussion of the key advancements and highlights of ROCm 6.4.0.
 
 ## Operating system and hardware support changes
 
@@ -1577,34 +1578,35 @@ modprobe.blacklist=ast
 ```bash
 modprobe.blacklist=mgag200
 ```
+See [GitHub issue #4589](https://github.com/ROCm/ROCm/issues/4589).
 
 ### Failure when using a generic target with compression and vice versa
 
-In ROCm 6.4.0, compilation for generic target with compression will fail. As a result, you won't be able to compile for a generic target and use compression simultaneously. As a workaround, it's recommended not to use compression when using generic targets and vice versa. This issue will be addressed in a future ROCm release.
+In ROCm 6.4.0, compilation for generic target with compression will fail. As a result, you won't be able to compile for a generic target and use compression simultaneously. As a workaround, it's recommended not to use compression when using generic targets and vice versa. This issue will be addressed in a future ROCm release. See [GitHub issue #4602](https://github.com/ROCm/ROCm/issues/4602).
 
 ### GFX Freq information is unavailable in the rocm-smi when running in SRIOV mode enabled on MI210
 
-In ROCm 6.4.0, you cannot see the GFX Freq information in the guest VM. In SRIOV mode, the AMD Platform Management Firmware (PMFW) does not share the graphics frequency information with the guest VMs and is only available to Host systems. This issue will be addressed in a future ROCm release.
+In ROCm 6.4.0, you cannot see the GFX Freq information in the guest VM. In SRIOV mode, the AMD Platform Management Firmware (PMFW) does not share the graphics frequency information with the guest VMs and is only available to Host systems. This issue will be addressed in a future ROCm release. See [GitHub issue #4603](https://github.com/ROCm/ROCm/issues/4603).
 
 ### Failure to use --kokkos-trace option in ROCm Compute Profiler
 
-In ROCm 6.4.0, it’s not recommended to use the `--kokkos-trace` option. `--kokkos-trace` has been partially implemented in the `rocprofv3` tool, resulting in a difference between the output of `--kokkos-trace` and the `counter_collection.csv` output file. The program will exit with a warning message if the `-kokkos-trace` option is detected in the ROCm Compute Profiler. The issue will be addressed in a future ROCm release.
+In ROCm 6.4.0, it’s not recommended to use the `--kokkos-trace` option. `--kokkos-trace` has been partially implemented in the `rocprofv3` tool, resulting in a difference between the output of `--kokkos-trace` and the `counter_collection.csv` output file. The program will exit with a warning message if the `-kokkos-trace` option is detected in the ROCm Compute Profiler. The issue will be addressed in a future ROCm release. See [GitHub issue #4604](https://github.com/ROCm/ROCm/issues/4604).
 
 ### Compute partition modification is restricted with concurrent operations running in parallel
  
-Modification to compute partition in GPU is prohibited by design while concurrent operations run in parallel. You must ensure no concurrent operations on the device are running when attempting to modify the compute partitions. Additional checks and error messaging to inform users of correct operation for partition modification are planned for future ROCm releases.
+Modification to compute partition in GPU is prohibited by design while concurrent operations run in parallel. You must ensure no concurrent operations on the device are running when attempting to modify the compute partitions. Additional checks and error messaging to inform users of correct operation for partition modification are planned for future ROCm releases. See [GitHub issue #4605](https://github.com/ROCm/ROCm/issues/4605).
 
 ### MIOpen generates incorrect results for particular input with FP32 data type
 
-In ROCm 6.4.0, MIOpen generates incorrect results on the `conv2dbackward` function for a particular input with 32-bit floating point (FP32) data types. The issue is only specific to FP32 data types with 2 * 2 kernel size and dilation 2 * 1. As a workaround, change the data type from FP32 to FP16. The issue will be addressed in a future ROCm release.  
+In ROCm 6.4.0, MIOpen generates incorrect results on the `conv2dbackward` function for a particular input with 32-bit floating point (FP32) data types. The issue is only specific to FP32 data types with 2 * 2 kernel size and dilation 2 * 1. As a workaround, change the data type from FP32 to FP16. The issue will be addressed in a future ROCm release. See [GitHub issue #4606](https://github.com/ROCm/ROCm/issues/4606). 
 
 ### ROCm Debugger (ROCgdb) might not work correctly on the AMD Radeon PRO W6800 SR-IOV virtualization environment
 
-The ROCm Debugger (ROCgdb) component needs access to some registers to fetch debugging information. These registers are blocked in the AMD Radeon PRO W6800 SR-IOV virtualization environment, resulting in the ROCm Debugger (ROCgdb) being non-functional. The issue is due to the limitation in the virtualization environment and isn't specific to ROCm. Further investigation is in progress.
+The ROCm Debugger (ROCgdb) component needs access to some registers to fetch debugging information. These registers are blocked in the AMD Radeon PRO W6800 SR-IOV virtualization environment, resulting in the ROCm Debugger (ROCgdb) being non-functional. The issue is due to the limitation in the virtualization environment and isn't specific to ROCm. Further investigation is in progress. See [GitHub issue #4607](https://github.com/ROCm/ROCm/issues/4607).
 
 ### Limited support for Sparse API and Pallas functionality in JAX
 
-In ROCm 6.4.0, due to limited support for Sparse API in JAX, some of the functionality of the Pallas extension is restricted. This results in issues porting existing workloads. The issue will be addressed in a future ROCm release.
+In ROCm 6.4.0, due to limited support for Sparse API in JAX, some of the functionality of the Pallas extension is restricted. This results in issues porting existing workloads. The issue will be addressed in a future ROCm release. See [GitHub issue #4608](https://github.com/ROCm/ROCm/issues/4608).
 
 ### Inconsistent log probabilities when using the Mixtral 8x7B model in vLLM and SGLang framework
 
@@ -1612,7 +1614,7 @@ In ROCm 6.4.0, using a Mixtral 8X7B model with different tensor parallelism (TP)
 
 The inconsistency primarily impacts the applications that rely on consistent log probabilities, such as those involving uncertainty estimation or probabilistic decision-making. This known limitation results from how TP distributes computations across multiple GPUs, resulting in slight variations in floating-point arithmetic. Currently, there is no direct resolution as this is a framework-level characteristic rather than a defect.
 
-As a workaround, you can standardize the TP sizes across all the deployments to minimize the inconsistency in the log probabilities. For information on the resolution of this inconsistency in the future, see the [SGlang](https://github.com/sgl-project/sglang) and [vLLM](https://github.com/vllm-project/vllm) GitHub repositories. 
+As a workaround, you can standardize the TP sizes across all the deployments to minimize the inconsistency in the log probabilities. For information on the resolution of this inconsistency in the future, see the [SGlang](https://github.com/sgl-project/sglang) and [vLLM](https://github.com/vllm-project/vllm) GitHub repositories. See [GitHub issue #4609](https://github.com/ROCm/ROCm/issues/4609).
 
 ### No module named more_itertools warning on Azure Linux 3
 
@@ -1621,14 +1623,15 @@ During the driver installation process on Azure Linux 3, you might encounter the
 ```
 sudo python3 -m pip install more_itertools
 ```
+See [GitHub issue #4610](https://github.com/ROCm/ROCm/issues/4610).
 
 ### Rare occurrence of AMDGPU driver failing to load in a VM on Quanta system
 
-In a rare occurrence (1 in 500 reboots), the guest kernel might display the call trace due to the AMDGPU driver failing to load in a repeated power cycle virtual machine (VM) on a Quanta system. This issue will limit you from using the AMD GPUs in the guest kernel. As a workaround, reboot the VM to avoid the failure.
+In a rare occurrence (1 in 500 reboots), the guest kernel might display the call trace due to the AMDGPU driver failing to load in a repeated power cycle virtual machine (VM) on a Quanta system. This issue will limit you from using the AMD GPUs in the guest kernel. As a workaround, reboot the VM to avoid the failure. See [GitHub issue #4611](https://github.com/ROCm/ROCm/issues/4611).
 
 ### Clang compilation failure might occur due to incorrectly installed GNU C++ runtime
 
-Clang compilation failure with the error `fatal error: 'cmath' file not found` might occur if the GNU C++ runtime is not installed correctly. The error indicates that the `libstdc++-dev` package, compatible with the latest installed GNU Compiler Collection (GCC) version, is missing. This issue is a result of Clang being unable to find the newest GNU C++ runtimes it recognizes and the associated header files. As a workaround, install the `libstdc++-dev` package compatible with the installed GCC version.
+Clang compilation failure with the error `fatal error: 'cmath' file not found` might occur if the GNU C++ runtime is not installed correctly. The error indicates that the `libstdc++-dev` package, compatible with the latest installed GNU Compiler Collection (GCC) version, is missing. This issue is a result of Clang being unable to find the newest GNU C++ runtimes it recognizes and the associated header files. As a workaround, install the `libstdc++-dev` package compatible with the installed GCC version. See [GitHub issue #4612](https://github.com/ROCm/ROCm/issues/4612).
 
 ### ROCProfiler with rocprof might fail to initialize in some PyTorch applications
 
@@ -1644,18 +1647,19 @@ Alternatively, you can modify the `rocprof` script located at `/opt/rocm-6.x.x/b
 ```
 ROCPROFV1_LD_PRELOAD=$MY_HSA_TOOLS_LIB
 ```
+See [GitHub issue #4613](https://github.com/ROCm/ROCm/issues/4613).
 
 ### Applications using HIP runtime might stop the graph capture process
 
-Applications using the HIP runtime might stop the graph capture process if the HIP runtime detects an invalid stale state from a previous capture on the same HIP stream. Resetting the stale set for every new capture in the HIP runtime can resolve the issue. The issue will be fixed in a future ROCm release.
+Applications using the HIP runtime might stop the graph capture process if the HIP runtime detects an invalid stale state from a previous capture on the same HIP stream. Resetting the stale set for every new capture in the HIP runtime can resolve the issue. The issue will be fixed in a future ROCm release. See [GitHub issue #4614](https://github.com/ROCm/ROCm/issues/4614).
 
 ### Incorrect computation results in hipBLASLt for specific transpose configuration
 
-When running the hipBLASLt library using the transpose configuration (TT) with FP32 and XF32 data types, you might receive incorrect computation results. As a workaround, select alternative solutions from the list returned by `hipblasLtMatmulAlgoGetHeuristic()`. Verify the result to identify the correct alternative solution. The issue will be fixed in a future ROCm release.
+When running the hipBLASLt library using the transpose configuration (TT) with FP32 and XF32 data types, you might receive incorrect computation results. As a workaround, select alternative solutions from the list returned by `hipblasLtMatmulAlgoGetHeuristic()`. Verify the result to identify the correct alternative solution. The issue will be fixed in a future ROCm release. See [GitHub issue #4615](https://github.com/ROCm/ROCm/issues/4615).
 
 ### Incorrect result in RCCL when using LL protocol in graph mode with MSCCL++ enabled
 
-In RCCL library, you might receive incorrect results in All-Reduce collective API, when using Link Layer (LL) protocol in graph mode while MSCCL++ is enabled. This issue occurs when the protocal state information are updated in the host-side code instead of in a kernel, which is not supported in graph mode. As a workaround, you can disable MSCCL++ by setting the environment variable `RCCL_MSCCLPP_ENABLE=0`. However, consider that this might negatively impact the performance. The issue will be fixed in a future ROCm release. 
+In RCCL library, you might receive incorrect results in All-Reduce collective API, when using Link Layer (LL) protocol in graph mode while MSCCL++ is enabled. This issue occurs when the protocal state information are updated in the host-side code instead of in a kernel, which is not supported in graph mode. As a workaround, you can disable MSCCL++ by setting the environment variable `RCCL_MSCCLPP_ENABLE=0`. However, consider that this might negatively impact the performance. The issue will be fixed in a future ROCm release. See [GitHub issue #4616](https://github.com/ROCm/ROCm/issues/4616).
 
 ## ROCm resolved issues
 

docs/compatibility/compatibility-matrix-historical-6.0.csv

@@ -1,121 +1,121 @@
-ROCm Version,6.4.0,6.3.3,6.3.2,6.3.1,6.3.0,6.2.4,6.2.2,6.2.1,6.2.0, 6.1.5, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
-      :ref:`Operating systems & kernels <OS-kernel-versions>`,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04.2,"Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04",Ubuntu 24.04,,,,,,
-      ,Ubuntu 22.04.5,Ubuntu 22.04.5,Ubuntu 22.04.5,Ubuntu 22.04.5,Ubuntu 22.04.5,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3, 22.04.2","Ubuntu 22.04.4, 22.04.3, 22.04.2"
-      ,,,,,,,,,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
-      ,"RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
-      ,RHEL 8.10,RHEL 8.10,RHEL 8.10,RHEL 8.10,RHEL 8.10,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
-      ,SLES 15 SP6,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
-      ,,,,,,,,,,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
-      ,"Oracle Linux 9, 8 [#mi300x-past-60]_",Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,,,
-,Debian 12 [#single-node-past-60]_,Debian 12 [#single-node-past-60]_,Debian 12 [#single-node-past-60]_,Debian 12 [#single-node-past-60]_,,,,,,,,,,,
-,Azure Linux 3.0 [#mi300x-past-60]_,Azure Linux 3.0 [#mi300x-past-60]_,Azure Linux 3.0 [#mi300x-past-60]_,,,,,,,,,,,,
-      ,.. _architecture-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
-      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
-      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
-      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
-      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
-      ,.. _gpu-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
-      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
-      ,gfx942,gfx942,gfx942,gfx942,gfx942,gfx942 [#mi300_624-past-60]_,gfx942 [#mi300_622-past-60]_,gfx942 [#mi300_621-past-60]_,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
-      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
-      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
-,,,,,,,,,,,,,,,
-      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`PyTorch <../compatibility/ml-compatibility/pytorch-compatibility>`,"2.6, 2.5, 2.4, 2.3","2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
-      :doc:`TensorFlow <../compatibility/ml-compatibility/tensorflow-compatibility>`,"2.18.1, 2.17.1, 2.16.2","2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1","2.14.0, 2.13.1, 2.12.1"
-      :doc:`JAX <../compatibility/ml-compatibility/jax-compatibility>`,0.4.35,0.4.31,0.4.31,0.4.31,0.4.31,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
-      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.20,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
-,,,,,,,,,,,,,,,
-      ,,,,,,,,,,,,,,,
-      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
-      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
-      ,,,,,,,,,,,,,,,
-      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      Thrust,2.5.0,2.3.2,2.3.2,2.3.2,2.3.2,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-      CUB,2.5.0,2.3.2,2.3.2,2.3.2,2.3.2,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
-,,,,,,,,,,,,,,,
-      KMD & USER SPACE [#kfd_support-past-60]_,.. _kfd-userspace-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      Tested user space versions,"6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x","6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.2.x, 6.1.x, 6.0.x, 5.7.x, 5.6.x","6.2.x, 6.1.x, 6.0.x, 5.7.x, 5.6.x"
-      ,,,,,,,,,,,,,,,
-      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
-      :doc:`MIGraphX <amdmigraphx:index>`,2.12.0,2.11.0,2.11.0,2.11.0,2.11.0,2.10.0,2.10.0,2.10.0,2.10.0,2.9.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
-      :doc:`MIOpen <miopen:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.0,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`MIVisionX <mivisionx:index>`,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0,3.0.0,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
-      :doc:`rocAL <rocal:index>`,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0,2.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`rocDecode <rocdecode:index>`,0.10.0,0.8.0,0.8.0,0.8.0,0.8.0,0.6.0,0.6.0,0.6.0,0.6.0,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
-      :doc:`rocJPEG <rocjpeg:index>`,0.8.0,0.6.0,0.6.0,0.6.0,0.6.0,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`rocPyDecode <rocpydecode:index>`,0.3.1,0.2.0,0.2.0,0.2.0,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`RPP <rpp:index>`,1.9.10,1.9.1,1.9.1,1.9.1,1.9.1,1.8.0,1.8.0,1.8.0,1.8.0,1.5.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
-      ,,,,,,,,,,,,,,,
-      COMMUNICATION,.. _commlibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`RCCL <rccl:index>`,2.22.3,2.21.5,2.21.5,2.21.5,2.21.5,2.20.5,2.20.5,2.20.5,2.20.5,2.18.6,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
-      ,,,,,,,,,,,,,,,
-      MATH LIBS,.. _mathlibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
-      :doc:`hipBLAS <hipblas:index>`,2.4.0,2.3.0,2.3.0,2.3.0,2.3.0,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
-      :doc:`hipBLASLt <hipblaslt:index>`,0.12.0,0.10.0,0.10.0,0.10.0,0.10.0,0.8.0,0.8.0,0.8.0,0.8.0,0.7.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
-      :doc:`hipFFT <hipfft:index>`,1.0.18,1.0.17,1.0.17,1.0.17,1.0.17,1.0.16,1.0.15,1.0.15,1.0.14,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
-      :doc:`hipfort <hipfort:index>`,0.6.0,0.5.1,0.5.1,0.5.0,0.5.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
-      :doc:`hipRAND <hiprand:index>`,2.12.0,2.11.1,2.11.1,2.11.1,2.11.0,2.11.1,2.11.0,2.11.0,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
-      :doc:`hipSOLVER <hipsolver:index>`,2.4.0,2.3.0,2.3.0,2.3.0,2.3.0,2.2.0,2.2.0,2.2.0,2.2.0,2.1.1,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
-      :doc:`hipSPARSE <hipsparse:index>`,3.2.0,3.1.2,3.1.2,3.1.2,3.1.2,3.1.1,3.1.1,3.1.1,3.1.1,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.3,0.2.2,0.2.2,0.2.2,0.2.2,0.2.1,0.2.1,0.2.1,0.2.1,0.2.0,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
-      :doc:`rocALUTION <rocalution:index>`,3.2.2,3.2.1,3.2.1,3.2.1,3.2.1,3.2.1,3.2.0,3.2.0,3.2.0,3.1.1,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
-      :doc:`rocBLAS <rocblas:index>`,4.4.0,4.3.0,4.3.0,4.3.0,4.3.0,4.2.4,4.2.1,4.2.1,4.2.0,4.1.2,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
-      :doc:`rocFFT <rocfft:index>`,1.0.32,1.0.31,1.0.31,1.0.31,1.0.31,1.0.30,1.0.29,1.0.29,1.0.28,1.0.27,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
-      :doc:`rocRAND <rocrand:index>`,3.3.0,3.2.0,3.2.0,3.2.0,3.2.0,3.1.1,3.1.0,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
-      :doc:`rocSOLVER <rocsolver:index>`,3.28.0,3.27.0,3.27.0,3.27.0,3.27.0,3.26.2,3.26.0,3.26.0,3.26.0,3.25.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
-      :doc:`rocSPARSE <rocsparse:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.1,3.2.0,3.2.0,3.2.0,3.1.2,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
-      :doc:`rocWMMA <rocwmma:index>`,1.7.0,1.6.0,1.6.0,1.6.0,1.6.0,1.5.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
-      :doc:`Tensile <tensile:src/index>`,4.43.0,4.42.0,4.42.0,4.42.0,4.42.0,4.41.0,4.41.0,4.41.0,4.41.0,4.40.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
-      ,,,,,,,,,,,,,,,
-      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`hipCUB <hipcub:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.1,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`hipTensor <hiptensor:index>`,1.5.0,1.4.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0,1.3.0,1.3.0,1.2.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
-      :doc:`rocPRIM <rocprim:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.2,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
-      :doc:`rocThrust <rocthrust:index>`,3.3.0,3.3.0,3.3.0,3.3.0,3.3.0,3.1.1,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
-      ,,,,,,,,,,,,,,,
-      SUPPORT LIBS,,,,,,,,,,,,,,,
-      `hipother <https://github.com/ROCm/hipother>`_,6.4.43482,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.4.0,6.3.3,6.3.2,6.3.1,6.3.0,6.2.4,6.2.2,6.2.1,6.2.0,6.1.2,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
-      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,20240607.5.7,20240607.5.7,20240607.4.05,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
-      ,,,,,,,,,,,,,,,
-      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`AMD SMI <amdsmi:index>`,25.3.0,24.7.1,24.7.1,24.7.1,24.7.1,24.6.3,24.6.3,24.6.3,24.6.2,24.5.1,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
-      :doc:`ROCm Data Center Tool <rdc:index>`,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
-      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.5.0,7.4.0,7.4.0,7.4.0,7.4.0,7.3.0,7.3.0,7.3.0,7.3.0,7.2.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
-      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.0.60204,1.0.60202,1.0.60201,1.0.60200,1.0.60102,1.0.60102,1.0.60101,1.0.60100,1.0.60002,1.0.60000
-      ,,,,,,,,,,,,,,,
-      PERFORMANCE TOOLS,,,,,,,,,,,,,,,
-      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
-      :doc:`ROCm Compute Profiler <rocprofiler-compute:index>`,3.1.0,3.0.0,3.0.0,3.0.0,3.0.0,2.0.1,2.0.1,2.0.1,2.0.1,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCm Systems Profiler <rocprofiler-systems:index>`,1.0.0,0.1.2,0.1.1,0.1.0,0.1.0,1.11.2,1.11.2,1.11.2,1.11.2,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60400,2.0.60303,2.0.60302,2.0.60301,2.0.60300,2.0.60204,2.0.60202,2.0.60201,2.0.60200,2.0.60102,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
-      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.6.0,0.5.0,0.5.0,0.5.0,0.5.0,0.4.0,0.4.0,0.4.0,0.4.0,N/A,N/A,N/A,N/A,N/A,N/A
-      :doc:`ROCTracer <roctracer:index>`,4.1.60400,4.1.60303,4.1.60302,4.1.60301,4.1.60300,4.1.60204,4.1.60202,4.1.60201,4.1.60200,4.1.60102,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
-      ,,,,,,,,,,,,,,,
-      DEVELOPMENT TOOLS,,,,,,,,,,,,,,,
-      :doc:`HIPIFY <hipify:index>`,19.0.0.25104,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24455,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.14.0,0.14.0,0.14.0,0.14.0,0.14.0,0.13.0,0.13.0,0.13.0,0.13.0,0.12.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
-      :doc:`ROCdbgapi <rocdbgapi:index>`,0.77.2,0.77.0,0.77.0,0.77.0,0.77.0,0.76.0,0.76.0,0.76.0,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
-      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,15.2.0,15.2.0,15.2.0,15.2.0,15.2.0,14.2.0,14.2.0,14.2.0,14.2.0,14.1.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
-      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.3.0,0.3.0,0.3.0,0.3.0,N/A,N/A
-      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.4,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
-      ,,,,,,,,,,,,,,,
-      COMPILERS,.. _compilers-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
-      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
-      `Flang <https://github.com/ROCm/flang>`_,19.0.0.25133,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24455,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      :doc:`llvm-project <llvm-project:index>`,19.0.0.25133,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24491,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,19.0.0.25133,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24491,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
-,,,,,,,,,,,,,,,
-      RUNTIMES,.. _runtime-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
-      :doc:`AMD CLR <hip:understand/amd_clr>`,6.4.43482,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      :doc:`HIP <hip:index>`,6.4.43482,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
-      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
-      :doc:`ROCr Runtime <rocr-runtime:index>`,1.15.0,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.13.0,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0
+ROCm Version,6.4.0,6.3.3,6.3.2,6.3.1,6.3.0,6.2.4,6.2.2,6.2.1,6.2.0, 6.1.5, 6.1.2, 6.1.1, 6.1.0, 6.0.2, 6.0.0
+      :ref:`Operating systems & kernels <OS-kernel-versions>`,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04.2,Ubuntu 24.04.2,"Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04","Ubuntu 24.04.1, 24.04",Ubuntu 24.04,,,,,,
+      ,Ubuntu 22.04.5,Ubuntu 22.04.5,Ubuntu 22.04.5,Ubuntu 22.04.5,Ubuntu 22.04.5,"Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.5, 22.04.4","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3","Ubuntu 22.04.4, 22.04.3, 22.04.2","Ubuntu 22.04.4, 22.04.3, 22.04.2"
+      ,,,,,,,,,,"Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5","Ubuntu 20.04.6, 20.04.5"
+      ,"RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.5, 9.4","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.4, 9.3, 9.2","RHEL 9.3, 9.2","RHEL 9.3, 9.2"
+      ,RHEL 8.10,RHEL 8.10,RHEL 8.10,RHEL 8.10,RHEL 8.10,"RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.10, 8.9","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8","RHEL 8.9, 8.8"
+      ,SLES 15 SP6,"SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP6, SP5","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4","SLES 15 SP5, SP4"
+      ,,,,,,,,,,,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9,CentOS 7.9
+      ,"Oracle Linux 9, 8 [#mi300x-past-60]_",Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.10 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,Oracle Linux 8.9 [#mi300x-past-60]_,,,
+,Debian 12 [#single-node-past-60]_,Debian 12 [#single-node-past-60]_,Debian 12 [#single-node-past-60]_,Debian 12 [#single-node-past-60]_,,,,,,,,,,,
+,Azure Linux 3.0 [#mi300x-past-60]_,Azure Linux 3.0 [#mi300x-past-60]_,Azure Linux 3.0 [#mi300x-past-60]_,,,,,,,,,,,,
+      ,.. _architecture-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      :doc:`Architecture <rocm-install-on-linux:reference/system-requirements>`,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3,CDNA3
+      ,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2,CDNA2
+      ,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA,CDNA
+      ,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3,RDNA3
+      ,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2,RDNA2
+      ,.. _gpu-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      :doc:`GPU / LLVM target <rocm-install-on-linux:reference/system-requirements>`,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100,gfx1100
+      ,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030,gfx1030
+      ,gfx942,gfx942,gfx942,gfx942,gfx942,gfx942 [#mi300_624-past-60]_,gfx942 [#mi300_622-past-60]_,gfx942 [#mi300_621-past-60]_,gfx942 [#mi300_620-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_612-past-60]_, gfx942 [#mi300_611-past-60]_, gfx942 [#mi300_610-past-60]_, gfx942 [#mi300_602-past-60]_, gfx942 [#mi300_600-past-60]_
+      ,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a,gfx90a
+      ,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908,gfx908
+,,,,,,,,,,,,,,,
+      FRAMEWORK SUPPORT,.. _framework-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      :doc:`PyTorch <../compatibility/ml-compatibility/pytorch-compatibility>`,"2.6, 2.5, 2.4, 2.3","2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 1.13","2.4, 2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.3, 2.2, 2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13","2.1, 2.0, 1.13"
+      :doc:`TensorFlow <../compatibility/ml-compatibility/tensorflow-compatibility>`,"2.18.1, 2.17.1, 2.16.2","2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.17.0, 2.16.2, 2.15.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.16.1, 2.15.1, 2.14.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.15.0, 2.14.0, 2.13.1","2.14.0, 2.13.1, 2.12.1","2.14.0, 2.13.1, 2.12.1"
+      :doc:`JAX <../compatibility/ml-compatibility/jax-compatibility>`,0.4.35,0.4.31,0.4.31,0.4.31,0.4.31,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26,0.4.26
+      `ONNX Runtime <https://onnxruntime.ai/docs/build/eps.html#amd-migraphx>`_,1.2,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.17.3,1.14.1,1.14.1
+,,,,,,,,,,,,,,,
+      ,,,,,,,,,,,,,,,
+      THIRD PARTY COMMS,.. _thirdpartycomms-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      `UCC <https://github.com/ROCm/ucc>`_,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.3.0,>=1.2.0,>=1.2.0
+      `UCX <https://github.com/ROCm/ucx>`_,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.15.0,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1,>=1.14.1
+      ,,,,,,,,,,,,,,,
+      THIRD PARTY ALGORITHM,.. _thirdpartyalgorithm-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      Thrust,2.5.0,2.3.2,2.3.2,2.3.2,2.3.2,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+      CUB,2.5.0,2.3.2,2.3.2,2.3.2,2.3.2,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.1,2.0.1
+,,,,,,,,,,,,,,,
+      KMD & USER SPACE [#kfd_support-past-60]_,.. _kfd-userspace-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      KMD versions,"6.4.x, 6.3.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x, 5.7.x","6.2.x, 6.1.x, 6.0.x, 5.7.x, 5.6.x","6.2.x, 6.1.x, 6.0.x, 5.7.x, 5.6.x"
+      ,,,,,,,,,,,,,,,
+      ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0
+      :doc:`MIGraphX <amdmigraphx:index>`,2.12.0,2.11.0,2.11.0,2.11.0,2.11.0,2.10.0,2.10.0,2.10.0,2.10.0,2.9.0,2.9.0,2.9.0,2.9.0,2.8.0,2.8.0
+      :doc:`MIOpen <miopen:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.0,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`MIVisionX <mivisionx:index>`,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0,3.0.0,3.0.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0,2.5.0
+      :doc:`rocAL <rocal:index>`,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0,2.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`rocDecode <rocdecode:index>`,0.10.0,0.8.0,0.8.0,0.8.0,0.8.0,0.6.0,0.6.0,0.6.0,0.6.0,0.6.0,0.6.0,0.5.0,0.5.0,N/A,N/A
+      :doc:`rocJPEG <rocjpeg:index>`,0.8.0,0.6.0,0.6.0,0.6.0,0.6.0,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A
+      :doc:`rocPyDecode <rocpydecode:index>`,0.3.1,0.2.0,0.2.0,0.2.0,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0,N/A,N/A,N/A,N/A,N/A,N/A
+      :doc:`RPP <rpp:index>`,1.9.10,1.9.1,1.9.1,1.9.1,1.9.1,1.8.0,1.8.0,1.8.0,1.8.0,1.5.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0
+      ,,,,,,,,,,,,,,,
+      COMMUNICATION,.. _commlibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      :doc:`RCCL <rccl:index>`,2.22.3,2.21.5,2.21.5,2.21.5,2.21.5,2.20.5,2.20.5,2.20.5,2.20.5,2.18.6,2.18.6,2.18.6,2.18.6,2.18.3,2.18.3
+      ,,,,,,,,,,,,,,,
+      MATH LIBS,.. _mathlibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      `half <https://github.com/ROCm/half>`_ ,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0,1.12.0
+      :doc:`hipBLAS <hipblas:index>`,2.4.0,2.3.0,2.3.0,2.3.0,2.3.0,2.2.0,2.2.0,2.2.0,2.2.0,2.1.0,2.1.0,2.1.0,2.1.0,2.0.0,2.0.0
+      :doc:`hipBLASLt <hipblaslt:index>`,0.12.0,0.10.0,0.10.0,0.10.0,0.10.0,0.8.0,0.8.0,0.8.0,0.8.0,0.7.0,0.7.0,0.7.0,0.7.0,0.6.0,0.6.0
+      :doc:`hipFFT <hipfft:index>`,1.0.18,1.0.17,1.0.17,1.0.17,1.0.17,1.0.16,1.0.15,1.0.15,1.0.14,1.0.14,1.0.14,1.0.14,1.0.14,1.0.13,1.0.13
+      :doc:`hipfort <hipfort:index>`,0.6.0,0.5.1,0.5.1,0.5.0,0.5.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0
+      :doc:`hipRAND <hiprand:index>`,2.12.0,2.11.1,2.11.1,2.11.1,2.11.0,2.11.1,2.11.0,2.11.0,2.11.0,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16,2.10.16
+      :doc:`hipSOLVER <hipsolver:index>`,2.4.0,2.3.0,2.3.0,2.3.0,2.3.0,2.2.0,2.2.0,2.2.0,2.2.0,2.1.1,2.1.1,2.1.1,2.1.0,2.0.0,2.0.0
+      :doc:`hipSPARSE <hipsparse:index>`,3.2.0,3.1.2,3.1.2,3.1.2,3.1.2,3.1.1,3.1.1,3.1.1,3.1.1,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      :doc:`hipSPARSELt <hipsparselt:index>`,0.2.3,0.2.2,0.2.2,0.2.2,0.2.2,0.2.1,0.2.1,0.2.1,0.2.1,0.2.0,0.2.0,0.1.0,0.1.0,0.1.0,0.1.0
+      :doc:`rocALUTION <rocalution:index>`,3.2.2,3.2.1,3.2.1,3.2.1,3.2.1,3.2.1,3.2.0,3.2.0,3.2.0,3.1.1,3.1.1,3.1.1,3.1.1,3.0.3,3.0.3
+      :doc:`rocBLAS <rocblas:index>`,4.4.0,4.3.0,4.3.0,4.3.0,4.3.0,4.2.4,4.2.1,4.2.1,4.2.0,4.1.2,4.1.2,4.1.0,4.1.0,4.0.0,4.0.0
+      :doc:`rocFFT <rocfft:index>`,1.0.32,1.0.31,1.0.31,1.0.31,1.0.31,1.0.30,1.0.29,1.0.29,1.0.28,1.0.27,1.0.27,1.0.27,1.0.26,1.0.25,1.0.23
+      :doc:`rocRAND <rocrand:index>`,3.3.0,3.2.0,3.2.0,3.2.0,3.2.0,3.1.1,3.1.0,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,2.10.17
+      :doc:`rocSOLVER <rocsolver:index>`,3.28.0,3.27.0,3.27.0,3.27.0,3.27.0,3.26.2,3.26.0,3.26.0,3.26.0,3.25.0,3.25.0,3.25.0,3.25.0,3.24.0,3.24.0
+      :doc:`rocSPARSE <rocsparse:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.1,3.2.0,3.2.0,3.2.0,3.1.2,3.1.2,3.1.2,3.1.2,3.0.2,3.0.2
+      :doc:`rocWMMA <rocwmma:index>`,1.7.0,1.6.0,1.6.0,1.6.0,1.6.0,1.5.0,1.5.0,1.5.0,1.5.0,1.4.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0
+      :doc:`Tensile <tensile:src/index>`,4.43.0,4.42.0,4.42.0,4.42.0,4.42.0,4.41.0,4.41.0,4.41.0,4.41.0,4.40.0,4.40.0,4.40.0,4.40.0,4.39.0,4.39.0
+      ,,,,,,,,,,,,,,,
+      PRIMITIVES,.. _primitivelibs-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      :doc:`hipCUB <hipcub:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.1,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`hipTensor <hiptensor:index>`,1.5.0,1.4.0,1.4.0,1.4.0,1.4.0,1.3.0,1.3.0,1.3.0,1.3.0,1.2.0,1.2.0,1.2.0,1.2.0,1.1.0,1.1.0
+      :doc:`rocPRIM <rocprim:index>`,3.4.0,3.3.0,3.3.0,3.3.0,3.3.0,3.2.2,3.2.0,3.2.0,3.2.0,3.1.0,3.1.0,3.1.0,3.1.0,3.0.0,3.0.0
+      :doc:`rocThrust <rocthrust:index>`,3.3.0,3.3.0,3.3.0,3.3.0,3.3.0,3.1.1,3.1.0,3.1.0,3.0.1,3.0.1,3.0.1,3.0.1,3.0.1,3.0.0,3.0.0
+      ,,,,,,,,,,,,,,,
+      SUPPORT LIBS,,,,,,,,,,,,,,,
+      `hipother <https://github.com/ROCm/hipother>`_,6.4.43482,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      `rocm-core <https://github.com/ROCm/rocm-core>`_,6.4.0,6.3.3,6.3.2,6.3.1,6.3.0,6.2.4,6.2.2,6.2.1,6.2.0,6.1.2,6.1.2,6.1.1,6.1.0,6.0.2,6.0.0
+      `ROCT-Thunk-Interface <https://github.com/ROCm/ROCT-Thunk-Interface>`_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,N/A [#ROCT-rocr-past-60]_,20240607.5.7,20240607.5.7,20240607.4.05,20240607.1.4246,20240125.5.08,20240125.5.08,20240125.5.08,20240125.3.30,20231016.2.245,20231016.2.245
+      ,,,,,,,,,,,,,,,
+      SYSTEM MGMT TOOLS,.. _tools-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      :doc:`AMD SMI <amdsmi:index>`,25.3.0,24.7.1,24.7.1,24.7.1,24.7.1,24.6.3,24.6.3,24.6.3,24.6.2,24.5.1,24.5.1,24.5.1,24.4.1,23.4.2,23.4.2
+      :doc:`ROCm Data Center Tool <rdc:index>`,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0,0.3.0
+      :doc:`rocminfo <rocminfo:index>`,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      :doc:`ROCm SMI <rocm_smi_lib:index>`,7.5.0,7.4.0,7.4.0,7.4.0,7.4.0,7.3.0,7.3.0,7.3.0,7.3.0,7.2.0,7.2.0,7.0.0,7.0.0,6.0.2,6.0.0
+      :doc:`ROCm Validation Suite <rocmvalidationsuite:index>`,1.1.0,1.1.0,1.1.0,1.1.0,1.1.0,1.0.60204,1.0.60202,1.0.60201,1.0.60200,1.0.60102,1.0.60102,1.0.60101,1.0.60100,1.0.60002,1.0.60000
+      ,,,,,,,,,,,,,,,
+      PERFORMANCE TOOLS,,,,,,,,,,,,,,,
+      :doc:`ROCm Bandwidth Test <rocm_bandwidth_test:index>`,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0,1.4.0
+      :doc:`ROCm Compute Profiler <rocprofiler-compute:index>`,3.1.0,3.0.0,3.0.0,3.0.0,3.0.0,2.0.1,2.0.1,2.0.1,2.0.1,N/A,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCm Systems Profiler <rocprofiler-systems:index>`,1.0.0,0.1.2,0.1.1,0.1.0,0.1.0,1.11.2,1.11.2,1.11.2,1.11.2,N/A,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCProfiler <rocprofiler:index>`,2.0.60400,2.0.60303,2.0.60302,2.0.60301,2.0.60300,2.0.60204,2.0.60202,2.0.60201,2.0.60200,2.0.60102,2.0.60102,2.0.60101,2.0.60100,2.0.60002,2.0.60000
+      :doc:`ROCprofiler-SDK <rocprofiler-sdk:index>`,0.6.0,0.5.0,0.5.0,0.5.0,0.5.0,0.4.0,0.4.0,0.4.0,0.4.0,N/A,N/A,N/A,N/A,N/A,N/A
+      :doc:`ROCTracer <roctracer:index>`,4.1.60400,4.1.60303,4.1.60302,4.1.60301,4.1.60300,4.1.60204,4.1.60202,4.1.60201,4.1.60200,4.1.60102,4.1.60102,4.1.60101,4.1.60100,4.1.60002,4.1.60000
+      ,,,,,,,,,,,,,,,
+      DEVELOPMENT TOOLS,,,,,,,,,,,,,,,
+      :doc:`HIPIFY <hipify:index>`,19.0.0.25104,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24455,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`ROCm CMake <rocmcmakebuildtools:index>`,0.14.0,0.14.0,0.14.0,0.14.0,0.14.0,0.13.0,0.13.0,0.13.0,0.13.0,0.12.0,0.12.0,0.12.0,0.12.0,0.11.0,0.11.0
+      :doc:`ROCdbgapi <rocdbgapi:index>`,0.77.2,0.77.0,0.77.0,0.77.0,0.77.0,0.76.0,0.76.0,0.76.0,0.76.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0,0.71.0
+      :doc:`ROCm Debugger (ROCgdb) <rocgdb:index>`,15.2.0,15.2.0,15.2.0,15.2.0,15.2.0,14.2.0,14.2.0,14.2.0,14.2.0,14.1.0,14.1.0,14.1.0,14.1.0,13.2.0,13.2.0
+      `rocprofiler-register <https://github.com/ROCm/rocprofiler-register>`_,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.4.0,0.3.0,0.3.0,0.3.0,0.3.0,N/A,N/A
+      :doc:`ROCr Debug Agent <rocr_debug_agent:index>`,2.0.4,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3,2.0.3
+      ,,,,,,,,,,,,,,,
+      COMPILERS,.. _compilers-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      `clang-ocl <https://github.com/ROCm/clang-ocl>`_,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0,0.5.0
+      :doc:`hipCC <hipcc:index>`,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.1.1,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0,1.0.0
+      `Flang <https://github.com/ROCm/flang>`_,19.0.0.25133,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24455,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      :doc:`llvm-project <llvm-project:index>`,19.0.0.25133,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24491,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+      `OpenMP <https://github.com/ROCm/llvm-project/tree/amd-staging/openmp>`_,19.0.0.25133,18.0.0.25012,18.0.0.25012,18.0.0.24491,18.0.0.24491,18.0.0.24392,18.0.0.24355,18.0.0.24355,18.0.0.24232,17.0.0.24193,17.0.0.24193,17.0.0.24154,17.0.0.24103,17.0.0.24012,17.0.0.23483
+,,,,,,,,,,,,,,,
+      RUNTIMES,.. _runtime-support-compatibility-matrix-past-60:,,,,,,,,,,,,,,
+      :doc:`AMD CLR <hip:understand/amd_clr>`,6.4.43482,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      :doc:`HIP <hip:index>`,6.4.43482,6.3.42134,6.3.42134,6.3.42133,6.3.42131,6.2.41134,6.2.41134,6.2.41134,6.2.41133,6.1.40093,6.1.40093,6.1.40092,6.1.40091,6.1.32831,6.1.32830
+      `OpenCL Runtime <https://github.com/ROCm/clr/tree/develop/opencl>`_,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0,2.0.0
+      :doc:`ROCr Runtime <rocr-runtime:index>`,1.15.0,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.14.0,1.13.0,1.13.0,1.13.0,1.13.0,1.13.0,1.12.0,1.12.0

docs/compatibility/compatibility-matrix.rst

@@ -62,7 +62,7 @@ compatibility and system requirements.
       CUB,2.5.0,2.3.2,2.2.0
       ,,,
       KMD & USER SPACE [#kfd_support]_,.. _kfd-userspace-support-compatibility-matrix:,,
-      Tested user space versions,"6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.3.x, 6.2.x, 6.1.x, 6.0.x"
+      KMD versions,"6.4.x, 6.3.x","6.4.x, 6.3.x, 6.2.x, 6.1.x","6.4.x, 6.3.x, 6.2.x, 6.1.x, 6.0.x"
       ,,,
       ML & COMPUTER VISION,.. _mllibs-support-compatibility-matrix:,,
       :doc:`Composable Kernel <composable_kernel:index>`,1.1.0,1.1.0,1.1.0

docs/conf.py

@@ -42,7 +42,7 @@ all_article_info_author = ""
 
 # pages with specific settings
 article_pages = [
-    {"file": "about/release-notes", "os": ["linux"], "date": "2025-04-10"},
+    {"file": "about/release-notes", "os": ["linux"], "date": "2025-04-11"},
     {"file": "release/changelog", "os": ["linux"],},
     {"file": "compatibility/compatibility-matrix", "os": ["linux"]},
     {"file": "compatibility/ml-compatibility/pytorch-compatibility", "os": ["linux"]},

docs/how-to/rocm-for-ai/inference-optimization/workload.rst

@@ -685,7 +685,7 @@ Two sample Llama scaling configuration files are in vLLM for ``llama2-70b`` and
 ``llama2-7b``.
 
 If building the vLLM using
-`Dockerfile.rocm <https://github.com/vllm-project/vllm/blob/main/Dockerfile.rocm>`_
+`Dockerfile.rocm <https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm>`_
 for ``llama2-70b`` scale config, find the file at
 ``/vllm-workspace/tests/fp8_kv/llama2-70b-fp8-kv/kv_cache_scales.json`` at
 runtime.

tools/rocm-build/ROCm.mk

@@ -12,17 +12,36 @@ RELEASE_FLAG?=-r
 ASAN_DEP:=
 ifeq (${ENABLE_ADDRESS_SANITIZER},true)
 	ASAN_DEP=lightning
-	SANITIZER_FLAG=-a
+	SANITIZER_FLAG=--address_sanitizer
+endif
+
+# Set STATIC_FLAG for static builds
+ifeq (${ENABLE_STATIC_BUILDS},true)
+	STATIC_FLAG=-s
 endif
 
 export INFRA_REPO:=ROCm/tools/rocm-build
 
+# # commannds to be run at makefile read time
+# # should only output $$OUT_DIR to stdout
+# # In an ideal world this would be another target
+# define INITBUILD
+# source ${INFRA_REPO}/envsetup.sh >/dev/null 2>&1;
+# [ -w "$${ROCM_INSTALL_PATH}" ] || sudo mkdir -p -m 775 "$${ROCM_INSTALL_PATH}" ;
+# sudo chown "$$(id -u):$$(id -g)" "$${ROCM_INSTALL_PATH}" "/home/$$(id -un)" ;
+# mkdir -p ${HOME}/.ccache ;
+# echo $${OUT_DIR} ;
+# endef
+
 OUT_DIR:=$(shell . ${INFRA_REPO}/envsetup.sh >/dev/null 2>&1 ; echo $${OUT_DIR})
 ROCM_INSTALL_PATH:=$(shell . ${INFRA_REPO}/envsetup.sh >/dev/null 2>&1 ; echo $${ROCM_INSTALL_PATH})
 
 $(info OUT_DIR=${OUT_DIR})
 $(info ROCM_INSTALL_PATH=${ROCM_INSTALL_PATH})
 
+# define SILENT to be empty to see the runner invocation
+SILENT?= @
+
 # -------------------------------------------------------------------------
 # Internal stuff. Could be put in a different file to hide it.
 # Internal macros, they need to be defined before being used.
@@ -47,7 +66,8 @@ endif
 # It should not be needed.
 define adddep =
 $(strip $(call peval,components+= $(1) $(2))
-$(foreach comp,$(strip $2),$(call peval,${OUT_DIR}/logs/${1}: ${OUT_DIR}/logs/${comp}))
+$(call peval,$(1)_DEPS += $(2))
+$(foreach comp,$(strip $2),$(call peval,${OUT_DIR}/logs/${1}.txt: ${OUT_DIR}/logs/${comp}.txt))
 )
 endef
 # End of internal stuff that is needed at the start of the file
@@ -83,27 +103,29 @@ $(call adddep,rocm-core,${ASAN_DEP})
 $(call adddep,rocm-gdb,dbgapi)
 $(call adddep,rocminfo,${ASAN_DEP} rocr)
 $(call adddep,rocprofiler-register,${ASAN_DEP})
-$(call adddep,rocprofiler-sdk,${ASAN_DEP} rocr aqlprofile opencl_on_rocclr hip_on_rocclr comgr)
+$(call adddep,rocprofiler-sdk,${ASAN_DEP} rocr aqlprofile opencl_on_rocclr hip_on_rocclr comgr rccl rocdecode)
 $(call adddep,rocprofiler-systems,${ASAN_DEP} hipcc rocr hip_on_rocclr rocm_smi_lib rocprofiler roctracer rocprofiler-sdk)
 $(call adddep,rocprofiler,${ASAN_DEP} rocr roctracer aqlprofile opencl_on_rocclr hip_on_rocclr comgr)
 $(call adddep,rocprofiler-compute,${ASAN_DEP})
 $(call adddep,rocr,${ASAN_DEP} lightning rocm_smi_lib devicelibs rocprofiler-register)
 $(call adddep,rocr_debug_agent,${ASAN_DEP} hip_on_rocclr rocr dbgapi)
+$(call adddep,rocrsamples,lightning devicelibs rocr )
 $(call adddep,roctracer,${ASAN_DEP} rocr hip_on_rocclr)
 
+
 # rocm-dev points to all possible last finish components of Stage1 build.
 rocm-dev-components :=amd_smi_lib aqlprofile comgr dbgapi devicelibs hip_on_rocclr hipcc hipify_clang \
 	lightning rocprofiler-compute opencl_on_rocclr openmp_extras rocm_bandwidth_test rocm_smi_lib \
 	rocm-cmake rocm-core rocm-gdb rocminfo rocprofiler-register rocprofiler-sdk rocprofiler-systems \
-	rocprofiler rocr rocr_debug_agent roctracer
-$(call adddep,rocm-dev,$(filter-out ${NOBUILD} kernel_ubuntu,${rocm-dev-components}))
+	rocprofiler rocr rocr_debug_agent rocrsamples roctracer
+$(call adddep,rocm-dev,$(filter-out ${NOBUILD},${rocm-dev-components}))
 
 $(call adddep,amdmigraphx,hip_on_rocclr half rocblas miopen-hip lightning hipcc hiptensor)
 $(call adddep,composable_kernel,lightning hipcc hip_on_rocclr rocm-cmake)
 $(call adddep,half,rocm-cmake)
 $(call adddep,hipblas-common,lightning)
 $(call adddep,hipblas,hip_on_rocclr rocblas rocsolver lightning hipcc)
-$(call adddep,hipblaslt,hip_on_rocclr openmp_extras lightning hipcc hipblas-common rocm-dev)
+$(call adddep,hipblaslt,hip_on_rocclr openmp_extras lightning hipcc hipblas-common roctracer)
 $(call adddep,hipcub,hip_on_rocclr rocprim lightning hipcc)
 $(call adddep,hipfft,hip_on_rocclr openmp_extras rocfft rocrand hiprand lightning hipcc)
 $(call adddep,hipfort,rocblas hipblas rocsparse hipsparse rocfft hipfft rocrand hiprand rocsolver hipsolver lightning hipcc)
@@ -113,26 +135,34 @@ $(call adddep,hipsparse,hip_on_rocclr rocsparse lightning hipcc)
 $(call adddep,hipsparselt,hip_on_rocclr hipsparse lightning hipcc openmp_extras)
 $(call adddep,hiptensor,hip_on_rocclr composable_kernel lightning hipcc)
 $(call adddep,miopen-deps,lightning hipcc)
-$(call adddep,miopen-hip,composable_kernel half hip_on_rocclr miopen-deps hipblas hipblaslt rocrand roctracer lightning hipcc)
+$(call adddep,miopen-hip,rocm-core composable_kernel half hip_on_rocclr miopen-deps hipblas hipblaslt rocrand roctracer lightning hipcc)
 $(call adddep,mivisionx,amdmigraphx miopen-hip rpp lightning hipcc)
 $(call adddep,rccl,rocm-core hip_on_rocclr rocr lightning hipcc rocm_smi_lib hipify_clang)
-$(call adddep,rdc,rocm_smi_lib rocprofiler rocmvalidationsuite)
+$(call adddep,rdc,amd_smi_lib rocprofiler-sdk rocm_smi_lib rocprofiler  rocmvalidationsuite)
 $(call adddep,rocalution,rocblas rocsparse rocrand lightning hipcc)
-$(call adddep,rocblas,hip_on_rocclr openmp_extras lightning hipcc hipblaslt)
+$(call adddep,rocblas,rocminfo hip_on_rocclr openmp_extras lightning hipcc hipblaslt)
 $(call adddep,rocal,mivisionx)
-$(call adddep,rocdecode,hip_on_rocclr lightning hipcc amdmigraphx)
+$(call adddep,rocdecode,hip_on_rocclr lightning hipcc)
 $(call adddep,rocfft,hip_on_rocclr rocrand hiprand lightning hipcc openmp_extras)
-$(call adddep,rocjpeg,hip_on_rocclr lightning hipcc rocm-dev)
+$(call adddep,rocjpeg,hip_on_rocclr lightning hipcc)
 $(call adddep,rocmvalidationsuite,hip_on_rocclr rocr hipblas hiprand hipblaslt rocm-core lightning hipcc rocm_smi_lib)
 $(call adddep,rocprim,hip_on_rocclr lightning hipcc)
 $(call adddep,rocrand,hip_on_rocclr lightning hipcc)
+$(call adddep,rocshmem,rccl )
 $(call adddep,rocsolver,hip_on_rocclr rocblas rocsparse rocprim lightning hipcc)
 $(call adddep,rocsparse,hip_on_rocclr rocprim lightning hipcc)
 $(call adddep,rocthrust,hip_on_rocclr rocprim lightning hipcc)
 $(call adddep,rocwmma,hip_on_rocclr rocblas lightning hipcc rocm-cmake rocm_smi_lib)
 $(call adddep,rpp,half lightning hipcc openmp_extras)
 $(call adddep,transferbench,hip_on_rocclr lightning hipcc)
+ifneq ($(filter rocm-dev upload-rocm-dev, ${MAKECMDGOALS}),)
+	components = $(rocm-dev-components)
+endif
+$(call adddep,rocm,$(filter-out ${NOBUILD} rocm,${components}))
 
+ifeq ($(DISTRO_NAME),rhel)
+    WHL_GEN :=
+endif
 
 # -------------------------------------------------------------------------
 # The rest of the file is internal
@@ -165,7 +195,7 @@ ifeq (${toplevel},)
 define toplevel =
 
 # The "target" make, this builds the package if it is out of date
-T_$1: ${OUT_DIR}/logs/$1 FRC
+T_$1: ${OUT_DIR}/logs/$1.txt FRC
 	:              $1 built
 
 # The "upload" for $1, it uploads the packages for $1 to the central storage
@@ -176,25 +206,26 @@ U_$1: T_$1 FRC
 # The "clean" for $1, it just marks the target as not existing so it will be built
 # in the future.
 C_$1: FRC
-	rm -f ${OUT_DIR}/logs/$1 ${OUT_DIR}/logs/$1.repackaged
+	rm -f ${OUT_DIR}/logs/$1.txt ${OUT_DIR}/logs/$1.repackaged
 
-# parallel build {
-${OUT_DIR}/logs/$1: | ${OUT_DIR}/logs
-ifneq ($(wildcard ${OUT_DIR}/logs/$1.repackaged),)
+# parallel build
+${OUT_DIR}/logs/$1.txt: | ${OUT_DIR}/logs
+ifneq ($(wildcard ${OUT_DIR}/logs/$1.repackaged),) # {
 	@echo  Skipping build of $1 as it has already been repackaged
-	cat $$@.repackaged > $$@
-	rm -f $$@.repackaged
+	cat $${@:.txt=.repackaged} > $$@
+	rm -f $${@:.txt=.repackaged}
 else # } {
 	@echo  $1 started due to $$? | sed "s:${OUT_DIR}/logs/::g"
 # Build in a subshell so we get the time output
 # Pass in jobserver info using the RMAKE variable
-	${RMAKE}@( if set -x && source $${INFRA_REPO}/envsetup.sh && \
-	rm -f $$@.errors $$@ $$@.repackaged && \
-	$${INFRA_REPO}/build_$1.sh -c && \
-	time bash -x $${INFRA_REPO}/build_$1.sh $${RELEASE_FLAG} $${SANITIZER_FLAG} && $${INFRA_REPO}/post_inst_pkg.sh "$1" ; \
-	then mv $$@.inprogress $$@ ; \
-	else mv $$@.inprogress $$@.errors ; echo Error in $1 >&2 ; exit 1 ;\
-	fi ) > $$@.inprogress 2>&1
+# Allow project specific flags e..g. ROCMBUILD_lightning.
+	${RMAKE}${SILENT}( if set -x && source $${INFRA_REPO}/envsetup.sh && \
+	rm -f $${@:$1.txt=1.Errors.$1.txt} $$@ $${@:.txt=.repackaged} && \
+	$${INFRA_REPO}/runner $1 $${RELEASE_FLAG} $${SANITIZER_FLAG} $${STATIC_FLAG} ${ROCMBUILD_$1}; \
+	then mv $${@:$1.txt=2.Inprogress.$1.txt} $$@ ; \
+	else mv $${@:$1.txt=2.Inprogress.$1.txt} $${@:$1.txt=1.Errors.$1.txt} ;\
+		echo Error in $1 >&2 ; exit 1 ;\
+	fi ) > $${@:$1.txt=2.Inprogress.$1.txt} 2>&1
 endif # }
 
 # end of toplevel macro
@@ -227,22 +258,45 @@ upload-rocm-dev: $(addprefix U_,$(filter-out ${NOBUILD},${components}))
 rocm-dev: $(addprefix T_,$(filter-out ${NOBUILD},${components}))
 	@echo rocm-dev built
 
+ifeq ($(DISTRO_NAME),almalinux)
+	@sudo chmod -R 777 "/home/builder"
+endif
+
+# This code is broken. It stops us exiting a container and
+# starting a new one and continueing the build. The attempt
+# is to have run-once code.
 ${OUT_DIR}/logs:
 	sudo mkdir -p -m 775 "${ROCM_INSTALL_PATH}" && \
 	sudo chown -R "$(shell id -u):$(shell id -g)" "/opt"
-	sudo chown -R "$(shell id -u):$(shell id -g)" "/home/$(shell id -un)"
+	sudo chown "$(shell id -u):$(shell id -g)" "/home/$(shell id -un)"
 	mkdir -p "${@}"
 	mkdir -p ${HOME}/.ccache
 
 ##help clean: remove the output directory and recreate it
 clean:
 	[ -n "${OUT_DIR}" ] && rm -rf "${OUT_DIR}"
-	mkdir -p ${OUT_DIR}/logs
+#	mkdir -p ${OUT_DIR}/logs
 
 .SECONDARY: ${components:%=${OUT_DIR}/logs/%}
 
 .PHONY: all clean repack help list_components
 
+# get_all_deps: Recursively get all dependencies for a given component.
+# Usage: $(call get_all_deps,component_name,)
+# - component_name: The name of the component to get dependencies for.
+# - The second parameter is an internal parameter used to track already
+#   processed components to avoid circular dependencies.
+define get_all_deps
+$(if $(filter $(1),$(2)),,\
+	$(sort $(1) $(foreach d,$($(1)_DEPS),$(call get_all_deps,$d,$(1) $(2))))
+)
+endef
+
+##help deps_<component>: output the dependencies for <component>
+deps_%:
+	@echo "=== Dependencies for [$*] ==="
+	@echo "$(filter-out $*,$(call get_all_deps,$*,))"
+
 ##help list_components: output the list of components
 ##help : Hint make list_components | paste - - - | column -t
 list_components:

tools/rocm-build/build_amd_smi_lib.sh

@@ -1,4 +1,3 @@
-#!/bin/bash
 
 source "$(dirname "${BASH_SOURCE}")/compute_utils.sh"
 
@@ -10,7 +9,9 @@ printUsage() {
     echo "  -c,  --clean              Removes all amd_smi build artifacts"
     echo "  -r,  --release            Build non-debug version amd_smi (default is debug)"
     echo "  -a,  --address_sanitizer  Enable address sanitizer"
-    echo "  -s,  --static             Build static lib (.a).  build instead of dynamic/shared(.so) "
+    echo "  -s,  --static             Component/Build does not support static builds just accepting this param & ignore. No effect of the param on this build"
+    echo "  -w,  --wheel              Creates python wheel package of amd-smi. 
+                                      It needs to be used along with -r option"
     echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of type referred to by pkg_type"
     echo "  -p,  --package <type>     Specify packaging format"
     echo "  -h,  --help               Prints this help"
@@ -25,7 +26,6 @@ printUsage() {
 PROJ_NAME="amdsmi"
 PACKAGE_ROOT="$(getPackageRoot)"
 TARGET="build"
-
 PACKAGE_LIB=$(getLibPath)
 PACKAGE_INCLUDE="$(getIncludePath)"
 AMDSMI_BUILD_DIR=$(getBuildPath $PROJ_NAME)
@@ -42,7 +42,7 @@ SHARED_LIBS="ON"
 CLEAN_OR_OUT=0;
 PKGTYPE="deb"
 
-VALID_STR=`getopt -o hcraso:p: --long help,clean,release,static,address_sanitizer,outdir:,package: -- "$@"`
+VALID_STR=`getopt -o hcraswo:p: --long help,clean,release,static,wheel,address_sanitizer,outdir:,package: -- "$@"`
 eval set -- "$VALID_STR"
 
 while true ;
@@ -60,6 +60,8 @@ do
                 ADDRESS_SANITIZER=true ; shift ;;
         (-s | --static)
                 ack_and_skip_static ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ; shift ;;
         (-o | --outdir)
                 TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 2 ;;
         (-p | --package)
@@ -135,7 +137,7 @@ verifyEnvSetup
 
 case $TARGET in
     (clean) clean_amdsmi ;;
-    (build) build_amdsmi ;;
+    (build) build_amdsmi; build_wheel "$AMDSMI_BUILD_DIR" "$PROJ_NAME" ;;
     (outdir) print_output_directory ;;
     (*) die "Invalid target $TARGET" ;;
 esac

tools/rocm-build/build_amdmigraphx.sh

@@ -1,41 +1,42 @@
 #!/bin/bash
 
 set -ex
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src AMDMIGraphX
 
 build_amdmigraphx() {
     echo "Start build"
 
+    if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
+        ack_and_skip_static
+    fi
+
     cd $COMPONENT_SRC
 
     if ! command -v rbuild &> /dev/null; then
         pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
     fi
 
+    # Remove CK
+    xargs -d '\n' -a ${OUT_DIR}/ck.files rm -- || true
+
     if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
          set_asan_env_vars
          set_address_sanitizer_on
     fi
 
-    if [ -n "$GPU_ARCHS" ]; then
-        GPU_TARGETS="$GPU_ARCHS"
-    else
-        GPU_TARGETS="gfx900;gfx906;gfx908;gfx90a;gfx1030;gfx1100;gfx1101;gfx1102;gfx942;gfx1200;gfx1201"
-    fi
     init_rocm_common_cmake_params
 
     mkdir -p ${BUILD_DIR} && rm -rf ${BUILD_DIR}/* && mkdir -p ${HOME}/amdmigraphx && rm -rf ${HOME}/amdmigraphx/*
     rbuild package -d "${HOME}/amdmigraphx" -B "${BUILD_DIR}" \
-        --cxx="${ROCM_PATH}/llvm/bin/clang++" \
-        --cc="${ROCM_PATH}/llvm/bin/clang" \
+        --cxx="$(set_build_variables __CLANG++__)" \
+        --cc="$(set_build_variables __CLANG__)" \
         "${rocm_math_common_cmake_params[@]}" \
         -DCMAKE_MODULE_LINKER_FLAGS="-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ROCM_LIB_RPATH" \
-	    -DGPU_TARGETS="${GPU_TARGETS}" \
         -DCMAKE_INSTALL_RPATH=""
 
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
     cd $BUILD_DIR && cmake --build . -- install -j${PROC}
 
     show_build_cache_stats
@@ -50,7 +51,7 @@ clean_amdmigraphx() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_amdmigraphx ;;
+    build) build_amdmigraphx; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_amdmigraphx ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_clang-ocl.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+
+source "$(dirname "${BASH_SOURCE}")/compute_utils.sh"
+
+printUsage() {
+    echo
+    echo "Usage: $(basename "${BASH_SOURCE}") [-c|-r|-h] [makeopts]"
+    echo
+    echo "Options:"
+    echo "  -c,  --clean              Removes all clang-ocl build artifacts"
+    echo "  -r,  --release            Build non-debug version clang-ocl (default is debug)"
+    echo "  -a,  --address_sanitizer  Enable address sanitizer"
+    echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of
+                                      type referred to by pkg_type"
+    echo "  -h,  --help               Prints this help"
+    echo "  -s,  --static             Supports static CI by accepting this param & not bailing out. No effect of the param though"
+    echo
+
+    return 0
+}
+
+
+TARGET="build"
+CLANG_OCL_DEST="$(getBinPath)"
+CLANG_OCL_SRC_ROOT="$CLANG_OCL_ROOT"
+CLANG_OCL_BUILD_DIR="$(getBuildPath clang-ocl)"
+
+MAKEARG="$DASH_JAY"
+PACKAGE_ROOT="$(getPackageRoot)"
+PACKAGE_UTILS="$(getUtilsPath)"
+CLANG_OCL_PACKAGE_DEB="$PACKAGE_ROOT/deb/clang-ocl"
+CLANG_OCL_PACKAGE_RPM="$PACKAGE_ROOT/rpm/clang-ocl"
+BUILD_TYPE="Debug"
+SHARED_LIBS="ON"
+CLEAN_OR_OUT=0;
+MAKETARGET="deb"
+PKGTYPE="deb"
+
+
+VALID_STR=`getopt -o hcraso:g: --long help,clean,release,clean,static,address_sanitizer,outdir:,gpu_list: -- "$@"`
+eval set -- "$VALID_STR"
+
+while true ;
+do
+    case "$1" in
+        (-h | --help)
+                printUsage ; exit 0;;
+        (-c | --clean)
+                TARGET="clean" ; ((CLEAN_OR_OUT|=1)) ; shift ;;
+        (-r | --release)
+                MAKEARG="$MAKEARG BUILD_TYPE=rel" ;  BUILD_TYPE="Release" ; shift ;;
+        (-a | --address_sanitizer)
+                set_asan_env_vars
+                set_address_sanitizer_on ; shift ;;
+        (-s | --static)
+                SHARED_LIBS="OFF" ; shift ;;
+        (-o | --outdir)
+                TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 2 ;;
+        (-g | --gpu_list )
+                GPU_LIST=$2; shift 2 ;;
+        --)     shift; break;;
+        (*)
+                echo " This should never come but just incase : UNEXPECTED ERROR Parm : [$1] ">&2 ; exit 20;;
+    esac
+
+done
+
+RET_CONFLICT=1
+check_conflicting_options $CLEAN_OR_OUT $PKGTYPE $MAKETARGET
+if [ $RET_CONFLICT -ge 30 ]; then
+   print_vars $API_NAME $TARGET $BUILD_TYPE $SHARED_LIBS $CLEAN_OR_OUT $PKGTYPE $MAKETARGET
+   exit $RET_CONFLICT
+fi
+
+clean_clang-ocl() {
+    echo "Removing clang-ocl"
+    rm -rf $CLANG_OCL_DEST/clang-ocl
+    rm -rf $CLANG_OCL_BUILD_DIR
+    rm -rf $CLANG_OCL_PACKAGE_DEB
+    rm -rf $CLANG_OCL_PACKAGE_RPM
+}
+
+build_clang-ocl() {
+    if [ ! -d "$CLANG_OCL_BUILD_DIR" ]; then
+        mkdir -p $CLANG_OCL_BUILD_DIR
+        pushd $CLANG_OCL_BUILD_DIR
+
+        if [ -e $PACKAGE_ROOT/lib/bitcode/opencl.amdgcn.bc ]; then
+            BC_DIR="$ROCM_INSTALL_PATH/lib"
+        else
+            BC_DIR="$ROCM_INSTALL_PATH/amdgcn/bitcode"
+        fi
+
+        cmake \
+            $(rocm_cmake_params) \
+            -DDISABLE_CHECKS="ON" \
+            -DCLANG_BIN="$ROCM_INSTALL_PATH/llvm/bin" \
+            -DBITCODE_DIR="$BC_DIR" \
+	    $(rocm_common_cmake_params) \
+            -DCPACK_SET_DESTDIR="OFF" \
+            $CLANG_OCL_SRC_ROOT
+
+        echo "Making clang-ocl:"
+        cmake --build . -- $MAKEARG
+        cmake --build . -- $MAKEARG install
+        cmake --build . -- $MAKEARG package
+        popd
+    fi
+
+    copy_if DEB "${CPACKGEN:-"DEB;RPM"}" "$CLANG_OCL_PACKAGE_DEB" $CLANG_OCL_BUILD_DIR/rocm-clang-ocl*.deb
+    copy_if RPM "${CPACKGEN:-"DEB;RPM"}" "$CLANG_OCL_PACKAGE_RPM" $CLANG_OCL_BUILD_DIR/rocm-clang-ocl*.rpm
+}
+
+
+print_output_directory() {
+     case ${PKGTYPE} in
+         ("deb")
+             echo ${CLANG_OCL_PACKAGE_DEB};;
+         ("rpm")
+             echo ${CLANG_OCL_PACKAGE_RPM};;
+         (*)
+             echo "Invalid package type \"${PKGTYPE}\" provided for -o" >&2; exit 1;;
+     esac
+     exit
+}
+
+case $TARGET in
+    (clean) clean_clang-ocl ;;
+    (build) build_clang-ocl ;;
+   (outdir) print_output_directory ;;
+        (*) die "Invalid target $TARGET" ;;
+esac
+
+echo "Operation complete"
+exit 0
+

tools/rocm-build/build_comgr.sh

@@ -14,6 +14,7 @@ printUsage() {
         type referred to by pkg_type"
     echo "  -h,  --help               Prints this help"
     echo "  -s,  --static             Build static lib (.a).  build instead of dynamic/shared(.so) "
+    echo "  -w,  --wheel              Creates python wheel package of comgr. It needs to be used along with -r option"
     echo "  -l,  --link_llvm_static   Link to LLVM statically.  Default is to dynamically link to LLVM; requires that LLVM dylibs are created."
     echo
     echo "Possible values for <type>:"
@@ -24,6 +25,7 @@ printUsage() {
     return 0
 }
 
+## Build environment variables
 API_NAME=amd_comgr
 PROJ_NAME=$API_NAME
 LIB_NAME=lib${API_NAME}
@@ -33,8 +35,8 @@ PACKAGE_ROOT=$(getPackageRoot)
 PACKAGE_LIB=$(getLibPath)
 PACKAGE_INCLUDE=$(getIncludePath)
 BUILD_DIR=$(getBuildPath $API_NAME)
-PACKAGE_DEB=$(getPackageRoot)/deb/$API_NAME
-PACKAGE_RPM=$(getPackageRoot)/rpm/$API_NAME
+PACKAGE_DEB=$PACKAGE_ROOT/deb/$PROJ_NAME
+PACKAGE_RPM=$PACKAGE_ROOT/rpm/$PROJ_NAME
 PACKAGE_PREFIX=$ROCM_INSTALL_PATH
 BUILD_TYPE=Debug
 MAKE_OPTS="$DASH_JAY CTEST_OUTPUT_ON_FAILURE=1 -C $BUILD_DIR"
@@ -44,13 +46,17 @@ CLEAN_OR_OUT=0;
 PKGTYPE="deb"
 MAKETARGET="deb"
 
+# link to LLVM dynamicaly.  Default is to link to llvm dynamically.
+# temporarily set this default to OFF until we resolve Issues
 LINK_LLVM_DYLIB="OFF"
 
-VALID_STR=`getopt -o hcraslo:p: --long help,clean,release,address_sanitizer,static,link_llvm_static,outdir:,package: -- "$@"`
+#parse the arguments
+VALID_STR=`getopt -o hcraswlo:p: --long help,clean,release,address_sanitizer,static,wheel,link_llvm_static,outdir:,package: -- "$@"`
 eval set -- "$VALID_STR"
 
 while true ;
 do
+    #echo "parocessing $1"
     case "$1" in
         (-h | --help)
                 printUsage ; exit 0;;
@@ -63,6 +69,8 @@ do
                 set_address_sanitizer_on ; shift ;;
         (-s | --static)
                 SHARED_LIBS="OFF" ; shift ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ; shift ;;
         (-l | --link_llvm_static)
                 LINK_LLVM_DYLIB="OFF"; shift ;;
         (-o | --outdir)
@@ -105,6 +113,7 @@ build() {
        echo " Building Shared Object "
     fi
 
+# Remove CTEST var once SWDEV-381396 is fixed
     cmake \
         $(rocm_cmake_params) \
         -DBUILD_SHARED_LIBS=$SHARED_LIBS \
@@ -143,7 +152,7 @@ verifyEnvSetup
 
 case $TARGET in
     (clean) clean ;;
-    (build) build ;;
+    (build) build; build_wheel "$BUILD_DIR" "$PROJ_NAME" ;;
     (outdir) print_output_directory ;;
     (*) die "Invalid target $TARGET" ;;
 esac

tools/rocm-build/build_composable_kernel.sh

@@ -2,10 +2,14 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
+
+# Temporarily disable Address Sanitizer
+ENABLE_ADDRESS_SANITIZER=false
 
 set_component_src composable_kernel
-
+disable_debug_package_generation
+# Set the GPU_ARCH_LIST to the supported GPUs needed after https://github.com/ROCm/composable_kernel/pull/1536/
 GPU_ARCH_LIST="gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201"
 
 build_miopen_ck() {
@@ -23,10 +27,17 @@ build_miopen_ck() {
         GPU_ARCH_LIST="gfx942"
         ack_and_skip_static
     fi
-
+    # if ENABLE_GPU_ARCH is set in env by Job parameter ENABLE_GPU_ARCH, then set GPU_ARCH_LIST to that value
+    # This needs to be removed when CK aligns with other component and uses -DAMDGPU_TARGET or _DGPO_TARGET
+    # then we can use set_gpu_arch from compure_helper.sh and get rid of all if clauses
+    if [ -n "$ENABLE_GPU_ARCH" ]; then
+        GPU_ARCH_LIST="$ENABLE_GPU_ARCH"
+    fi
+    # Latest CK requiring Python 3.8 as the minimum.
+    # Point CMake to that explicit location and adjust LD_LIBRARY_PATH.
     PYTHON_VERSION_WORKAROUND=''
     echo "DISTRO_ID: ${DISTRO_ID}"
-    if [ "$DISTRO_ID" = "rhel-8.8" ] || [ "$DISTRO_ID" = "sles-15.5" ] ; then
+    if [ "$DISTRO_ID" = "rhel-8.8" ] || [ "$DISTRO_NAME" == "sles" ] || [ "$DISTRO_ID" = "debian-10" ]; then
         EXTRA_PYTHON_PATH=/opt/Python-3.8.13
         PYTHON_VERSION_WORKAROUND="-DCK_USE_ALTERNATIVE_PYTHON=${EXTRA_PYTHON_PATH}/bin/python3.8"
         # For the python interpreter we need to export LD_LIBRARY_PATH.
@@ -42,19 +53,20 @@ build_miopen_ck() {
         "${rocm_math_common_cmake_params[@]}" \
         ${PYTHON_VERSION_WORKAROUND} \
         -DCPACK_GENERATOR="${PKGTYPE^^}" \
-        -DCMAKE_CXX_COMPILER="${ROCM_PATH}/llvm/bin/clang++" \
-        -DCMAKE_C_COMPILER="${ROCM_PATH}/llvm/bin/clang" \
+        -DCMAKE_CXX_COMPILER=$(set_build_variables __CLANG++__) \
+        -DCMAKE_C_COMPILER=$(set_build_variables __CLANG__) \
         ${LAUNCHER_FLAGS} \
         -DGPU_ARCHS="${GPU_ARCH_LIST}" \
-        -DCMAKE_BUILD_TYPE=Release \
-        -DCMAKE_CXX_FLAGS=" -O3 " \
         "$COMPONENT_SRC"
 
     cmake --build . -- -j${PROC} package
     cmake --build "$BUILD_DIR" -- install
-    mkdir -p $PACKAGE_DIR && cp ./*.${PKGTYPE} $PACKAGE_DIR
+
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 }
 
+# Use the function to unset the LDFLAGS and CXXFLAGS
+# specifically set for ASAN
 unset_asan_env_vars() {
     ASAN_CMAKE_PARAMS="false"
     export ADDRESS_SANITIZER="OFF"
@@ -68,85 +80,6 @@ set_address_sanitizer_off() {
     export LDFLAGS=""
 }
 
-build_miopen_ckProf() {
-    ENABLE_ADDRESS_SANITIZER=false
-    echo "Start Building Composable Kernel Profiler"
-    if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
-       set_asan_env_vars
-       set_address_sanitizer_on
-    else
-       unset_asan_env_vars
-       set_address_sanitizer_off
-    fi
-
-    cd $COMPONENT_SRC
-    cd "$BUILD_DIR"
-    rm -rf *
-
-    architectures='gfx10 gfx11 gfx90 gfx94'
-    if [ -n "$GPU_ARCHS" ]; then
-        architectures=$(echo ${GPU_ARCHS} | awk -F';' '{for(i=1;i<=NF;i++) a[substr($i,1,5)]} END{for(i in a) printf i" "}')
-    fi
-
-    for arch in ${architectures}
-        do
-            if [ "${ASAN_CMAKE_PARAMS}" == "true" ] ; then
-                cmake -DBUILD_DEV=OFF \
-                    -DCMAKE_PREFIX_PATH="${ROCM_PATH%-*}/lib/cmake;${ROCM_PATH%-*}/$ASAN_LIBDIR;${ROCM_PATH%-*}/llvm;${ROCM_PATH%-*}" \
-                    -DCMAKE_BUILD_TYPE=${BUILD_TYPE:-'RelWithDebInfo'} \
-                    -DCMAKE_SHARED_LINKER_FLAGS_INIT="-Wl,--enable-new-dtags,--rpath,$ROCM_ASAN_LIB_RPATH" \
-                    -DCMAKE_EXE_LINKER_FLAGS_INIT="-Wl,--enable-new-dtags,--rpath,$ROCM_ASAN_EXE_RPATH" \
-                    -DCMAKE_VERBOSE_MAKEFILE=1 \
-                    -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE \
-                    -DCMAKE_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DCMAKE_PACKAGING_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF \
-                    -DROCM_SYMLINK_LIBS=OFF \
-                    -DCPACK_PACKAGING_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DROCM_DISABLE_LDCONFIG=ON \
-                    -DROCM_PATH="${ROCM_PATH}" \
-                    -DCPACK_GENERATOR="${PKGTYPE^^}" \
-                    -DCMAKE_CXX_COMPILER="${ROCM_PATH}/llvm/bin/clang++" \
-                    -DCMAKE_C_COMPILER="${ROCM_PATH}/llvm/bin/clang" \
-                    ${LAUNCHER_FLAGS} \
-                    -DPROFILER_ONLY=ON \
-                    -DENABLE_ASAN_PACKAGING=true \
-                    -DGPU_ARCH="${arch}" \
-                    "$COMPONENT_SRC"
-            else
-                cmake -DBUILD_DEV=OFF \
-                    -DCMAKE_PREFIX_PATH="${ROCM_PATH%-*}" \
-                    -DCMAKE_BUILD_TYPE=Release \
-                    -DCMAKE_SHARED_LINKER_FLAGS_INIT='-Wl,--enable-new-dtags,--rpath,$ORIGIN' \
-                    -DCMAKE_EXE_LINKER_FLAGS_INIT='-Wl,--enable-new-dtags,--rpath,$ORIGIN/../lib' \
-                    -DCMAKE_VERBOSE_MAKEFILE=1 \
-                    -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE \
-                    -DCMAKE_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DCMAKE_PACKAGING_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF \
-                    -DROCM_SYMLINK_LIBS=OFF \
-                    -DCPACK_PACKAGING_INSTALL_PREFIX="${ROCM_PATH}" \
-                    -DROCM_DISABLE_LDCONFIG=ON \
-                    -DROCM_PATH="${ROCM_PATH}" \
-                    -DCPACK_GENERATOR="${PKGTYPE^^}" \
-                    -DCMAKE_CXX_COMPILER="${ROCM_PATH}/llvm/bin/clang++" \
-                    -DCMAKE_C_COMPILER="${ROCM_PATH}/llvm/bin/clang" \
-                    ${LAUNCHER_FLAGS} \
-                    -DPROFILER_ONLY=ON \
-                    -DGPU_ARCH="${arch}" \
-                    "$COMPONENT_SRC"
-            fi
-
-            cmake --build . -- -j${PROC} package
-            cp ./*ckprofiler*.${PKGTYPE} $PACKAGE_DIR
-            rm -rf *
-        done
-    rm -rf _CPack_Packages/ && find -name '*.o' -delete
-
-    echo "Finished building Composable Kernel"
-    show_build_cache_stats
-}
-
 clean_miopen_ck() {
     echo "Cleaning MIOpen-CK build directory: ${BUILD_DIR} ${PACKAGE_DIR}"
     rm -rf "$BUILD_DIR" "$PACKAGE_DIR"
@@ -156,7 +89,7 @@ clean_miopen_ck() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_miopen_ck ;;
+    build) build_miopen_ck; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_miopen_ck ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_dbgapi.sh

@@ -11,6 +11,8 @@ printUsage() {
     echo "  -r,  --release            Make a release build instead of a debug build"
     echo "       --enable-assertions  Enable assertions"
     echo "  -a,  --address_sanitizer  Enable address sanitizer"
+    echo "  -w,  --wheel              Creates python wheel package of dbgapi. 
+                                      It needs to be used along with -r option"
     echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of
             type referred to by pkg_type"
     echo "  -h,  --help               Prints this help"
@@ -25,6 +27,7 @@ printUsage() {
     return 0
 }
 
+## Build environment variables
 API_NAME=rocm-dbgapi
 AMD_DBGAPI_NAME=amd-dbgapi
 MAKEINSTALL_MANIFEST=makeinstall_manifest.txt
@@ -36,21 +39,24 @@ PACKAGE_ROOT=$(getPackageRoot)
 PACKAGE_LIB=$(getLibPath)
 PACKAGE_INCLUDE=$(getIncludePath)
 BUILD_DIR=$(getBuildPath $API_NAME)
-PACKAGE_DEB=$(getPackageRoot)/deb/$API_NAME
-PACKAGE_RPM=$(getPackageRoot)/rpm/$API_NAME
+PACKAGE_DEB=$(getPackageRoot)/deb/$PROJ_NAME
+PACKAGE_RPM=$(getPackageRoot)/rpm/$PROJ_NAME
+#PACKAGE_PREFIX=$ROCM_INSTALL_PATH
 BUILD_TYPE=Debug
-MAKE_OPTS=($DASH_JAY -C "$BUILD_DIR")
+MAKE_OPTS=($DASH_JAY -C "$BUILD_DIR") # Note that DASH_JAY might have a space after the -j
 SHARED_LIBS="ON"
 CLEAN_OR_OUT=0;
 MAKETARGET="deb"
 PKGTYPE="deb"
 DODOCSBUILD=true
 
-VALID_STR=$(getopt -o hcraso:p:M --long help,clean,release,enable-assertions,static,address_sanitizer,outdir:,package:skip_man_pages -- "$@")
+#parse the arguments
+VALID_STR=$(getopt -o hcraswo:p:M --long help,clean,release,enable-assertions,static,wheel,address_sanitizer,outdir:,package:skip_man_pages -- "$@")
 eval set -- "$VALID_STR"
 
 while true ;
 do
+    #echo "parocessing $1"
     case "$1" in
         (-h | --help)
                 printUsage ; exit 0;;
@@ -66,12 +72,14 @@ do
                 set_address_sanitizer_on ;;
         (-s | --static)
                 ack_and_skip_static ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ;;
         (-o | --outdir)
                 TARGET="outdir"; PKGTYPE=$2 ; ((CLEAN_OR_OUT|=2)) ; shift 1 ;;
         (-M | --skip_man_pages) DODOCSBUILD=false;;
         (-p | --package)
                 MAKETARGET="$2" ; shift 1;;
-        --)     shift; break;;
+        --)     shift; break;; # end delimiter
         (*)
                 echo " This should never come but just incase : UNEXPECTED ERROR Parm : [$1] ">&2 ; exit 20;;
     esac
@@ -104,7 +112,7 @@ build() {
     then
            echo " No $ROCM_DBGAPI_ROOT/CMakeLists.txt file, skipping rocm-dbgapi" >&2
            echo " No $ROCM_DBGAPI_ROOT/CMakeLists.txt file, skipping rocm-dbgapi"
-       exit 0
+       exit 0 # THis is not an error
     fi
     echo "Building $PROJ_NAME"
 
@@ -121,11 +129,14 @@ build() {
     cmake --build "$BUILD_DIR" -- "${MAKE_OPTS[@]}"
     "$DODOCSBUILD" && cmake --build "$BUILD_DIR" -- "${MAKE_OPTS[@]}" doc
     cmake --build "$BUILD_DIR" -- "${MAKE_OPTS[@]}" install
-
+    #install_manifest.txt is created by make install and make package with same name unless
+    #component packaging is enabled. To avoid overwriting by make package,move the manifest file
+    #to a different name and can be used for build clean up
     mv "$BUILD_DIR/install_manifest.txt" "$BUILD_DIR/$MAKEINSTALL_MANIFEST"
     cmake --build "$BUILD_DIR" -- "${MAKE_OPTS[@]}" package
 
     mkdir -p "$PACKAGE_LIB"
+    # handle the library being in more than one place to avoid breaking the build.
     (
         shopt -s nullglob
         cp -R "$BUILD_DIR/lib/${LIB_NAME}"* "$BUILD_DIR/${LIB_NAME}"* "$PACKAGE_LIB"
@@ -151,7 +162,7 @@ verifyEnvSetup
 
 case $TARGET in
     (clean) clean ;;
-    (build) build ;;
+    (build) build; build_wheel "$BUILD_DIR" "$PROJ_NAME";;
     (outdir) print_output_directory ;;
     (*) die "Invalid target $TARGET" ;;
 esac

tools/rocm-build/build_devicelibs.sh

@@ -10,7 +10,9 @@ printUsage() {
     echo "  -c,  --clean              Clean output and delete all intermediate work"
     echo "  -r,  --release            Build a release version of the package"
     echo "  -a,  --address_sanitizer  Enable address sanitizer"
-    echo "  -s,  --static             Supports static CI by accepting this param & not bailing out. No effect of the param though"
+    echo "  -s,  --static             Build static lib (.a).  build instead of dynamic/shared(.so) "
+    echo "  -w,  --wheel              Creates python wheel package of devicelibs.
+                                      It needs to be used along with -r option"
     echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of
     type referred to by pkg_type"
     echo "  -h,  --help             Prints this help"
@@ -19,10 +21,11 @@ printUsage() {
 
     return 0
 }
+PROJ_NAME="devicelibs"
 PACKAGE_ROOT="$(getPackageRoot)"
 PACKAGE_BIN="$(getBinPath)"
 PACKAGE_LIB="$(getLibPath)"
-BUILD_PATH="$(getBuildPath devicelibs)"
+BUILD_PATH="$(getBuildPath $PROJ_NAME)"
 INSTALL_PATH="$(getPackageRoot)"
 LIGHTNING_BUILD_PATH="$(getBuildPath lightning)"
 DEB_PATH="$(getDebPath devicelibs)"
@@ -36,11 +39,13 @@ CLEAN_OR_OUT=0;
 PKGTYPE="deb"
 MAKETARGET="deb"
 
-VALID_STR=`getopt -o hcraso: --long help,clean,release,static,address_sanitizer,outdir: -- "$@"`
+#parse the arguments
+VALID_STR=`getopt -o hcraswo: --long help,clean,release,static,wheel,address_sanitizer,outdir: -- "$@"`
 eval set -- "$VALID_STR"
 
 while true ;
 do
+    #echo "parocessing $1"
     case "$1" in
         (-h | --help)
                 printUsage ; exit 0;;
@@ -53,6 +58,8 @@ do
                 ack_and_ignore_asan ; shift ;;
         (-s | --static)
                 SHARED_LIBS="OFF" ; shift ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ; shift ;;
         (-o | --outdir)
                 TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 2 ;;
         --)     shift; break;; # end delimiter
@@ -71,7 +78,9 @@ fi
 
 
 clean_devicelibs() {
+    # Delete cmake output directory
     rm -rf "$BUILD_PATH"
+    # Delete the *.bc files from the PACKAGE_LIB directory
     rm -f $PACKAGE_LIB/hc*.bc
     rm -f $PACKAGE_LIB/irif*.bc
     rm -f $PACKAGE_LIB/ockl*.bc
@@ -80,6 +89,7 @@ clean_devicelibs() {
     rm -f $PACKAGE_LIB/opencl*.bc
     rm -f $PACKAGE_LIB/openmp*.bc
     rm -rf $PACKAGE_ROOT/amdgcn
+    # Delete any packages generated
     rm -rf "$DEB_PATH"
     rm -rf "$RPM_PATH"
 }
@@ -96,6 +106,7 @@ build_devicelibs() {
     if [ ! -e Makefile ]; then
         cmake $(rocm_cmake_params) \
               $(rocm_common_cmake_params) \
+              ${GEN_NINJA} \
               -DBUILD_SHARED_LIBS=$SHARED_LIBS \
               -DROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_NEW="$bitcodeInstallLoc/amdgcn" \
               -DROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_OLD="amdgcn" \
@@ -133,7 +144,7 @@ print_output_directory() {
 }
 case $TARGET in
     (clean) clean_devicelibs ;;
-    (build) build_devicelibs; package_devicelibs ;;
+    (build) build_devicelibs; package_devicelibs; build_wheel "$BUILD_PATH" "$PROJ_NAME" ;;
     (outdir) print_output_directory ;;
     (*) die "Invalid target $TARGET" ;;
 esac

tools/rocm-build/build_half.sh

@@ -1,22 +1,31 @@
 #!/bin/bash
 
 set -ex
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src half
 
 build_half() {
     echo "Start build"
 
+    if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
+        ack_and_skip_static
+    fi
+
     if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
          set_asan_env_vars
          set_address_sanitizer_on
+         # ASAN packaging is not required for HALF, since its header only package
+         # Setting the asan_cmake_params to false will disable ASAN packaging
          ASAN_CMAKE_PARAMS="false"
     fi
     mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
 
     cmake \
         -DCMAKE_INSTALL_PREFIX="$ROCM_PATH" \
+        -DCPACK_PACKAGING_INSTALL_PREFIX=${ROCM_PATH}\
+        -DCPACK_SET_DESTDIR="OFF" \
+        -DCPACK_RPM_PACKAGE_RELOCATABLE="ON" \
         -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF \
         "$COMPONENT_SRC"
 
@@ -25,7 +34,7 @@ build_half() {
     cmake --build "$BUILD_DIR" -- install
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -39,7 +48,7 @@ clean_half() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_half ;;
+    build) build_half; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_half ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hip_on_rocclr.sh

@@ -10,6 +10,8 @@ printUsage() {
     echo "  -r,  --release              Make a release build instead of a debug build"
     echo "  -a,  --address_sanitizer    Enable address sanitizer"
     echo "  -s,  --static               Build static lib (.a).  build instead of dynamic/shared(.so) "
+    echo "  -w,  --wheel                Creates python wheel package of hip-on-rocclr.
+                                        It needs to be used along with -r option"
     echo "  -o,  --outdir <pkg_type>    Print path of output directory containing packages of type referred to by pkg_type"
     echo "  -t,  --offload-arch=<arch>  Specify arch for catch tests ex: --offload-arch=gfx1030 --offload-arch=gfx1100"
     echo "  -p,  --package <type>       Specify packaging format"
@@ -25,7 +27,6 @@ printUsage() {
 source "$(dirname "${BASH_SOURCE}")/compute_utils.sh"
 MAKEOPTS="$DASH_JAY"
 PROJ_NAME="hip-on-rocclr"
-
 BUILD_PATH="$(getBuildPath $PROJ_NAME)"
 
 TARGET="build"
@@ -53,9 +54,10 @@ MAKETARGET="deb"
 PKGTYPE="deb"
 OFFLOAD_ARCH=()
 
-DEFAULT_OFFLOAD_ARCH=(gfx900 gfx906 gfx908 gfx90a gfx940 gfx941 gfx942 gfx1030 gfx1031 gfx1033 gfx1034 gfx1035 gfx1100 gfx1101 gfx1102 gfx1200 gfx1201)
+DEFAULT_OFFLOAD_ARCH=(gfx900 gfx906 gfx908 gfx90a gfx940 gfx941 gfx942 gfx1030 gfx1031 gfx1033 gfx1034 gfx1035 gfx1100 gfx1101 gfx1102 gfx1103 gfx1150 gfx1151 gfx1200 gfx1201)
 
-VALID_STR=`getopt -o hcrast:o: --long help,clean,release,address_sanitizer,static,offload-arch=:,outdir: -- "$@"`
+#parse the arguments
+VALID_STR=`getopt -o hcraswt:o: --long help,clean,release,address_sanitizer,static,wheel,offload-arch=:,outdir: -- "$@"`
 eval set -- "$VALID_STR"
 
 while true ;
@@ -72,11 +74,13 @@ do
                 set_address_sanitizer_on ; shift ;;
         (-s | --static)
                 SHARED_LIBS="OFF" ; shift ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ; shift ;;
         (-t | --offload-arch=)
                 OFFLOAD_ARCH+=( "$2" ); ((CLEAN_OR_OUT|=2)); shift 2 ;;
         (-o | --outdir)
                 TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 2 ;;
-        --)     shift; break;;
+        --)     shift; break;; # end delimiter
 
         (*)
                 echo " This should never come but just incase : UNEXPECTED ERROR Parm : [$1] ">&2 ; exit 20;;
@@ -98,7 +102,10 @@ if [ $RET_CONFLICT -ge 30 ]; then
    exit $RET_CONFLICT
 fi
 
+
+
 clean_hip_on_rocclr() {
+    # Delete cmake output directory
     rm -rf "$BUILD_PATH"
     rm -rf "$PACKAGE_DEB"
     rm -rf "$PACKAGE_RPM"
@@ -106,17 +113,23 @@ clean_hip_on_rocclr() {
 }
 
 build_hip_on_rocclr() {
+    # TODO This if condition is a temporary workaround so that mainline builds dont error out
+    # until build migrated from hipamd to clr repo
     if [  -e "$CLR_ROOT/CMakeLists.txt" ]; then
+        # We are in a branch that has migrated to clr repo
         _HIP_CMAKELIST_DIR="$CLR_ROOT"
         _HIP_CMAKELIST_OPT="-DCLR_BUILD_HIP=ON -DCLR_BUILD_OCL=OFF"
         if [ -e "$HIPOTHER_ROOT/hipnv" ]; then
+            # We are in a branch that has hipnv headers migrated
             _HIP_CMAKELIST_OPT="$_HIP_CMAKELIST_OPT -DHIPNV_DIR=$HIPOTHER_ROOT/hipnv"
         fi
     elif [ ! -e "$HIPAMD_ROOT/CMakeLists.txt" ]; then
+        # We seem to have hit a branch in which both the old and the new repo don't exist
         echo "No $HIPAMD_ROOT/CMakeLists.txt file, skipping hip on rocclr" >&2
         echo "No $HIPAMD_ROOT/CMakeLists.txt file, skipping hip on rocclr"
-        exit 0
+        exit 0 # This is not an error
     else
+        # We are in a branch that has not yet migrated to clr repo yet
         _HIP_CMAKELIST_DIR="$HIPAMD_ROOT"
         _HIP_CMAKELIST_OPT=""
     fi
@@ -125,6 +138,7 @@ build_hip_on_rocclr() {
     mkdir -p "$BUILD_PATH"
     pushd "$BUILD_PATH"
 
+    # FIXME: Remove -DROCclr_DIR/LIBROCclr_STATIC_DIR
     if [ ! -e Makefile ]; then
         echo "Building HIP-On-ROCclr CMake environment"
         print_lib_type $SHARED_LIBS
@@ -165,8 +179,13 @@ build_catch_tests() {
    rm -rf "$CATCH_BUILD_DIR"
    mkdir -p "$CATCH_BUILD_DIR"
    pushd "$CATCH_BUILD_DIR"
+   # use the newly built hip as HIP for catch
    export HIP_PATH="$ROCM_INSTALL_PATH"
    export ROCM_PATH="$ROCM_INSTALL_PATH"
+   # Note: The rocm_common_cmake_params will provide CMAKE_EXE_LINKER_FLAGS_INIT
+   # for binaries located in /opt/rocm/bin
+   # hip-catch binaries are located in each test folder under /opt/rocm/share/hip/catch_test/
+   # Append the EXE LINKER flags with ROCm library path
    cmake \
          -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
          -DBUILD_SHARED_LIBS=$SHARED_LIBS \
@@ -177,6 +196,7 @@ build_catch_tests() {
          $(rocm_common_cmake_params) \
          -DCPACK_RPM_DEBUGINFO_PACKAGE=FALSE \
          -DCPACK_DEBIAN_DEBUGINFO_PACKAGE=FALSE \
+         -DCMAKE_EXE_LINKER_FLAGS_INIT=-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$ROCM_EXE_RPATH:$ROCM_INSTALL_PATH/lib:/opt/rocm/lib \
          -DCPACK_INSTALL_PREFIX="$ROCM_INSTALL_PATH" \
          "$CATCH_SRC"
 
@@ -190,6 +210,7 @@ build_catch_tests() {
 }
 
 package_samples() {
+   # TODO: To be removed once the issue related to ASAN builds are fixed
    if [ "$ASAN_CMAKE_PARAMS" == "true" ] ; then
       echo "Disable the packaging of HIP samples" >&2
       return
@@ -198,13 +219,17 @@ package_samples() {
    if [ ! -e "$SAMPLES_SRC/CMakeLists.txt" ]; then
       echo "HIP samples source not found at: $SAMPLES_SRC" >&2
       echo "Using samples package from hip project: $BUILD_PATH" >&2
+      # TODO: change to return failure after hip-tests samples change is available in mainline
       return
    fi
-
+   # package samples
    rm -rf "$SAMPLES_BUILD_DIR"
    mkdir -p "$SAMPLES_BUILD_DIR"
    pushd "$SAMPLES_BUILD_DIR"
+   # The cmake path is different for asan and non-asan builds.
+   # Fetch after getting build type. Default will be non-asan build
    local CMAKE_PATH="$(getCmakePath)"
+   # use the newly built hip as HIP for samples
    export HIP_PATH="$ROCM_INSTALL_PATH"
    export ROCM_PATH="$ROCM_INSTALL_PATH"
    cmake \
@@ -291,10 +316,24 @@ print_output_directory() {
 }
 
 case $TARGET in
-    (clean) clean_hip_on_rocclr; clean_hip_tests ;;
-    (build) build_hip_on_rocclr; build_catch_tests; package_hip_on_rocclr; package_samples; copy_hip_tests;;
-   (outdir) print_output_directory ;;
-        (*) die "Invalid target $TARGET" ;;
+    (clean)
+        clean_hip_on_rocclr
+        clean_hip_tests
+        ;;
+    (build)
+        build_hip_on_rocclr
+        build_catch_tests
+        package_hip_on_rocclr
+        package_samples
+        build_wheel "$BUILD_PATH" "$PROJ_NAME"
+        copy_hip_tests
+        ;;
+    (outdir)
+        print_output_directory
+        ;;
+    (*)
+        die "Invalid target $TARGET"
+        ;;
 esac
 
 echo "Operation complete"

tools/rocm-build/build_hipblas-common.sh

@@ -2,16 +2,19 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipBLAS-common
 
 build_hipblas-common() {
     echo "Start build"
 
+    CXX=$(set_build_variables __C_++__)
     cd $COMPONENT_SRC
     mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
 
+    echo "C compiler: $CC"
+    echo "CXX compiler: $CXX"
     init_rocm_common_cmake_params
     cmake \
         "${rocm_math_common_cmake_params[@]}" \
@@ -20,8 +23,7 @@ build_hipblas-common() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
-
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
     show_build_cache_stats
 }
 
@@ -34,7 +36,7 @@ clean_hipblas-common() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipblas-common ;;
+    build) build_hipblas-common; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipblas-common ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipblas.sh

@@ -2,28 +2,32 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipBLAS
 
 build_hipblas() {
     echo "Start build"
 
-    CXX="g++"
+    CXX=$(set_build_variables __G_++__)
     CXX_FLAG=
     if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
-        CXX="amdclang++"
-        CXX_FLAG="-DCMAKE_CXX_COMPILER=${ROCM_PATH}/llvm/bin/clang++"
+        CXX=$(set_build_variables __AMD_CLANG_++__)
+        CXX_FLAG=$(set_build_variables __CMAKE_CXX_PARAMS__)
     fi
 
     CLIENTS_SAMPLES="ON"
     if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
        set_asan_env_vars
        set_address_sanitizer_on
+       # fixme: remove CLIENTS_SAMPLES=OFF once SWDEV-417076 is fixed
        CLIENTS_SAMPLES="OFF"
     fi
 
     SHARED_LIBS="ON"
+    if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
+        SHARED_LIBS="OFF"
+    fi
 
     echo "C compiler: $CC"
     echo "CXX compiler: $CXX"
@@ -39,10 +43,10 @@ build_hipblas() {
     init_rocm_common_cmake_params
     cmake \
         ${LAUNCHER_FLAGS} \
-        "${rocm_math_common_cmake_params[@]}" \
+	    "${rocm_math_common_cmake_params[@]}" \
         -DUSE_CUDA=OFF \
         -DBUILD_SHARED_LIBS=$SHARED_LIBS \
-        -DBUILD_CLIENTS_TESTS=ON \
+	    -DBUILD_CLIENTS_TESTS=ON \
         -DBUILD_CLIENTS_BENCHMARKS=ON \
         -DBUILD_CLIENTS_SAMPLES="${CLIENTS_SAMPLES}" \
         -DBUILD_ADDRESS_SANITIZER="${ADDRESS_SANITIZER}" \
@@ -54,7 +58,7 @@ build_hipblas() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -68,7 +72,7 @@ clean_hipblas() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipblas ;;
+    build) build_hipblas; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipblas ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipblaslt.sh

@@ -1,10 +1,20 @@
 #!/bin/bash
 
 set -ex
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipBLASLt
 
+#not enabling the debug files by defalut
+disable_debug_package_generation
+# if ENABLE_GPU_ARCH is set in env by Job parameter ENABLE_GPU_ARCH, then set GFX_ARCH to that value
+if [ -n "$ENABLE_GPU_ARCH" ]; then
+    set_gpu_arch "$ENABLE_GPU_ARCH"
+else
+    # gfx90a:xnack+;gfx90a:xnack-;gfx940;gfx941;gfx942
+    set_gpu_arch "all"
+fi
+
 build_hipblaslt() {
     echo "Start build"
 
@@ -20,31 +30,31 @@ build_hipblaslt() {
     cd $COMPONENT_SRC
     mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
 
-    if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
-       rebuild_lapack
-    fi
-
-    if [ -n "$GPU_ARCHS" ]; then
-        GPU_TARGETS="$GPU_ARCHS"
-    else
-        # gfx90a:xnack+;gfx90a:xnack-;gfx940;gfx941;gfx942
-        GPU_TARGETS=all
+    # hipblaslt requiring Python 3.12.
+    # Point CMake to that explicit location and adjust LD_LIBRARY_PATH.
+    if [ "$DISTRO_ID" = "rhel-8.8" ] || [ "$DISTRO_NAME" == "sles" ] || \
+        [ "$DISTRO_ID" = "rhel-9.1" ] || [ "$DISTRO_ID" = "almalinux-8.10" ] || \
+        [ "$DISTRO_ID" = "debian-10" ]; then
+        EXTRA_PYTHON_PATH=/opt/Python-3.12.7
+        EXTRA_CMAKE_OPTIONS="-DPython_ROOT=/opt/Python-3.12.7"
+        export LD_LIBRARY_PATH=${EXTRA_PYTHON_PATH}/lib
     fi
 
     init_rocm_common_cmake_params
-    CXX=$(set_build_variables CXX)\
+    CXX=$(set_build_variables __CXX__)\
     cmake \
-        -DAMDGPU_TARGETS=${GPU_TARGETS} \
         ${LAUNCHER_FLAGS} \
-        "${rocm_math_common_cmake_params[@]}" \
+         "${rocm_math_common_cmake_params[@]}" \
         -DTensile_LOGIC= \
         -DTensile_CODE_OBJECT_VERSION=default \
-        -DTensile_CPU_THREADS= \
+        -DTensile_CPU_THREADS=$((PROC / 4)) \
         -DTensile_LIBRARY_FORMAT=msgpack \
         -DBUILD_CLIENTS_SAMPLES=ON \
         -DBUILD_CLIENTS_TESTS=ON \
+        -DLINK_BLIS=ON \
         -DBUILD_CLIENTS_BENCHMARKS=ON \
         -DBUILD_ADDRESS_SANITIZER="${ADDRESS_SANITIZER}" \
+        ${EXTRA_CMAKE_OPTIONS} \
         "$COMPONENT_SRC"
 
     cmake --build "$BUILD_DIR" -- -j${PROC}
@@ -52,8 +62,7 @@ build_hipblaslt() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
-
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
     show_build_cache_stats
 }
 
@@ -66,7 +75,7 @@ clean_hipblaslt() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipblaslt ;;
+    build) build_hipblaslt; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipblaslt ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipcc.sh

@@ -9,6 +9,8 @@ printUsage() {
     echo "Options:"
     echo "  -a,  --address_sanitizer  Enable address sanitizer"
     echo "  -c,  --clean              Clean output and delete all intermediate work"
+    echo "  -w,  --wheel              Creates python wheel package of hipcc. 
+                                      It needs to be used along with -r option"
     echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of
                             type referred to by pkg_type"
     echo "  -r,  --release            Makes a release build"
@@ -21,6 +23,7 @@ printUsage() {
 }
 
 
+## Build environment variables
 API_NAME=hipcc
 PROJ_NAME=$API_NAME
 
@@ -29,10 +32,11 @@ MAKEOPTS="$DASH_JAY"
 BUILD_TYPE="Debug"
 SHARED_LIBS="ON"
 BUILD_DIR=$(getBuildPath $API_NAME)
-PACKAGE_DEB=$(getPackageRoot)/deb/$API_NAME
-PACKAGE_RPM=$(getPackageRoot)/rpm/$API_NAME
+PACKAGE_DEB=$(getPackageRoot)/deb/$PROJ_NAME
+PACKAGE_RPM=$(getPackageRoot)/rpm/$PROJ_NAME
 PACKAGE_SRC="$(getSrcPath)"
 
+#parse the arguments
 VALID_STR=`getopt -o hcraswo:p: --long help,clean,release,address_sanitizer,static,outdir,wheel:,package: -- "$@"`
 eval set -- "$VALID_STR"
 
@@ -45,13 +49,15 @@ do
             TARGET="clean" ;;
         (-o  | --outdir)
             TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 1 ;;
+        (-w  | --wheel)
+            WHEEL_PACKAGE=true ;;
         (-r  | --release)
             BUILD_TYPE="RelWithDebInfo" ;;
         (-s | --static)
             SHARED_LIBS="OFF" ;;
         (-h  | --help)
             printUsage ; exit 0 ;;
-        --)     shift; break;;
+        --)     shift; break;; # end delimiter
         (*)
             echo "Invalid option [$1]" >&2; printUsage; exit 1 ;;
     esac
@@ -91,7 +97,7 @@ build() {
         $(rocm_cmake_params) \
         $(rocm_common_cmake_params) \
         -DHIPCC_BACKWARD_COMPATIBILITY=OFF \
-        -DCMAKE_INSTALL_PREFIX="$OUT_DIR" \
+        -DCMAKE_INSTALL_PREFIX="$ROCM_PATH" \
         $HIPCC_ROOT
     popd
 
@@ -118,10 +124,20 @@ print_output_directory() {
 }
 
 case $TARGET in
-    (clean) clean ;;
-    (build) build ; copy_hipcc_sources ;;
-    (outdir) print_output_directory ;;
-    (*) die "Invalid target $TARGET" ;;
+    (clean)
+        clean
+        ;;
+    (build)
+        build
+        build_wheel "$BUILD_DIR" "$PROJ_NAME"
+        copy_hipcc_sources
+        ;;
+    (outdir)
+        print_output_directory
+        ;;
+    (*)
+        die "Invalid target $TARGET"
+        ;;
 esac
 
 echo "Operation complete"

tools/rocm-build/build_hipcub.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipCUB
 
@@ -17,26 +17,21 @@ build_hipcub() {
     if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
          set_asan_env_vars
          set_address_sanitizer_on
+         # ASAN packaging is not required for HIPCUB, since its header only package
+         # Setting the asan_cmake_params to false will disable ASAN packaging
          ASAN_CMAKE_PARAMS="false"
     fi
 
     mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
     init_rocm_common_cmake_params
 
-    if [ -n "$GPU_ARCHS" ]; then
-        GPU_TARGETS="$GPU_ARCHS"
-    else
-        GPU_TARGETS="gfx908:xnack-;gfx90a:xnack-;gfx90a:xnack+;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201"
-    fi
-
-    CXX=$(set_build_variables CXX)\
+    CXX=$(set_build_variables __CXX__)\
     cmake \
         ${LAUNCHER_FLAGS} \
-        "${rocm_math_common_cmake_params[@]}" \
+	    "${rocm_math_common_cmake_params[@]}" \
         -DCMAKE_MODULE_PATH="${ROCM_PATH}/lib/cmake/hip;${ROCM_PATH}/hip/cmake" \
         -Drocprim_DIR="${ROCM_PATH}/rocprim"  \
         -DBUILD_TEST=ON \
-        -DAMDGPU_TARGETS=${GPU_TARGETS} \
         "$COMPONENT_SRC"
 
     cmake --build "$BUILD_DIR" -- -j${PROC}
@@ -44,7 +39,7 @@ build_hipcub() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -58,7 +53,7 @@ clean_hipcub() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipcub ;;
+    build) build_hipcub; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipcub ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipfft.sh

@@ -2,13 +2,17 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipFFT
 
 build_hipfft() {
     echo "Start build"
 
+    if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
+        ack_and_skip_static
+    fi
+
     if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
        set_asan_env_vars
        set_address_sanitizer_on
@@ -18,17 +22,10 @@ build_hipfft() {
     mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
     init_rocm_common_cmake_params
 
-    if [ -n "$GPU_ARCHS" ]; then
-        GPU_TARGETS="$GPU_ARCHS"
-    else
-        GPU_TARGETS="gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201"
-    fi
-
     cmake \
-        -DCMAKE_CXX_COMPILER=$(set_build_variables CXX) \
+        "$(set_build_variables __CMAKE_CXX_PARAMS__)" \
         ${LAUNCHER_FLAGS} \
-        "${rocm_math_common_cmake_params[@]}" \
-        -DAMDGPU_TARGETS=${GPU_TARGETS} \
+	    "${rocm_math_common_cmake_params[@]}" \
         -DCMAKE_MODULE_PATH="${ROCM_PATH}/lib/cmake/hip" \
         -DCMAKE_SKIP_BUILD_RPATH=TRUE \
         -DBUILD_ADDRESS_SANITIZER="${ADDRESS_SANITIZER}" \
@@ -41,7 +38,7 @@ build_hipfft() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR  && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -55,7 +52,7 @@ clean_hipfft() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipfft ;;
+    build) build_hipfft; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipfft ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipfort.sh

@@ -1,13 +1,18 @@
 #!/bin/bash
 
 set -ex
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipfort
 
 build_hipfort() {
     echo "Start build"
 
+    if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
+       set_asan_env_vars
+       set_address_sanitizer_on
+    fi
+
     if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
         ack_and_skip_static
     fi
@@ -26,7 +31,7 @@ build_hipfort() {
         -DCMAKE_Fortran_FLAGS_DEBUG="" \
         ${LAUNCHER_FLAGS} \
         -DROCM_SYMLINK_LIBS=OFF \
-	-DCMAKE_INSTALL_PREFIX=${ROCM_PATH} \
+        -DCMAKE_INSTALL_PREFIX=${ROCM_PATH} \
         -DHIPFORT_AR="${ROCM_PATH}/${ROCM_LLVMDIR}/bin/llvm-ar" \
         -DHIPFORT_RANLIB="${ROCM_PATH}/${ROCM_LLVMDIR}/bin/llvm-ranlib" \
         "$COMPONENT_SRC"
@@ -36,7 +41,7 @@ build_hipfort() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -50,7 +55,7 @@ clean_hipfort() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipfort ;;
+    build) build_hipfort; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipfort ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipify_clang.sh

@@ -10,15 +10,21 @@ printUsage() {
     echo "  -c,  --clean              Clean output and delete all intermediate work"
     echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of
     type referred to by pkg_type"
-    echo "  -h,  --help             Prints this help"
-    echo "  -r,  --release          Make a release build"
+    echo "  -r,  --release            Make a release build"
+    echo "  -n,  --skip_hipify_tests  Skip hipify-clang testing"
     echo "  -a,  --address_sanitizer  Enable address sanitizer"
-    echo "  -s,  --static             Supports static CI by accepting this param & not bailing out. No effect of the param though"
+    echo "  -s,  --static             Component/Build does not support static builds just accepting this param & ignore. No effect of the param on this build"
+    echo "  -i,  --clang_headers      Install clang headers"
+    echo "  -w,  --wheel              Creates python wheel package of hipify-clang.
+                                      It needs to be used along with -r option"
+    echo "  -h,  --help               Prints this help"
     echo
 
     return 0
 }
 
+# Build Environmental Variables
+PROJ_NAME="hipify"
 TARGET="build"
 MAKEOPTS="$DASH_JAY"
 HIPIFY_CLANG_BUILD_DIR="$(getBuildPath $HIPIFY_ROOT)"
@@ -26,6 +32,11 @@ BUILD_TYPE="Debug"
 PACKAGE_ROOT="$(getPackageRoot)"
 HIPIFY_CLANG_HASH=""
 LIGHTNING_PATH="$ROCM_INSTALL_PATH/llvm"
+LIGHTNING_BUILD_PATH="$PACKAGE_ROOT/build/lightning"
+RUN_HIPIFY_TESTS=false
+CUDA_DEFAULT_VERSION="12.3.2"
+CUDNN_DEFAULT_VERSION="9.2.0"
+GCC_MIN_VERSION="9.2"
 ADDRESS_SANITIZER=false
 INSTALL_CLANG_HEADERS="OFF"
 DEB_PATH="$(getDebPath hipify)"
@@ -36,7 +47,8 @@ MAKETARGET="deb"
 PKGTYPE="deb"
 
 
-VALID_STR=`getopt -o hcraso: --long help,clean,release,static,address_sanitizer,outdir: -- "$@"`
+#parse the arguments
+VALID_STR=`getopt -o hcrnawsio: --long help,clean,release,skip_hipify_tests,wheel,static,address_sanitizer,clang_headers,outdir: -- "$@"`
 eval set -- "$VALID_STR"
 
 while true ;
@@ -48,15 +60,21 @@ do
                 TARGET="clean" ; ((CLEAN_OR_OUT|=1)) ; shift ;;
         (-r | --release)
                 BUILD_TYPE="RelWithDebInfo" ; shift ;;
+        (-n | --skip_hipify_tests)
+                RUN_HIPIFY_TESTS=false; shift ;;
         (-a | --address_sanitizer)
                 set_asan_env_vars
                 set_address_sanitizer_on
                 ADDRESS_SANITIZER=true ; shift ;;
         (-s | --static)
                 ack_and_skip_static ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ; shift ;;
+        (-i | --clang_headers)
+                INSTALL_CLANG_HEADERS="ON" ; shift ;;
         (-o | --outdir)
                 TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 2 ;;
-        --)     shift; break;;
+        --)     shift; break;; # end delimiter
         (*)
                 echo " This should never come but just incase : UNEXPECTED ERROR Parm : [$1] ">&2 ; exit 20;;
     esac
@@ -79,6 +97,7 @@ clean_hipify() {
 }
 
 package_hipify() {
+    # set-up dirs
     if [ "$PACKAGEEXT" = "deb" ]; then
         rm -rf "$DEB_PATH"
         mkdir -p "$DEB_PATH"
@@ -89,6 +108,7 @@ package_hipify() {
         mkdir -p "$RPM_PATH"
     fi
 
+    # make the pkg
     pushd "$HIPIFY_CLANG_BUILD_DIR"
     make $MAKEOPTS package_hipify-clang
     popd
@@ -102,7 +122,37 @@ build_hipify() {
     mkdir -p "$HIPIFY_CLANG_BUILD_DIR"
 
     pushd "$HIPIFY_CLANG_BUILD_DIR"
+
+    # Check the installed GCC version
+    INSTALLED_GCC_VERSION=$(gcc --version | head -n 1 | sed -E 's/[^0-9]*([0-9]+\.[0-9]+).*/\1/')
+    if echo "$INSTALLED_GCC_VERSION $GCC_MIN_VERSION" | awk '{exit !($1 < $2)}'; then
+        RUN_HIPIFY_TESTS=false
+        echo "Minimum required GCC version: $GCC_MIN_VERSION"
+        echo "Installed GCC version $INSTALLED_GCC_VERSION does not meet the minimum GCC version requirement."
+        echo "Skipping hipify tests"
+    fi
+
+    if $ADDRESS_SANITIZER ; then
+        echo "Skipping hipify tests becasue of Address Sanitizer"
+        RUN_HIPIFY_TESTS=false
+    fi
+
+    if $RUN_HIPIFY_TESTS ; then
+        # TODO: Add option for user defined cuda version?
+        CUDA_VERSION="${CUDA_DEFAULT_VERSION}"
+        CUDNN_VERSION="${CUDNN_DEFAULT_VERSION}"
+
+        if [[ "$DISTRO_ID" == "rhel-8"* || "$DISTRO_NAME" == "sles" || "$DISTRO_ID" == "debian-10" ]]; then
+            EXTRA_PYTHON_PATH=/opt/Python-3.8.13
+            export LD_LIBRARY_PATH=${EXTRA_PYTHON_PATH}/lib:$LD_LIBRARY_PATH
+        fi
+
+        echo "Copy FileCheck into ROCM_INSTALL_PATH"
+        cp "$LIGHTNING_BUILD_PATH/bin/FileCheck" "$LIGHTNING_PATH/bin/FileCheck"
+    fi
+
     cmake \
+        -DHIPIFY_CLANG_TESTS="$RUN_HIPIFY_TESTS" \
         -DCMAKE_BUILD_TYPE="$BUILD_TYPE" \
         $(rocm_common_cmake_params) \
         -DCMAKE_INSTALL_PREFIX="$ROCM_INSTALL_PATH" \
@@ -110,10 +160,20 @@ build_hipify() {
         -DCMAKE_PREFIX_PATH="$LIGHTNING_PATH" \
         -DADDRESS_SANITIZER="$ADDRESS_SANITIZER" \
         -DHIPIFY_INSTALL_CLANG_HEADERS="$INSTALL_CLANG_HEADERS" \
+        -DCUDA_TOOLKIT_ROOT_DIR="/usr/local/cuda-${CUDA_DEFAULT_VERSION}" \
+        -DCUDA_DNN_ROOT_DIR="/usr/local/cuDNN/${CUDNN_DEFAULT_VERSION}" \
+        -DCUDA_CUB_ROOT_DIR="/usr/local/cuda-${CUDA_DEFAULT_VERSION}" \
+        -DLLVM_EXTERNAL_LIT="${LIGHTNING_BUILD_PATH}/bin/llvm-lit" \
         $HIPIFY_ROOT
 
+    if $RUN_HIPIFY_TESTS ; then
+        echo "Running hipify tests"
+        cmake --build . -- $MAKEOPTS test-hipify
+    fi
+
     cmake --build . -- $MAKEOPTS install
     popd
+
     pushd "$HIPIFY_ROOT"
         HIPIFY_CLANG_HASH=`git describe --dirty --long --match [0-9]* --always`
     popd
@@ -132,10 +192,20 @@ print_output_directory() {
 }
 
 case $TARGET in
-    (clean) clean_hipify ;;
-    (build) build_hipify; package_hipify ;;
-    (outdir) print_output_directory ;;
-    (*) die "Invalid target $TARGET" ;;
+    (clean)
+        clean_hipify
+        ;;
+    (build)
+        build_hipify
+        package_hipify
+        build_wheel "$HIPIFY_CLANG_BUILD_DIR" "$PROJ_NAME"
+        ;;
+    (outdir)
+        print_output_directory
+        ;;
+    (*)
+        die "Invalid target $TARGET"
+        ;;
 esac
 
 echo "Operation complete"

tools/rocm-build/build_hiprand.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipRAND
 
@@ -41,17 +41,10 @@ build_hiprand() {
 
     init_rocm_common_cmake_params
 
-    if [ -n "$GPU_ARCHS" ]; then
-        GPU_TARGETS="$GPU_ARCHS"
-    else
-        GPU_TARGETS="gfx908:xnack-;gfx90a:xnack-;gfx90a:xnack+;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201"
-    fi
-
-    CXX=$(set_build_variables CXX)\
+    CXX=$(set_build_variables __CXX__)\
     cmake \
         ${LAUNCHER_FLAGS} \
 	    "${rocm_math_common_cmake_params[@]}" \
-        -DAMDGPU_TARGETS=${GPU_TARGETS} \
         -DBUILD_SHARED_LIBS=$SHARED_LIBS \
         -DBUILD_TEST=ON \
         -DBUILD_BENCHMARK=ON \
@@ -67,7 +60,7 @@ build_hiprand() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/  && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 }
 
 clean_hiprand() {
@@ -89,7 +82,7 @@ print_output_directory() {
 }
 
 case $TARGET in
-    build) build_hiprand ;;
+    build) build_hiprand; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hiprand ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipsolver.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipSOLVER
 
@@ -10,12 +10,12 @@ build_hipsolver() {
     echo "Start build"
 
     if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
-        CXX_FLAG="-DCMAKE_CXX_COMPILER=${ROCM_PATH}/llvm/bin/clang++"
+        CXX_FLAG=$(set_build_variables __CMAKE_CXX_PARAMS__)
     fi
 
     cd $COMPONENT_SRC
 
-    CXX="amdclang++"
+    CXX=$(set_build_variables __AMD_CLANG_++__)
     if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
        set_asan_env_vars
        set_address_sanitizer_on
@@ -39,7 +39,7 @@ build_hipsolver() {
     init_rocm_common_cmake_params
     cmake \
         -DUSE_CUDA=OFF \
-	-DCMAKE_CXX_COMPILER=${CXX} \
+	    -DCMAKE_CXX_COMPILER=${CXX} \
         ${LAUNCHER_FLAGS} \
         "${rocm_math_common_cmake_params[@]}" \
         -DBUILD_SHARED_LIBS=$SHARED_LIBS \
@@ -55,7 +55,7 @@ build_hipsolver() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -69,7 +69,7 @@ clean_hipsolver() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipsolver ;;
+    build) build_hipsolver; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipsolver ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipsparse.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 PATH=${ROCM_PATH}/bin:$PATH
 set_component_src hipSPARSE
@@ -10,12 +10,12 @@ set_component_src hipSPARSE
 build_hipsparse() {
     echo "Start build"
 
-    CXX="g++"
+    CXX=$(set_build_variables __G_++__)
     CXX_FLAG=
 
     if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
-        CXX="${ROCM_PATH}/llvm/bin/clang++"
-        CXX_FLAG="-DCMAKE_CXX_COMPILER=${ROCM_PATH}/llvm/bin/clang++"
+        CXX=$(set_build_variables __CXX__)
+        CXX_FLAG=$(set_build_variables __CMAKE_CXX_PARAMS__)
     fi
 
     cd $COMPONENT_SRC
@@ -54,7 +54,7 @@ build_hipsparse() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -68,7 +68,7 @@ clean_hipsparse() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hipsparse ;;
+    build) build_hipsparse; build_wheel  ;;
     outdir) print_output_directory ;;
     clean) clean_hipsparse ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hipsparselt.sh

@@ -1,10 +1,18 @@
 #!/bin/bash
 
 set -ex
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipSPARSELt
 
+disable_debug_package_generation
+# if ENABLE_GPU_ARCH is set in env by Job parameter ENABLE_GPU_ARCH, then set GFX_ARCH to that value
+if [ -n "$ENABLE_GPU_ARCH" ]; then
+    set_gpu_arch "$ENABLE_GPU_ARCH"
+else
+    # gfx90a:xnack+;gfx90a:xnack-;gfx940;gfx941;gfx942
+    set_gpu_arch "all"
+fi
 while [ "$1" != "" ];
 do
     case $1 in
@@ -18,6 +26,28 @@ do
     shift 1
 done
 
+create_blis_link()
+{
+    #find the pre-installed blis library and create the link under $BUILD_DIR/deps/blis
+    BLIS_REF_ROOT="$BUILD_DIR/deps/blis"
+    mkdir -p "$BLIS_REF_ROOT"/include
+    if [[ -e "/opt/AMD/aocl/aocl-linux-gcc-4.2.0/gcc/lib_ILP64/libblis-mt.a" ]]; then
+        ln -sf /opt/AMD/aocl/aocl-linux-gcc-4.2.0/gcc/include_ILP64 ${BLIS_REF_ROOT}/include/blis
+        ln -sf /opt/AMD/aocl/aocl-linux-gcc-4.2.0/gcc/lib_ILP64 ${BLIS_REF_ROOT}/lib
+    elif [[ -e "/opt/AMD/aocl/aocl-linux-gcc-4.1.0/gcc/lib_ILP64/libblis-mt.a" ]]; then
+        ln -sf /opt/AMD/aocl/aocl-linux-gcc-4.1.0/gcc/include_ILP64 ${BLIS_REF_ROOT}/include/blis
+        ln -sf /opt/AMD/aocl/aocl-linux-gcc-4.1.0/gcc/lib_ILP64 ${BLIS_REF_ROOT}/lib
+    elif [[ -e "/opt/AMD/aocl/aocl-linux-gcc-4.0.0/gcc/lib_ILP64/libblis-mt.a" ]]; then
+        ln -sf /opt/AMD/aocl/aocl-linux-gcc-4.0.0/gcc/include_ILP64 ${BLIS_REF_ROOT}/include/blis
+        ln -sf /opt/AMD/aocl/aocl-linux-gcc-4.0.0/gcc/lib_ILP64 ${BLIS_REF_ROOT}/lib
+    elif [[ -e "/usr/local/lib/libblis.a" ]]; then
+        ln -sf /usr/local/include/blis ${BLIS_REF_ROOT}/include/blis
+        ln -sf /usr/local/lib ${BLIS_REF_ROOT}/lib
+    else
+        echo "error: BLIS lib not found."
+    fi
+}
+
 build_hipsparselt() {
     echo "Start build"
 
@@ -32,21 +62,17 @@ build_hipsparselt() {
 
     cd $COMPONENT_SRC
     mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
-    init_rocm_common_cmake_params
-
-    if [ -n "$GPU_ARCHS" ]; then
-        GPU_TARGETS="$GPU_ARCHS"
-    else
-        # gfx940;gfx941;gfx942
-        GPU_TARGETS=all
+    if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
+        create_blis_link
+        EXTRA_CMAKE_OPTIONS="-DLINK_BLIS=ON -DBUILD_DIR=${BUILD_DIR}"
     fi
 
-    FC=gfortran \
-    CXX="${ROCM_PATH}/bin/hipcc" \
+    init_rocm_common_cmake_params
+
+    CXX=$(set_build_variables __CXX__) \
     cmake \
-        -DAMDGPU_TARGETS=${GPU_TARGETS} \
         ${LAUNCHER_FLAGS} \
-        "${rocm_math_common_cmake_params[@]}" \
+         "${rocm_math_common_cmake_params[@]}" \
         -DTensile_LOGIC= \
         -DTensile_CODE_OBJECT_VERSION=default \
         -DTensile_CPU_THREADS= \
@@ -56,13 +82,14 @@ build_hipsparselt() {
         -DBUILD_CLIENTS_BENCHMARKS=ON \
         -DCMAKE_INSTALL_PREFIX=${ROCM_PATH} \
         -DBUILD_ADDRESS_SANITIZER="${ADDRESS_SANITIZER}" \
+        ${EXTRA_CMAKE_OPTIONS} \
         "$COMPONENT_SRC"
 
     cmake --build "$BUILD_DIR" -- -j${PROC}
     cmake --build "$BUILD_DIR" -- install
     cmake --build "$BUILD_DIR" -- package
 
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     $SCCACHE_BIN -s || echo "Unable to display sccache stats"
 }
@@ -86,7 +113,7 @@ print_output_directory() {
 }
 
 case $TARGET in
-    build) build_hipsparselt ;;
+    build) build_hipsparselt; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hipsparselt ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hiptensor.sh

@@ -2,9 +2,10 @@
 
 set -ex
 
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src hipTensor
+disable_debug_package_generation
 
 build_hiptensor() {
     echo "Start build hipTensor"
@@ -22,17 +23,11 @@ build_hiptensor() {
     mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR"
     init_rocm_common_cmake_params
 
-    if [ -n "$GPU_ARCHS" ]; then
-        GPU_TARGETS="$GPU_ARCHS"
-    else
-        GPU_TARGETS="gfx908:xnack-;gfx90a:xnack-;gfx90a:xnack+;gfx940;gfx941;gfx942"
-    fi
-
     cmake \
         -B "${BUILD_DIR}" \
         "${rocm_math_common_cmake_params[@]}" \
-        $(set_build_variables CMAKE_C_CXX) \
-        -DAMDGPU_TARGETS=${GPU_TARGETS} \
+        "$(set_build_variables __CMAKE_CC_PARAMS__)" \
+        "$(set_build_variables __CMAKE_CXX_PARAMS__)" \
         ${LAUNCHER_FLAGS} \
         "$COMPONENT_SRC"
 
@@ -41,7 +36,7 @@ build_hiptensor() {
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p "$PACKAGE_DIR" && cp ${BUILD_DIR}/*.${PKGTYPE} "$PACKAGE_DIR"
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -55,7 +50,7 @@ clean_hiptensor() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_hiptensor ;;
+    build) build_hiptensor; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_hiptensor ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_hsa.sh

@@ -0,0 +1,151 @@
+#!/bin/bash
+
+source "$(dirname "${BASH_SOURCE}")/compute_utils.sh"
+
+printUsage() {
+    echo
+    echo "Usage: $(basename "${BASH_SOURCE}") [options ...] [make options]"
+    echo
+    echo "Options:"
+    echo "  -c,  --clean              Clean output and delete all intermediate work"
+    echo "  -r,  --release            Make a release build instead of a debug build"
+    echo "  -a,  --address_sanitizer  Enable address sanitizer"
+    echo "  -o,  --outdir <pkg_type>  Print path of output directory containing packages of type referred to by pkg_type"
+    echo "  -s,  --static             Build static lib (.a).  build instead of dynamic/shared(.so) "
+    echo "  -w,  --wheel              Creates python wheel package of hsa. It needs to be used along with -r option"
+    echo "  -h,  --help               Prints this help"
+    echo
+    echo
+
+    return 0
+}
+
+PROJ_NAME="rocr"
+TARGET="build"
+PACKAGE_ROOT="$(getPackageRoot)"
+PACKAGE_SRC="$(getSrcPath)"
+PACKAGE_LIB="$(getLibPath)"
+PACKAGE_BIN="$(getBinPath)"
+PACKAGE_DEB="$(getPackageRoot)/deb/$PROJ_NAME"
+PACKAGE_RPM="$(getPackageRoot)/rpm/$PROJ_NAME"
+MAKEARG=""
+CORE_BUILD_DIR="$(getBuildPath hsa-core)"
+ROCR_DEV_BUILD_DIR="$(getBuildPath hsa-rocr-dev)"
+PREFIX_PATH="$PACKAGE_ROOT"
+BUILD_TYPE="Debug"
+SHARED_LIBS="ON"
+CLEAN_OR_OUT=0;
+MAKETARGET="deb"
+PKGTYPE="deb"
+
+unset HIP_DEVICE_LIB_PATH
+unset ROCM_PATH
+
+#parse the arguments
+VALID_STR=`getopt -o hcraswo: --long help,clean,release,static,wheel,address_sanitizer,outdir: -- "$@"`
+eval set -- "$VALID_STR"
+
+while true ;
+do
+    #echo "parocessing $1"
+    case "$1" in
+        (-h | --help)
+                printUsage ; exit 0;;
+        (-c | --clean)
+                TARGET="clean" ; ((CLEAN_OR_OUT|=1)) ; shift ;;
+        (-r | --release)
+                BUILD_TYPE="RelWithDebInfo" ; shift ;;
+        (-a | --address_sanitizer)
+                set_asan_env_vars
+                set_address_sanitizer_on ; shift ;;
+        (-s | --static)
+                SHARED_LIBS="OFF" ; shift ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ; shift ;;
+        (-o | --outdir)
+                TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 2 ;;
+        --)     shift; break;; # end delimiter
+        (*)
+                echo " This should never come but just incase : UNEXPECTED ERROR Parm : [$1] ">&2 ; exit 20;;
+    esac
+
+done
+
+RET_CONFLICT=1
+check_conflicting_options $CLEAN_OR_OUT $PKGTYPE $MAKETARGET
+if [ $RET_CONFLICT -ge 30 ]; then
+   print_vars $API_NAME $TARGET $BUILD_TYPE $SHARED_LIBS $CLEAN_OR_OUT $PKGTYPE $MAKETARGET
+   exit $RET_CONFLICT
+fi
+
+clean_hsa() {
+    echo "Cleaning HSA"
+
+    rm -rf "$CORE_BUILD_DIR"
+    rm -rf "$PACKAGE_RPM"
+    rm -rf "$PACKAGE_DEB"
+    rm -f  "$PACKAGE_ROOT"/lib/libhsa-runtime*
+    rm -rf "$PACKAGE_ROOT/lib/cmake/hsa-runtime64"
+    rm -rf "$PACKAGE_ROOT/include/hsa"
+    rm -rf "$PACKAGE_ROOT/share/doc/hsa-runtime64"
+    # Remove when switching to flat directory layout.
+    rm -rf "$PACKAGE_ROOT/hsa"
+}
+
+
+build_hsa_core() {
+    echo "Build HSA"
+    local coreMakeOpts="$DASH_JAY -C $CORE_BUILD_DIR"
+
+    echo "$HSA_CORE_ROOT"
+
+    if [ ! -d "$CORE_BUILD_DIR" ]; then
+        mkdir -p "$CORE_BUILD_DIR"
+        pushd "$CORE_BUILD_DIR"
+        print_lib_type $SHARED_LIBS
+
+        cmake $(rocm_cmake_params) \
+            -DBUILD_SHARED_LIBS=$SHARED_LIBS \
+            -DENABLE_LDCONFIG=OFF \
+            $(rocm_common_cmake_params) \
+            -DADDRESS_SANITIZER="$ADDRESS_SANITIZER" \
+            "$HSA_CORE_ROOT"
+        popd
+    fi
+    time cmake --build "$CORE_BUILD_DIR" -- $coreMakeOpts
+    time cmake --build "$CORE_BUILD_DIR" -- $coreMakeOpts install
+    time cmake --build "$CORE_BUILD_DIR" -- $coreMakeOpts package
+
+    copy_if DEB "${CPACKGEN:-"DEB;RPM"}" "$PACKAGE_DEB" $CORE_BUILD_DIR/hsa-rocr*.deb
+    copy_if RPM "${CPACKGEN:-"DEB;RPM"}" "$PACKAGE_RPM" $CORE_BUILD_DIR/hsa-rocr*.rpm
+}
+
+print_output_directory() {
+    case ${PKGTYPE} in
+        ("deb")
+            echo ${PACKAGE_DEB};;
+        ("rpm")
+            echo ${PACKAGE_RPM};;
+        (*)
+            echo "Invalid package type \"${PKGTYPE}\" provided for -o" >&2; exit 1;;
+    esac
+    exit
+}
+
+case $TARGET in
+    (clean)
+        clean_hsa
+        ;;
+    (build)
+        build_hsa_core
+        build_wheel "$CORE_BUILD_DIR" "$PROJ_NAME"
+        ;;
+    (outdir)
+        print_output_directory
+        ;;
+    (*)
+        die "Invalid target $TARGET"
+        ;;
+esac
+
+echo "Operation complete"

tools/rocm-build/build_lightning.sh

@@ -25,7 +25,10 @@ printUsage() {
     echo "  -P,  --package               Generate packages"
     echo "  -N,  --skip_lit_tests        Skip llvm lit testing (llvm lit testing is enabled by default)"
     echo "  -M,  --skip_man_pages        Skip llvm documentation generation (man pages and rocm-llvm-docs package generation is enabled by default)"
+    echo "       --skip_utils            Skip installation of LLVM Utils (FileCheck, not, etc.)"
     echo "  -h,  --help                  Prints this help"
+    echo "       --compiler_rt_debug     Build a debug version of runtimes"
+    echo "       --compiler_rt_release   Build a release version of runtimes (enabled by default)"
     echo
     echo
 
@@ -42,17 +45,27 @@ DEB_PATH="$(getDebPath $PROJ_NAME)"
 RPM_PATH="$(getRpmPath $PROJ_NAME)"
 INSTALL_PATH="${ROCM_INSTALL_PATH}/lib/llvm"
 LLVM_ROOT_LCL="${LLVM_ROOT}"
-ROCM_WHEEL_DIR="${BUILD_PATH}/_wheel"
 
 TARGET="all"
 MAKEOPTS="$DASH_JAY"
+# Default to build release with Assertions ON
 BUILD_TYPE="Release"
-ENABLE_ASSERTIONS=0
+COMPILER_RT_DEBUG="OFF"
+# Enable assertions for all builds except release, nfar, and afar jobs.
+case "${JOB_NAME}" in
+   ( *"rel"*                  | \
+     *"nfar"*                 )
+       ENABLE_ASSERTIONS=0 ;;
+     ( * )
+       ENABLE_ASSERTIONS=1 ;;
+esac
 SHARED_LIBS="ON"
 
+# Build LLVM libraries for dynamically linking.  Default is to build LLVM with statically linked libraries
 BUILD_LLVM_DYLIB="OFF"
 
 FLANG_NEW=0
+CHK_FLANG=""
 CLEAN_OR_OUT=0;
 PKGTYPE="deb"
 MAKETARGET="deb"
@@ -62,19 +75,31 @@ ASSERT_LLVM_VERSION_MINOR=""
 
 SKIP_LIT_TESTS=0
 BUILD_MANPAGES="ON"
+LLVM_INSTALL_UTILS="ON"
 STATIC_FLAG=
 
 SANITIZER_AMDGPU=1
+
+if [ -d "$WORK_ROOT/ROCR-Runtime/runtime/hsa-runtime/inc/" ]; then
     HSA_INC_PATH="$WORK_ROOT/ROCR-Runtime/runtime/hsa-runtime/inc/"
+else
+    HSA_INC_PATH="$WORK_ROOT/ROCR-Runtime/src/inc"
+fi
 COMGR_INC_PATH="$COMGR_ROOT/include"
 
-VALID_STR=`getopt -o hcV:v:draAswlo:BPNM --long help,clean,assert_llvm_ver_major:,assert_llvm_ver_minor:,debug,release,address_sanitizer,no_address_sanitizer,static,build_llvm_static,wheel,build,package,skip_lit_tests,skip_man_pages,outdir: -- "$@"`
+#parse the arguments
+VALID_STR=`getopt -o hcV:v:draAswlo:BPNM --long help,clean,assert_llvm_ver_major:,assert_llvm_ver_minor:,debug,release,address_sanitizer,no_address_sanitizer,static,build_llvm_static,wheel,build,package,skip_lit_tests,skip_man_pages,skip_utils,compiler_rt_debug,compiler_rt_release,outdir: -- "$@"`
 eval set -- "$VALID_STR"
 
 set_dwarf_version(){
+  # In SLES and RHEL splitting debuginfo is getting failed(dwarf-5 unhandled) when compiler is set to clang.
+  # By default -gdwarf-5 is used for the compression of debug symbols
+  # So setting -gdwarf-4
+  # SWDEV-462774: In RHEL/SLES with dwarf-4 enabled, seeing ASAN llvm build getting killed due to high memory usage.
+  # So adding -gsplit-dwarf which splits the debug information into separate files to reduce memory usage during linking.
   case "$DISTRO_ID" in
     (sles*|rhel*)
-       SET_DWARF_VERSION_4="-gdwarf-4"
+       SET_DWARF_VERSION_4="-gdwarf-4 -gsplit-dwarf"
        ;;
     (*)
        SET_DWARF_VERSION_4=""
@@ -103,9 +128,7 @@ do
                 BUILD_TYPE="Release" ; shift ;;
         (-a | --address_sanitizer)
                 set_dwarf_version
-                SANITIZER_AMDGPU=1 ;
-                HSA_INC_PATH="$WORK_ROOT/hsa/runtime/opensrc/hsa-runtime/inc" ;
-                COMGR_INC_PATH="$COMGR_ROOT/include" ; shift ;;
+                SANITIZER_AMDGPU=1 ; shift ;;
         (-A | --no_address_sanitizer)
                 SANITIZER_AMDGPU=0 ;
                 unset HSA_INC_PATH ;
@@ -127,6 +150,12 @@ do
                 SKIP_LIT_TESTS=1; shift ;;
         (-M | --skip_man_pages)
                 BUILD_MANPAGES="OFF"; shift ;;
+        (--skip_utils)
+                LLVM_INSTALL_UTILS="OFF"; shift ;;
+        (--compiler_rt_debug)
+                COMPILER_RT_DEBUG="ON"; shift ;;
+        (--compiler_rt_release)
+                COMPILER_RT_DEBUG="OFF"; shift ;;
         --)     shift; break;; # end delimiter
         (*)
                 echo " This should not happen : UNEXPECTED ERROR Parm : [$1] ">&2 ; exit 20;;
@@ -145,38 +174,51 @@ LLVM_PROJECTS="clang;lld;clang-tools-extra"
 ENABLE_RUNTIMES="compiler-rt;libunwind"
 BOOTSTRAPPING_BUILD_LIBCXX=0
 BUILD_AMDCLANG="ON"
-
+# build libcxx to support the amdclang binary
 ENABLE_RUNTIMES="$ENABLE_RUNTIMES;libcxx;libcxxabi"
 BOOTSTRAPPING_BUILD_LIBCXX=1
 
 clean_lightning() {
-    rm -rf "$ROCM_WHEEL_DIR"
+    # Delete cmake output directory
     rm -rf "$BUILD_PATH"
     rm -rf "$DEB_PATH"
     rm -rf "$RPM_PATH"
 }
 
+# Collect LLVM/Clang git information for clang --version output.
+# Include repo name, branch name and gitdate.
 setup_llvm_info() {
+    # Allow this function to fail repo and git commands, but not crash the build.
     set +e
     mkdir -p "$LLVM_ROOT_LCL"
     pushd "$LLVM_ROOT_LCL"
+    # Initialize local variable. Does not include
+    # LLVM_COMMIT_GITDATE and LLVM_REPO_INFO.
     local LLVM_REMOTE_NAME
     local LLVM_URL_NAME
     local LLVM_BRANCH_NAME
     local LLVM_URL_BRANCH
 
+    # Release Build: set url to external repo.
+    # Internal Build:  set url to actual url.
     if [[ "${JOB_NAME}" == *rel* ]]; then
-        LLVM_URL_NAME="https://github.com/RadeonOpenCompute/llvm-project"
-        LLVM_BRANCH_NAME="roc-${ROCM_VERSION}"
-        LLVM_URL_BRANCH="${LLVM_URL_NAME} ${LLVM_BRANCH_NAME}"
+      LLVM_URL_NAME="https://github.com/RadeonOpenCompute/llvm-project"
+      # Release branch follows convention roc-X.Y.x according to CI.
+      LLVM_BRANCH_NAME="roc-${ROCM_VERSION}"
+      LLVM_URL_BRANCH="${LLVM_URL_NAME} ${LLVM_BRANCH_NAME}"
     else
-        LLVM_REMOTE_NAME=$(git remote)
-        LLVM_URL_NAME=$(git config --get remote."${LLVM_REMOTE_NAME}".url)
-        LLVM_BRANCH_NAME=$(repo manifest | sed -n 's/.*path="external\/llvm-project".* upstream="\([^"]*\)".*/\1/p' )
-        LLVM_URL_BRANCH="${LLVM_URL_NAME} ${LLVM_BRANCH_NAME}"
+      LLVM_REMOTE_NAME=$(git remote)
+      LLVM_URL_NAME=$(git config --get remote."${LLVM_REMOTE_NAME}".url)
+      # Parse the manifest to obtain upstream branch.
+      LLVM_BRANCH_NAME=$(repo manifest | sed -n 's/.*path="external\/llvm-project".* upstream="\([^"]*\)".*/\1/p' )
+      LLVM_URL_BRANCH="${LLVM_URL_NAME} ${LLVM_BRANCH_NAME}"
     fi
 
+    # Obtain git date information based on UTC time.
+    # Format date as yyuuw - [last two digits of year][work week][day of week].
+    # Use xargs to remove trailing newline character.
     LLVM_COMMIT_GITDATE=$(git show -s --format=@%ct | xargs | date -f - --utc +%y%U%w)
+    # Organize into REPO BRANCH GITDATE
     LLVM_REPO_INFO="${LLVM_URL_BRANCH} ${LLVM_COMMIT_GITDATE}"
 
     popd
@@ -200,25 +242,37 @@ get_llvm_version() {
     echo "Detected LLVM version from source: ${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}${LLVM_VERSION_SUFFIX}"
 }
 
+# Generate compiler config files with default ROCm settings
 create_compiler_config_files() {
     local llvm_bin_dir="${INSTALL_PATH}/bin"
     local rocm_cfg="rocm.cfg"
 
+    # rocm.cfg content
     {
         echo "-Wl,--enable-new-dtags"
         echo "--rocm-path='<CFGDIR>/../../..'"
         echo "-frtlib-add-rpath"
     } > "${llvm_bin_dir}/$rocm_cfg"
 
-    local compiler_commands=("clang" "clang++" "clang-cpp" "clang-${LLVM_VERSION_MAJOR}" "clang-cl" "flang" "flang-new")
+    local compiler_commands=("clang" "clang++" "clang-cpp" "clang-${LLVM_VERSION_MAJOR}" "clang-cl" )
     for i in "${compiler_commands[@]}"; do
         if [ -f "$llvm_bin_dir/$i" ]; then
             local config_file="${llvm_bin_dir}/${i}.cfg"
             echo "@${rocm_cfg}" > $config_file
         fi
     done
+
+    if [ $FLANG_NEW -eq 1 ]; then
+        local flang_commands=("flang" "flang-new" "flang-classic")
+        for i in "${flang_commands[@]}"; do
+            local config_file="${llvm_bin_dir}/${i}.cfg"
+            echo "@${rocm_cfg}" > $config_file
+        done
+    fi
 }
 
+# Function to check if a list contains an element
+# return true if $1 is in $2...
 contains(){
 
     local target=$1 element
@@ -230,6 +284,7 @@ contains(){
 }
 
 build_lightning() {
+    # Obtain LLVM build information
     setup_llvm_info
 
     get_llvm_version
@@ -248,11 +303,59 @@ build_lightning() {
         fi
     fi
 
+    # Upstream llvm option -DCLANG_DEFAULT_PIE_ON_LINUX changed to ON
+    # by default however, it breaks ROCM builds on SLES/CentOS distros
+    # so disabling it for now in lighting compiler builds.
+    # Todo : this should be removed to follow the upstream's default.
     DISABLE_PIE=0
 
+    # Find out distro / RHEL build here to adjust Python version for building LLVM if needed.
+    case "$DISTRO_ID" in
+    (rhel*|centos*)
+       RHEL_BUILD=1
+       ;;
+    (*)
+       RHEL_BUILD=0
+       ;;
+    esac
+
+    # Special handling for Debian 10
+    #  - gcc is too ancient and can't build LLVM, switch to clang-13 instead
+    #  - zstd is severely out-of-date and can't support LLVM, disable it
+    case "$DISTRO_ID" in
+    (debian-10)
+       BUILD_COMPILER_LINKER_FLAGS=("-DCMAKE_C_COMPILER=/usr/bin/clang-13"  "-DCMAKE_CXX_COMPILER=/usr/bin/clang++-13" "-DLLVM_USE_LINKER=lld")
+       LLVM_COMPRESSION_FLAGS=("-DLLVM_ENABLE_ZLIB=ON" "-DLLVM_ENABLE_ZSTD=OFF")
+       ;;
+    (*)
+       BUILD_COMPILER_LINKER_FLAGS=()
+       LLVM_COMPRESSION_FLAGS=("-DLLVM_ENABLE_ZLIB=ON")
+       ;;
+    esac
+
+    # Upstream moved to requiring Python 3.8 as the minimum.
+    # Point CMake to that explicit location and adjust LD_LIBRARY_PATH.
+    PYTHON_VERSION_WORKAROUND=''
+    # TODO: Workshop the condition before landing, only to show
+    # what needs to be done for the Python issue.
+    echo "build_lightning DISTRO_ID: ${DISTRO_ID}"
+    if [[ "$DISTRO_ID" == "rhel-8"* || "$DISTRO_NAME" == "sles" || "$DISTRO_ID" == "debian-10" ]]; then
+      EXTRA_PYTHON_PATH=/opt/Python-3.8.13
+      PYTHON_VERSION_WORKAROUND="-DPython3_EXECUTABLE=${EXTRA_PYTHON_PATH}/bin/python3.8"
+      # For the python interpreter we need to export LD_LIBRARY_PATH.
+      # XXX: This is only needed during CMake and ninja check-* targets, maybe we can limit scope further?
+      export LD_LIBRARY_PATH=${EXTRA_PYTHON_PATH}/lib:$LD_LIBRARY_PATH
+    fi
+
+    # Start CMake Build
     mkdir -p "$BUILD_PATH"
     pushd "$BUILD_PATH"
 
+    #Extra llvm cmake parameter can be passed from an environment variable EXTRA_LLVM_CMAKE_PARAMS
+    #For example:
+    #  export EXTRA_LLVM_CMAKE_PARAMS='-DLLVM_LIT_ARGS="-sv -j8" -DP2=1'
+    #Alternatively, this variable can be prefixed to this script:
+    #  EXTRA_LLVM_CMAKE_PARAMS='-DLLVM_LIT_ARGS="-sv -j8" -DP2=1' build_lightning.sh
     eval EXTRA_LLVM_CMAKE_PARAMS_ARRAY=($EXTRA_LLVM_CMAKE_PARAMS)
 
     if [ ! -e Makefile ]; then
@@ -261,14 +364,8 @@ build_lightning() {
         if [ -e "$LLVM_ROOT_LCL/../flang/EnableFlangBuild" ]; then
             FLANG_NEW=1
             LLVM_PROJECTS="$LLVM_PROJECTS;flang"
-            ENABLE_RUNTIMES="$ENABLE_RUNTIMES;openmp";
         else
-            if [[ "${JOB_NAME}" != *afar* ]] && [ -e "$LLVM_ROOT_LCL/../flang/DoROCmRelease" ]; then
-              FLANG_NEW=1
-              LLVM_PROJECTS="$LLVM_PROJECTS;flang"
-            else
-                echo "NOT building project flang"
-            fi
+            echo "NOT building project flang"
         fi
         set -x
         cmake $(rocm_cmake_params) ${GEN_NINJA} \
@@ -313,8 +410,14 @@ build_lightning() {
               -DROCM_LLVM_BACKWARD_COMPAT_LINK="$ROCM_INSTALL_PATH/llvm" \
               -DROCM_LLVM_BACKWARD_COMPAT_LINK_TARGET="./lib/llvm" \
               -DCLANG_LINK_FLANG_LEGACY=ON \
+              -DFLANG_RUNTIME_F128_MATH_LIB=libquadmath \
+              -DLIBOMPTARGET_BUILD_DEVICE_FORTRT=ON \
               -DCMAKE_CXX_STANDARD=17 \
               -DFLANG_INCLUDE_DOCS=OFF \
+              -DLLVM_INSTALL_UTILS="$LLVM_INSTALL_UTILS" \
+              -DCOMPILER_RT_DEBUG="$COMPILER_RT_DEBUG" \
+              "${BUILD_COMPILER_LINKER_FLAGS[@]}" \
+              "${LLVM_COMPRESSION_FLAGS[@]}" \
               "${EXTRA_LLVM_CMAKE_PARAMS_ARRAY[@]}" \
               "$LLVM_ROOT_LCL"
         set +x
@@ -324,28 +427,42 @@ build_lightning() {
     echo "Building LLVM"
 
     if [ $BOOTSTRAPPING_BUILD_LIBCXX -eq 1 ]; then
+        # The following is to do a bootstrap build of libc++ to support
+        # amdclang, which needs the C++17 std::filesystem library
+        # build clang first
         cmake --build . -- $MAKEOPTS clang lld compiler-rt
+        # build libc++ and libcxxabi using the just-built clang
         cmake --build . -- $MAKEOPTS runtimes cxx
     fi
 
+    # Remove these 12 lines
     echo "Workaround for race condition"
+    # find make targets like clang_rt.asan-dynamic-ARCH-version-list
+    # and prebuild them. ARCH will include x86_64 but might also
+    # include i386, depending on the build environment.
+    # TODO figure out why ubuntu 18 builds the i386, but sles, centos
+    # and ubuntu 20 do not. If we can stop ubuntu 18 then it might
+    # give us some speed.
+    #for prebuild in $(make help | sed -n '/-version-list/s/^... //p') ; do
+     #   make $MAKEOPTS $prebuild
+    #done
     echo "End Workaround for race condition"
     cmake --build . -- $MAKEOPTS
-
-    case "$DISTRO_ID" in
-    (rhel*|centos*)
-       RHEL_BUILD=1
-       ;;
-    (*)
-       RHEL_BUILD=0
-       ;;
-     esac
-
+    # FIXME: check-all would be better, but some tests require sys calls that
+    # are not allowed in the CI dockers and so fail. Switch to check-all
+    # when possible.
+    # FIXME: disable testing on sles; a test is failing for unknown reasons
+    # and breaking the sles build; keep it enabled everywhere else
+    # FIXME: disable check-clang on rhel; python 3.8 issue
+    # and breaking the rhel build; keep it enabled everywhere else
     if [ $SKIP_LIT_TESTS -eq 0 ]; then
+        if [ $FLANG_NEW -eq 1 ]; then
+            CHK_FLANG="check-flang"
+        fi
         if [ $RHEL_BUILD -eq 1 ]; then
             cmake --build . -- $MAKEOPTS check-lld check-mlir
         elif [ "$DISTRO_NAME" != "sles" ]; then
-            cmake --build . -- $MAKEOPTS check-llvm check-clang check-lld check-mlir
+            cmake --build . -- $MAKEOPTS check-llvm check-clang check-lld check-mlir $CHK_FLANG
         fi
     fi
     cmake --build . -- $MAKEOPTS clang-tidy
@@ -356,17 +473,21 @@ build_lightning() {
 
 package_lightning_dynamic(){
 
+    # do common stuff here:
     if [ "$BUILD_TYPE" == "Debug" ]; then
         return
     fi
 
+    # Echos ${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}${LLVM_VERSION_SUFFIX}
     get_llvm_version
+    # We require the LLVM major, minor and patch. Scrub the LLVM_VERSION_SUFFIX.
     local llvmParsedVersion="${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}"
     local packageName="rocm-llvm"
     local packageSummary="ROCm compiler"
     local packageSummaryLong="ROCm compiler based on LLVM $llvmParsedVersion"
     local installPath="$ROCM_INSTALL_PATH/lib/llvm/"
 
+    # setup for rocm-llvm-core package
     if [ "$BUILD_LLVM_DYLIB" == "ON" ] ; then
       local packageNameCore="rocm-llvm-core"
       local packageSummaryCore="ROCm core compiler dylibs"
@@ -385,14 +506,16 @@ package_lightning_dynamic(){
     local packageDir="$BUILD_PATH/package"
     local backwardsCompatibleSymlink="$ROCM_INSTALL_PATH/llvm"
 
+    # Debian specific
     local packageDeb="$packageDir/deb"
     local controlFile="$packageDeb/DEBIAN/control"
     local postinstFile="$packageDeb/DEBIAN/postinst"
     local prermFile="$packageDeb/DEBIAN/prerm"
     local specFile="$packageDir/$packageName.spec"
     local debDependencies="python3, libc6, libstdc++6|libstdc++8, libstdc++-5-dev|libstdc++-7-dev|libstdc++-11-dev, libgcc-5-dev|libgcc-7-dev|libgcc-11-dev, rocm-core"
-    local debRecommends="gcc, g++, gcc-multilib, g++-multilib"
+    local debRecommends="gcc, g++, gcc-multilib, g++-multilib, rocm-device-libs"
 
+    # RPM specific
     local packageRpm="$packageDir/rpm"
     local packageRpmCore="$packageDir/rpm"
     local specFile="$packageDir/$packageName.spec"
@@ -402,14 +525,15 @@ package_lightning_dynamic(){
         rpmRequires+=", rocm-llvm-core"
     fi
     local rpmRequiresCore="rocm-core"
-    local rpmRecommends="gcc, gcc-c++, devtoolset-7-gcc-c++"
+    local rpmRecommends="gcc, gcc-c++, devtoolset-7-gcc-c++, rocm-device-libs"
 
+    # Cleanup previous packages
     rm -rf "$packageDir"
     echo "rm -rf $packageDir"
     rm -rf "$DEB_PATH"
 
-    local amd_compiler_commands=("amdclang" "amdclang++" "amdclang-cl" "amdclang-cpp" "amdflang" "amdlld" "offload-arch")
-    local amd_man_pages=("amdclang.1.gz" "flang.1.gz" "amdflang.1.gz")
+    local amd_compiler_commands=("amdclang" "amdclang++" "amdclang-cl" "amdclang-cpp" "amdflang" "amdflang-new" "amdflang-legacy" "amdflang-classic" "amdlld" "offload-arch")
+    local amd_man_pages=("amdclang.1.gz" "amdclang++.1.gz" "flang.1.gz" "amdflang.1.gz")
     local man_pages=("bugpoint.1" "FileCheck.1" "llvm-ar.1" "llvm-cxxmap.1" "llvm-extract.1" "llvm-lipo.1" "llvm-otool.1" "llvm-readobj.1" "llvm-symbolizer.1" "tblgen.1"
                      "clang.1" "lit.1" "llvm-as.1" "llvm-diff.1" "llvm-ifs.1" "llvm-locstats.1" "llvm-pdbutil.1" "llvm-remarkutil.1" "llvm-tblgen.1" "clang-tblgen.1"
                      "llc.1" "llvm-bcanalyzer.1" "llvm-dis.1" "llvm-install-name-tool.1" "llvm-mca.1" "llvm-profdata.1" "llvm-size.1" "llvm-tli-checker.1" "diagtool.1"
@@ -417,17 +541,22 @@ package_lightning_dynamic(){
                      "llvm-cov.1" "llvm-dwarfutil.1" "llvm-libtool-darwin.1" "llvm-objcopy.1" "llvm-ranlib.1" "llvm-strings.1" "opt.1" "extraclangtools.1" "llvm-addr2line.1"
                      "llvm-cxxfilt.1" "llvm-exegesis.1" "llvm-link.1" "llvm-objdump.1" "llvm-readelf.1" "llvm-strip.1")
 
+    # Only build deb in Ubuntu environment
     if [ "$PACKAGEEXT" = "deb" ]; then
+        # Debian packaging
 
         mkdir -p "$packageDeb/$installPath"
         mkdir -p "$(dirname $controlFile)"
         mkdir -p "$DEB_PATH"
         mkdir -p "$packageDeb/$licenseDir"
 
+        # build the rocm-llvm-core package first:
+        # we only need some .so files, and we're doing a MV so that they're not picked up in the next (rocm-llvm) package.
         if [ "$BUILD_LLVM_DYLIB" == "ON" ] ; then
 
           mkdir -p "$packageDeb/$licenseDirCore"
 
+          # Copy the LICENSE file to rocm-core
           cp -r "$LLVM_ROOT_LCL/LICENSE.TXT" "$packageDeb/$licenseDirCore"
 
           cp -P "$distLib/libLLVM"*"so"* "$packageDeb/$installPath/"
@@ -449,28 +578,36 @@ package_lightning_dynamic(){
           fakeroot dpkg-deb -Zgzip --build $packageDeb \
           "${DEB_PATH}/${packageNameCore}_${packageVersion}.${ROCM_LIBPATCH_VERSION}-${JOB_DESIGNATOR}${BUILD_ID}~${DISTRO_RELEASE}_${packageArch}.deb"
 
+          # clean up, get ready for rocm-llvm
           rm -rf "$controlFile"
           rm -rf "$packageDeb/$licenseDirCore"
 
+          #remove the LLVM/Clang dylibs we already packaged in rocm-llvm-core
           rm -f "$packageDeb/$installPath/libLLVM"*"so"*
           rm -f "$packageDeb/$installPath/libFortran"*"so"*
           rm -f "$packageDeb/$installPath/libclang"*"so"*
 
           mkdir -p "$(dirname $controlFile)"
 
+          # clean up dylibs we already packaged:
           rm -rf "$packageDeb/$installPath/*"
 
+          # add this rocm-llvm-core package as a dependency for rocm-llvm
           debDependencies="${debDependencies}, ${packageNameCore}"
-        fi
+        fi  # end of building rocm-llvm-core package
 
+        # now build rocm-llvm:
+        # Install license file in rocm-llvm package .
         cp -r "$LLVM_ROOT_LCL/LICENSE.TXT" "$packageDeb/$licenseDir"
         cp -r "$distBin" "$packageDeb/$installPath/bin"
         cp -r "$distInc" "$packageDeb/$installPath/include"
         cp -r "$distLib" "$packageDeb/$installPath/lib"
         if [ "$BUILD_MANPAGES" == "ON" ]; then
+            # zip the generated man pages before installing them in the package
             for i in "${man_pages[@]}"; do
                 gzip -f "$distMan/man1/$i"
             done
+            # Create symbolic links for amd man pages
             if [ -f "$distMan/man1/clang.1.gz" ]; then
                 for i in "${amd_man_pages[@]}"; do
                     ln -sf "clang.1.gz" "$distMan/man1/$i"
@@ -507,11 +644,14 @@ package_lightning_dynamic(){
         "${DEB_PATH}/${packageName}_${packageVersion}.${ROCM_LIBPATCH_VERSION}-${JOB_DESIGNATOR}${BUILD_ID}~${DISTRO_RELEASE}_${packageArch}.deb"
     fi
 
+    # Only build RPM in CentOS or SLES or RHEL or Mariner environment
     if [ "$PACKAGEEXT" = "rpm" ]; then
+        # RPM packaging
         mkdir -p "$(dirname $specFile)"
         rm -rf "$RPM_PATH"
         mkdir -p "$RPM_PATH"
 
+        # set up for rocm-llvm-core package
         if [ "$BUILD_LLVM_DYLIB" == "ON" ] ; then
           echo "Name:       $packageNameCore" > $specFileCore
           echo "Version:    ${packageVersion}.${ROCM_LIBPATCH_VERSION}" >> $specFileCore
@@ -542,6 +682,11 @@ package_lightning_dynamic(){
           echo "rm -rf \$RPM_BUILD_ROOT/$installPath" >> $specFileCore
           echo "%files " >> $specFileCore
           echo "%defattr(-,root,root,-)" >> $specFileCore
+          # Note: In some OS like SLES, during upgrade rocm-core is getting upgraded first and followed by other packages
+          # rocm-core cannot delete rocm-ver folder, since it contains files of other packages that are yet to be upgraded
+          # To remove rocm-ver folder after upgrade the spec file of other packages should contain the rocm-ver directory
+          # Otherwise after upgrade empty old rocm-ver folder will be left out.
+          # If empty remove /opt/rocm-ver folder and its subdirectories created by rocm-llvm-core package
           echo "$ROCM_INSTALL_PATH" >> $specFileCore
 
           echo "%post" >> $specFileCore
@@ -552,8 +697,9 @@ package_lightning_dynamic(){
           rpmbuild --define "_topdir $packageRpmCore" -ba $specFileCore
 
           mv $packageRpm/RPMS/x86_64/*.rpm $RPM_PATH
-        fi
+        fi # end of building rocm-lllvm-core package
 
+        # setup for rocm-llvm package
         echo "Name:       $packageName" > $specFile
         echo "Version:    ${packageVersion}.${ROCM_LIBPATCH_VERSION}" >> $specFile
         echo "Release:    ${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}%{?dist}" >> $specFile
@@ -586,9 +732,11 @@ package_lightning_dynamic(){
         echo "cp -R $distInc \$RPM_BUILD_ROOT/$installPath" >> $specFile
         echo "cp -R $distLib \$RPM_BUILD_ROOT/$installPath" >> $specFile
         if [ "$BUILD_MANPAGES" == "ON" ]; then
+            # zip the generated man pages before installing them in the package
             for i in "${man_pages[@]}"; do
                 echo "gzip -f $distMan/man1/$i" >> $specFile
             done
+            # Create symbolic links for amd man pages
             if [ -f "$distMan/man1/clang.1.gz" ]; then
                 for i in "${amd_man_pages[@]}"; do
                     echo "ln -sf clang.1.gz \"$distMan/man1/$i\"" >> $specFile
@@ -607,6 +755,11 @@ package_lightning_dynamic(){
         fi
 
         echo "%defattr(-,root,root,-)" >> $specFile
+        # Note: In some OS like SLES, during upgrade rocm-core is getting upgraded first and followed by other packages
+        # rocm-core cannot delete rocm-ver folder, since it contains files of other packages that are yet to be upgraded
+        # To remove rocm-ver folder after upgrade the spec file of other packages should contain the rocm-ver directory
+        # Otherwise after upgrade empty old rocm-ver folder will be left out.
+        # If empty remove /opt/rocm-ver folder and its subdirectories created by rocm-llvm package
         echo "$ROCM_INSTALL_PATH" >> $specFile
 
         echo "%post" >> $specFile
@@ -633,16 +786,21 @@ package_lightning_dynamic(){
 
 package_lightning_static() {
 
+    # do common stuff here:
     if [ "$BUILD_TYPE" == "Debug" ]; then
         return
     fi
 
+    # Echo the LLVM version ${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}${LLVM_VERSION_SUFFIX}
     get_llvm_version
+    # We require the LLVM major, minor and patch. Scrub the LLVM_VERSION_SUFFIX.
     local llvmParsedVersion="${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}"
 
+    # setup for rocm-llvm package
     local packageName="rocm-llvm"
     local packageSummary="ROCm core compiler"
     local packageSummaryLong="ROCm core compiler based on LLVM $llvmParsedVersion"
+    # setup for rocm-llvm-dev package
     if [ "$PACKAGEEXT" = "deb" ]; then
         local packageNameExtra="rocm-llvm-dev"
     else
@@ -663,40 +821,46 @@ package_lightning_static() {
     local packageDir="$BUILD_PATH/package"
     local backwardsCompatibleSymlink="$ROCM_INSTALL_PATH/llvm"
 
+    # Debian specific
     local packageDeb="$packageDir/deb"
     local controlFile="$packageDeb/DEBIAN/control"
     local postinstFile="$packageDeb/DEBIAN/postinst"
     local prermFile="$packageDeb/DEBIAN/prerm"
     local specFile="$packageDir/$packageName.spec"
     local debDependencies="python3, libc6, libstdc++6|libstdc++8, libstdc++-5-dev|libstdc++-7-dev|libstdc++-11-dev, libgcc-5-dev|libgcc-7-dev|libgcc-11-dev, rocm-core"
-    local debRecommends="gcc, g++, gcc-multilib, g++-multilib"
+    local debRecommends="gcc, g++, gcc-multilib, g++-multilib, rocm-device-libs"
 
+    # RPM specific
     local packageRpm="$packageDir/rpm"
     local packageRpmExtra="$packageDir/rpm"
     local specFile="$packageDir/$packageName.spec"
     local specFileExtra="$packageDir/$packageNameExtra.spec"
     local rpmRequires="rocm-core"
     local rpmRequiresExtra="rocm-core, $packageName"
-    local rpmRecommends="gcc, gcc-c++, devtoolset-7-gcc-c++"
+    #local rpmRecommends="gcc, gcc-c++, devtoolset-7-gcc-c++, rocm-device-libs"
+    local rpmRecommends="rocm-device-libs"
 
+    # Cleanup previous packages
     rm -rf "$packageDir"
     echo "rm -rf $packageDir"
     rm -rf "$DEB_PATH"
 
-    local amd_compiler_commands=("amdclang" "amdclang++" "amdclang-cl" "amdclang-cpp" "amdflang" "amdlld" "offload-arch")
-    local amd_man_pages=("amdclang.1.gz" "flang.1.gz" "amdflang.1.gz")
-    local core_bin=("amdgpu-arch" "amdgpu-offload-arch" "amdlld" "amdllvm" "clang" "clang++" "clang-${LLVM_VERSION_MAJOR}" "clang-cl"
-                    "clang-cpp" "clang-build-select-link" "clang-offload-bundler" "clang-offload-packager" "clang-offload-wrapper" "clang-linker-wrapper" "clang-nvlink-wrapper" "flang" "flang-new"
+    local amd_compiler_commands=("amdclang" "amdclang++" "amdclang-cl" "amdclang-cpp" "amdflang" "amdflang-new" "amdflang-legacy" "amdflang-classic" "amdlld" "offload-arch")
+    local amd_man_pages=("amdclang.1.gz" "amdclang++.1.gz" "flang.1.gz" "amdflang.1.gz")
+    local core_bin=("amd-llvm-spirv" "amdgpu-arch" "amdgpu-offload-arch" "amdlld" "amdllvm" "clang" "clang++" "clang-${LLVM_VERSION_MAJOR}" "clang-cl"
+                    "clang-cpp" "clang-build-select-link" "clang-offload-bundler" "clang-offload-packager" "clang-offload-wrapper" "clang-linker-wrapper" "clang-nvlink-wrapper" "flang" "flang-new" "flang-legacy" "flang-classic" "llvm-symbolizer"
                     "ld64.lld" "ld.lld" "llc" "lld" "lld-link" "llvm-ar" "llvm-bitcode-strip" "llvm-dwarfdump" "llvm-install-name-tool"
                     "llvm-link" "llvm-mc" "llvm-objcopy" "llvm-objdump" "llvm-otool" "llvm-ranlib" "llvm-readelf" "llvm-readobj" "llvm-strip"
                     "nvidia-arch" "nvptx-arch" "offload-arch" "opt" "wasm-ld" "amdclang" "amdclang++" "amdclang-${LLVM_VERSION_MAJOR}" "amdclang-cl"
-                    "amdclang-cpp" "amdflang"
-                    "clang++.cfg" "clang-${LLVM_VERSION_MAJOR}.cfg" "clang-cl.cfg" "clang-cpp.cfg" "clang.cfg" "rocm.cfg")
+                    "amdclang-cpp" "amdflang" "amdflang-new" "amdflang-legacy" "amdflang-classic"
+                    "clang++.cfg" "clang-${LLVM_VERSION_MAJOR}.cfg" "clang-cl.cfg" "clang-cpp.cfg" "clang.cfg" "rocm.cfg"
+                    "flang.cfg" "flang-new.cfg" "flang-classic.cfg" )
     local core_lib=("libclang-cpp.so.${LLVM_VERSION_MAJOR}${LLVM_VERSION_SUFFIX}" "libclang-cpp.so" "libclang.so"
+                    "libclang-cpp.so.${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}${LLVM_VERSION_SUFFIX}"
                     "libclang.so.${LLVM_VERSION_MAJOR}${LLVM_VERSION_SUFFIX}"
                     "libclang.so.${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}${LLVM_VERSION_SUFFIX}"
                     "libFortranSemantics.a" "libFortranLower.a" "libFortranParser.a" "libFortranCommon.a"
-                    "libFortranRuntime.a" "libFortran_main.a" "libFortranDecimal.a" "libFortranEvaluate.a")
+                    "libFortranRuntime.a" "libFortranFloat128Math.a" "libFortranDecimal.a" "libFortranEvaluate.a")
     local core_man_pages=("llvm-otool.1" "llvm-readobj.1" "clang.1" "lit.1" "llc.1" "llvm-ar.1" "llvm-dwarfdump.1" "llvm-objcopy.1" "opt.1"
                           "llvm-link.1" "llvm-mc.1" "llvm-objdump.1" "llvm-ranlib.1" "llvm-readelf.1" "llvm-strip.1")
     local dev_man_pages=("bugpoint.1" "FileCheck.1" "llvm-cxxmap.1" "llvm-extract.1" "llvm-lipo.1" "llvm-symbolizer.1"
@@ -707,27 +871,44 @@ package_lightning_static() {
                            "lli.1" "llvm-cov.1" "llvm-dwarfutil.1" "llvm-libtool-darwin.1" "llvm-strings.1"
                            "extraclangtools.1" "llvm-addr2line.1" "llvm-cxxfilt.1" "llvm-exegesis.1" "scan-build.1")
 
+    # Only build deb in Ubuntu environment
     if [ "$PACKAGEEXT" = "deb" ]; then
+        # Debian packaging
+
         mkdir -p "$packageDeb/$installPath"
         mkdir "${controlFile%/*}"
         mkdir -p "$DEB_PATH"
         mkdir -p "$packageDeb/$licenseDir"
 
+        # Build the static rocm-llvm package first:
+        # We're doing a MV so that they're not picked up in the next (rocm-dev) package.
+
+        # Install license file in rocm-llvm package.
         cp -r "$LLVM_ROOT_LCL/LICENSE.TXT" "$packageDeb/$licenseDir"
 
+        # Copy the core binaries
         mkdir -p "$packageDeb/$installPath/bin"
         for i in "${core_bin[@]}"; do
             if [ -f "$distBin/$i" ]; then
                 cp -d "$distBin"/$i "$packageDeb/$installPath/bin/"
             fi
         done
+        # FIXME: At this point flang is pointing to flang-classic, which does
+        # not exist yet (created later), and code above does not copy it
+        # because it check if it exists first. Blindly copy flang for now.
+        # Permanent fix will be merged into openmp-extras by moving symlink
+        # creation there.
+        if [ -h $distBin/flang ]; then
+          cp -d "$distBin/flang" "$packageDeb/$installPath/bin/"
+        fi
 
-        cp -d "$distBin/flang" "$packageDeb/$installPath/bin/"
-
+        # Copy the core libraries
+        # TODO: Only copy llvm/lib/clang/X.0.0/include and llvm/lib/clang/X.0.0/lib/linux
         mkdir -p "$packageDeb/$installPath/lib/clang"
         cp -r "$distLib/clang/" "$packageDeb/$installPath/lib/"
 
         if [ $FLANG_NEW -eq 1 ]; then
+          # flang-new mod files
           mkdir -p "$packageDeb/$installPath/include/flang"
           cp -r "$distInc/flang/" "$packageDeb/$installPath/include/"
         fi
@@ -738,14 +919,17 @@ package_lightning_static() {
             fi
         done
 
+        # Copy core specific man pages
         if [ "$BUILD_MANPAGES" == "ON" ]; then
             mkdir -p "$packageDeb/$installPath/share/man1"
+            # zip the generated man pages before installing them in the package
             for i in "${core_man_pages[@]}"; do
                 if [ -f "$distMan/man1/$i" ]; then
                     gzip -f "$distMan/man1/$i"
                     cp -d "$distMan/man1/${i}.gz" "$packageDeb/$installPath/share/man1/"
                 fi
             done
+            # Create symbolic links for amd man pages
             if [ -f "$distMan/man1/clang.1.gz" ]; then
                 for i in "${amd_man_pages[@]}"; do
                     ln -sf "clang.1.gz" "$distMan/man1/$i"
@@ -764,7 +948,9 @@ package_lightning_static() {
         done
         echo "rmdir --ignore-fail-on-non-empty \"$ROCM_INSTALL_PATH/bin\"" >> $prermFile
         chmod 0555 "$postinstFile" "$prermFile"
-        cp -P "$backwardsCompatibleSymlink" "$packageDeb/$ROCM_INSTALL_PATH"
+        if [ -h $backwardsCompatibleSymlink ]; then
+          cp -P "$backwardsCompatibleSymlink" "$packageDeb/$ROCM_INSTALL_PATH"
+        fi
 
         {
             echo "Package: $packageName"
@@ -783,6 +969,7 @@ package_lightning_static() {
         fakeroot dpkg-deb -Zgzip --build $packageDeb \
             "${DEB_PATH}/${packageName}_${packageVersion}.${ROCM_LIBPATCH_VERSION}-${JOB_DESIGNATOR}${BUILD_ID}~${DISTRO_RELEASE}_${packageArch}.deb"
 
+        # clean up, get ready for rocm-llvm-dev
         rm -rf "$controlFile"
         rm -rf "$packageDeb"
 
@@ -790,18 +977,22 @@ package_lightning_static() {
         mkdir "${controlFile%/*}"
         mkdir -p "$DEB_PATH"
 
+        # now build rocm-llvm-dev:
         mkdir -p "$packageDeb/$installPath/bin"
+        # Only copy binaries not packaged in core
         for i in "$distBin"/*; do
             bin=$(basename "$i")
             contains "$bin" "${core_bin[@]}" "${amd_compiler_commands[@]}" && continue
             cp -d "$i" "$packageDeb/$installPath/bin/"
         done
 
+        # Only copy libraries not packaged in core
         for i in "$distLib"/*; do
             lib=$(basename "$i")
             contains "$lib" "${core_lib[@]}" && continue
             cp -dr "$i" "$packageDeb/$installPath/lib/"
         done
+        # Remove dirs already packaged in rocm-llvm-core
         rm -rf "$packageDeb/$installPath/lib/clang"
 
         cp -r "$distInc" "$packageDeb/$installPath/include"
@@ -812,6 +1003,7 @@ package_lightning_static() {
 
         if [ "$BUILD_MANPAGES" == "ON" ]; then
             mkdir -p "$packageDeb/$installPath/share/man1"
+            # zip the generated man pages before installing them in the package
             for i in "${dev_man_pages[@]}"; do
                 if [ -f "$distMan/man1/$i" ]; then
                     gzip -f "$distMan/man1/$i"
@@ -820,6 +1012,7 @@ package_lightning_static() {
             done
         fi
 
+        # add this rocm-llvm package as a dependency for rocm-llvm-dev
         debDependencies="${debDependencies}, ${packageName}"
 
         echo "Package: $packageNameExtra"  > $controlFile
@@ -838,20 +1031,26 @@ package_lightning_static() {
         "${DEB_PATH}/${packageNameExtra}_${packageVersion}.${ROCM_LIBPATCH_VERSION}-${JOB_DESIGNATOR}${BUILD_ID}~${DISTRO_RELEASE}_${packageArch}.deb"
     fi
 
+    # Only build RPM in CentOS or SLES or RHEL or Mariner environment
     if [ "$PACKAGEEXT" = "rpm" ]; then
+        # RPM packaging
 
         mkdir -p "$(dirname $specFile)"
         rm -rf "$RPM_PATH"
         mkdir -p "$RPM_PATH"
 
-        echo "Name:       $packageName" > $specFile
+        # set up for rocm-llvm package
+        echo "Name:       $packageName" >> $specFile
         echo "Version:    ${packageVersion}.${ROCM_LIBPATCH_VERSION}" >> $specFile
         echo "Release:    ${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}%{?dist}" >> $specFile
         echo "Summary:    $packageSummary" >> $specFile
         echo "Group:      System Environment/Libraries" >> $specFile
         echo "License:    ASL 2.0 with exceptions" >> $specFile
+        echo "Prefix:     $ROCM_INSTALL_PATH" >> $specFile
         echo "Requires:   $rpmRequires" >> $specFile
-
+        if [[ "$DISTRO_ID" != "centos"* ]]; then
+            echo "Recommends: $rpmRecommends" >> $specFile
+        fi
         echo "%description" >> $specFile
         echo "$packageSummaryLong" >> $specFile
 
@@ -863,23 +1062,38 @@ package_lightning_static() {
         echo "mkdir -p  \$RPM_BUILD_ROOT/$installPath/bin" >> $specFile
         echo "mkdir -p  \$RPM_BUILD_ROOT/$licenseDir" >> $specFile
 
+        # Install the license file to rocm-llvm
         echo "cp -R $LLVM_ROOT_LCL/LICENSE.TXT \$RPM_BUILD_ROOT/$licenseDir" >> $specFile
-        echo "cp -P $backwardsCompatibleSymlink \$RPM_BUILD_ROOT/$ROCM_INSTALL_PATH" >> $specFile
+        if [ -h $backwardsCompatibleSymlink ]; then
+          echo "cp -P $backwardsCompatibleSymlink \$RPM_BUILD_ROOT/$ROCM_INSTALL_PATH" >> $specFile
+        fi
 
+        # Copy the core binaries
         for i in "${core_bin[@]}"; do
             if [ -f "$distBin/$i" ]; then
                 echo "cp -d \"$distBin\"/$i \$RPM_BUILD_ROOT/$installPath/bin/" >> $specFile
             fi
         done
 
-        echo "cp -d \"$distBin/flang\" \$RPM_BUILD_ROOT/$installPath/bin/" >> $specFile
+        # FIXME: At this point flang is pointing to flang-classic, which does
+        # not exist yet (created later), and code above does not copy it
+        # because it check if it exists first. Blindly copy flang for now.
+        # Permanent fix will be merged into openmp-extras by moving symlink
+        # creation there.
+        if [ -h $distBin/flang ]; then
+          echo "cp -d \"$distBin/flang\" \$RPM_BUILD_ROOT/$installPath/bin/" >> $specFile
+        fi
 
+        # Copy the config files
         echo "cp -d \"$distBin\"/*.cfg \$RPM_BUILD_ROOT/$installPath/bin/" >> $specFile
 
+        # Copy the core libraries
+        # TODO: Only copy llvm/lib/clang/X.0.0/include and llvm/lib/clang/X.0.0/lib/linux
         echo "mkdir -p \$RPM_BUILD_ROOT/$installPath/lib/clang" >> $specFile
         echo "cp -R \"$distLib/clang/\" \$RPM_BUILD_ROOT/$installPath/lib/" >> $specFile
 
         if [ $FLANG_NEW -eq 1 ]; then
+          # flang-new mod files
           echo "mkdir -p \$RPM_BUILD_ROOT/$installPath/include/flang" >> $specFile
           echo "cp -R \"$distInc/flang/\" \$RPM_BUILD_ROOT/$installPath/include/" >> $specFile
         fi
@@ -890,26 +1104,32 @@ package_lightning_static() {
             fi
         done
 
+        # Copy core specific man pages
         if [ "$BUILD_MANPAGES" == "ON" ]; then
             echo "mkdir -p  \$RPM_BUILD_ROOT/$installPath/share/man/man1" >> $specFile
+            # zip the generated man pages before installing them in the package
             for i in "${core_man_pages[@]}"; do
                 if [ -f "$distMan/man1/$i" ]; then
-                    echo "gzip -f $distMan/man1/$i" >> $specFile
-                    echo "cp -d $distMan/man1/${i}.gz \$RPM_BUILD_ROOT/$installPath/share/man/man1/" >> $specFile
+                    echo "gzip -f ${distMan}/man1/${i}" >> $specFile
+                    echo "cp -d ${distMan}/man1/${i}.gz \$RPM_BUILD_ROOT/$installPath/share/man/man1/" >> $specFile
                 fi
             done
-            if [ -f "$distMan/man1/clang.1.gz" ]; then
-                for i in "${amd_man_pages[@]}"; do
-                    echo "ln -sf clang.1.gz \"$distMan/man1/$i\"" >> $specFile
-                    echo "cp -d $distMan/man1/${i} \$RPM_BUILD_ROOT/$installPath/share/man/man1/" >> $specFile
-                done
-            fi
+            # Create symbolic links for amd man pages
+            for i in "${amd_man_pages[@]}"; do
+                echo "ln -sf clang.1.gz ${distMan}/man1/${i}" >> $specFile
+                echo "cp -d ${distMan}/man1/${i} \$RPM_BUILD_ROOT/$installPath/share/man/man1/" >> $specFile
+            done
         fi
 
         echo "%clean" >> $specFile
         echo "rm -rf \$RPM_BUILD_ROOT/$installPath" >> $specFile
         echo "%files " >> $specFile
         echo "%defattr(-,root,root,-)" >> $specFile
+        # Note: In some OS like SLES, during upgrade rocm-core is getting upgraded first and followed by other packages
+        # rocm-core cannot delete rocm-ver folder, since it contains files of other packages that are yet to be upgraded
+        # To remove rocm-ver folder after upgrade the spec file of other packages should contain the rocm-ver directory
+        # Otherwise after upgrade empty old rocm-ver folder will be left out.
+        # If empty remove /opt/rocm-ver folder and its subdirectories created by rocm-llvm-core package
         {
             echo "$ROCM_INSTALL_PATH"
 
@@ -937,6 +1157,7 @@ package_lightning_static() {
 
         mv $packageRpm/RPMS/x86_64/*.rpm $RPM_PATH
 
+        # setup for rocm-llvm-dev package
         echo "Name:       $packageNameExtra" > $specFileExtra
         echo "Version:    ${packageVersion}.${ROCM_LIBPATCH_VERSION}" >> $specFileExtra
         echo "Release:    ${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}%{?dist}" >> $specFileExtra
@@ -964,11 +1185,13 @@ package_lightning_static() {
 
         echo "cp -P $backwardsCompatibleSymlink \$RPM_BUILD_ROOT/$ROCM_INSTALL_PATH" >> $specFileExtra
 
+        # Only copy binaries not packaged in core
         for i in "$distBin"/*; do
             bin=$(basename "$i")
             contains "$bin" "${core_bin[@]}" "${amd_compiler_commands[@]}" && continue
             echo "cp -d \"$i\" \$RPM_BUILD_ROOT/$installPath/bin/" >> $specFileExtra
         done
+        # Only copy libraries not packaged in core
         for i in "$distLib"/*; do
             lib=$(basename "$i")
             contains "$lib" "${core_lib[@]}" && continue
@@ -976,14 +1199,17 @@ package_lightning_static() {
         done
 
         echo "cp -R $distInc \$RPM_BUILD_ROOT/$installPath" >> $specFileExtra
+        # Remove dirs already packaged in rocm-llvm-core
         echo "rm -rf \$RPM_BUILD_ROOT/$installPath/lib/clang" >> $specFileExtra
 
         if [ $FLANG_NEW -eq 1 ]; then
+          # Remove flang mods packaged in rocm-llvm-core
           echo "rm -rf \$RPM_BUILD_ROOT/$installPath/include/flang" >> $specFileExtra
         fi
 
         if [ "$BUILD_MANPAGES" == "ON" ]; then
             echo "mkdir -p  \$RPM_BUILD_ROOT/$installPath/share/man/man1" >> $specFileExtra
+            # zip the generated man pages before installing them in the package
             for i in "${dev_man_pages[@]}"; do
                 if [ -f "$distMan/man1/$i" ]; then
                     echo "gzip -f $distMan/man1/$i" >> $specFileExtra
@@ -997,6 +1223,11 @@ package_lightning_static() {
         echo "%files " >> $specFileExtra
 
         echo "%defattr(-,root,root,-)" >> $specFileExtra
+        # Note: In some OS like SLES, during upgrade rocm-core is getting upgraded first and followed by other packages
+        # rocm-core cannot delete rocm-ver folder, since it contains files of other packages that are yet to be upgraded
+        # To remove rocm-ver folder after upgrade the spec file of other packages should contain the rocm-ver directory
+        # Otherwise after upgrade empty old rocm-ver folder will be left out.
+        # If empty remove /opt/rocm-ver folder and its subdirectories created by rocm-llvm package
         echo "$ROCM_INSTALL_PATH" >> $specFileExtra
 
         echo "%post" >> $specFileExtra
@@ -1010,6 +1241,7 @@ package_lightning_static() {
 
 package_docs() {
 
+    # do common stuff here:
     if [ "$BUILD_TYPE" == "Debug"  ]; then
         return
     fi
@@ -1018,7 +1250,9 @@ package_docs() {
         return
     fi
 
+    # Echo the LLVM version ${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}${LLVM_VERSION_SUFFIX}
     get_llvm_version
+    # We require the LLVM major, minor and patch. Scrub the LLVM_VERSION_SUFFIX.
     local llvmParsedVersion="${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}"
 
     local packageName="rocm-llvm-docs"
@@ -1033,24 +1267,31 @@ package_docs() {
     local licenseDir="$ROCM_INSTALL_PATH/share/doc/$packageName"
     local packageDir="$BUILD_PATH/package"
 
+    # Debian specific
     local packageDeb="$packageDir/deb"
     local controlFile="$packageDeb/DEBIAN/control"
     local debDependencies="rocm-core"
 
+    # RPM specific
     local packageRpm="$packageDir/rpm"
     local specFile="$packageDir/$packageName.spec"
     local rpmRequires="rocm-core"
 
+    # Cleanup previous packages
     rm -rf "$packageDir"
     echo "rm -rf $packageDir"
 
+    # Only build deb in Ubuntu environment
     if [ "$PACKAGEEXT" = "deb" ]; then
 
+        # Debian packaging
         mkdir -p "$packageDeb/$licenseDir"
         mkdir "${controlFile%/*}"
 
+        # Install license file in rocm-llvm-docs package.
         cp -r "$LLVM_ROOT_LCL/LICENSE.TXT" "$packageDeb/$licenseDir"
 
+        # Copy the docs
         cp -r "$distDoc" "$packageDeb/$licenseDir"
 
         {
@@ -1071,17 +1312,21 @@ package_docs() {
         "${DEB_PATH}/${packageName}_${packageVersion}.${ROCM_LIBPATCH_VERSION}-${JOB_DESIGNATOR}${BUILD_ID}~${DISTRO_RELEASE}_${packageArch}.deb"
     fi
 
+    # Only build RPM in CentOS or SLES or RHEL or Mariner environment
     if [ "$PACKAGEEXT" = "rpm" ]; then
 
+        # RPM packaging
         mkdir -p "$(dirname $specFile)"
         mkdir -p "$RPM_PATH"
 
+        # set up for rocm-llvm-docs package
         echo "Name:       $packageName" > $specFile
         echo "Version:    ${packageVersion}.${ROCM_LIBPATCH_VERSION}" >> $specFile
         echo "Release:    ${JOB_DESIGNATOR}${SLES_BUILD_ID_PREFIX}${BUILD_ID}%{?dist}" >> $specFile
         echo "Summary:    $packageSummary" >> $specFile
         echo "Group:      System Environment/Libraries" >> $specFile
         echo "License:    ASL 2.0 with exceptions" >> $specFile
+        echo "Prefix:     $ROCM_INSTALL_PATH" >> $specFile
         echo "Requires:   $rpmRequires" >> $specFile
 
         echo "%description" >> $specFile
@@ -1093,8 +1338,10 @@ package_docs() {
         echo "%install" >> $specFile
         echo "mkdir -p  \$RPM_BUILD_ROOT/$licenseDir" >> $specFile
 
+        # Install the license file to rocm-llvm-docs
         echo "cp -R $LLVM_ROOT_LCL/LICENSE.TXT \$RPM_BUILD_ROOT/$licenseDir" >> $specFile
 
+        # Copy the docs
         echo "cp -R \"$distDoc\" \$RPM_BUILD_ROOT/$licenseDir" >> $specFile
 
         echo "%clean" >> $specFile
@@ -1128,24 +1375,13 @@ build() {
     create_compiler_config_files
 }
 
-create_wheel_package() {
-    echo "Creating rocm-llvm wheel package"
-    mkdir -p "$ROCM_WHEEL_DIR"
-    cp -f $SCRIPT_ROOT/generate_setup_py.py $ROCM_WHEEL_DIR
-    cp -f $SCRIPT_ROOT/repackage_wheel.sh $ROCM_WHEEL_DIR
-    cd $ROCM_WHEEL_DIR
-    # Currently only supports python3.6
-    ./repackage_wheel.sh $RPM_PATH/rocm-llvm*.rpm python3.6
-    # Copy the wheel created to RPM folder which will be uploaded to artifactory
-    mv "$ROCM_WHEEL_DIR"/dist/*.whl "$RPM_PATH"
-}
-
 case $TARGET in
     (clean) clean_lightning ;;
     (all)
         time build
         time package_lightning_static
         time package_docs
+        build_wheel "$BUILD_PATH" "$PROJ_NAME"
         ;;
     (build)
         build
@@ -1158,9 +1394,4 @@ case $TARGET in
     (*) die "Invalid target $TARGET" ;;
 esac
 
-if [[ $WHEEL_PACKAGE == true ]]; then
-    echo "Wheel Package build started !!!!"
-    create_wheel_package
-fi
-
 echo "Operation complete"

tools/rocm-build/build_miopen-deps.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 set -ex
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src MIOpen
 
@@ -19,18 +19,18 @@ build_miopen_mlir() {
     mkdir build && cd build
     cmake \
         -G Ninja \
-        -DCMAKE_C_COMPILER="${ROCM_PATH}/llvm/bin/clang" \
-        -DCMAKE_CXX_COMPILER="${ROCM_PATH}/llvm/bin/clang++" \
+        -DCMAKE_C_COMPILER="$(set_build_variables __CLANG__)" \
+        -DCMAKE_CXX_COMPILER="$(set_build_variables __CLANG++__)" \
         -DCMAKE_BUILD_TYPE=Release \
-	-DCMAKE_PREFIX_PATH="${ROCM_PATH};${HOME}/miopen-deps" \
+        -DCMAKE_PREFIX_PATH="${ROCM_PATH};${HOME}/miopen-deps" \
         -DCMAKE_INSTALL_PREFIX="$ROCM_PATH" \
         -DBUILD_FAT_LIBROCKCOMPILER=1 \
-       ..
+        ..
     cmake --build . -- librockCompiler -j${PROC}
     cmake --build . -- install
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-
+    
     echo "Finished building rocMLIR"
 }
 
@@ -43,12 +43,6 @@ build_miopen_deps() {
     echo "Start build"
     cd "$COMPONENT_SRC"
 
-    # Update rocm-recipes for boost link
-    # https://github.com/ROCm/MIOpen/pull/3457/files
-    sed -i 's/329203d79f9fe77ae5d0d742af0966bc57f4dfc8/92c6695449c85887962f45509b376f2eb0d284f7/g' \
-        rbuild.ini \
-        install_deps.cmake \
-        dev-requirements.txt
     # Commenting the rocMLIR & composable_kernel from requirements.txt
     sed -i '/ROCm\/rocMLIR@\|ROCm\/composable_kernel@/s/^/#/' requirements.txt
     # Extract MLIR commit from requirements.txt
@@ -56,7 +50,7 @@ build_miopen_deps() {
 
 
     pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
-    PATH="${PATH}:${ROCM_PATH}:${HOME}/.local/bin" rbuild prepare -d "$HOME/miopen-deps" --cxx=${ROCM_PATH}/llvm/bin/clang++ --cc ${ROCM_PATH}/llvm/bin/clang
+    PATH="${PATH}:${ROCM_PATH}:${HOME}/.local/bin" rbuild prepare -d "$HOME/miopen-deps" --cxx="$(set_build_variables __CLANG++__)" --cc "$(set_build_variables __CLANG__)"
     build_miopen_mlir "$MLIR_COMMIT"
 
     show_build_cache_stats

tools/rocm-build/build_miopen-hip.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 set -ex
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src MIOpen
 
@@ -9,6 +9,8 @@ PACKAGE_DIR=${PACKAGE_DIR%\/*}/miopen-hip
 DEB_PATH=$PACKAGE_DIR
 RPM_PATH=$PACKAGE_DIR
 
+disable_debug_package_generation
+
 build_miopen_hip() {
     echo "Start build"
 
@@ -31,19 +33,19 @@ build_miopen_hip() {
         "${rocm_math_common_cmake_params[@]}" \
         -DMIOPEN_BACKEND=HIP \
         -DMIOPEN_OFFLINE_COMPILER_PATHS_V2=1 \
-        -DCMAKE_CXX_COMPILER="${ROCM_PATH}/llvm/bin/clang++" \
-        -DCMAKE_C_COMPILER="${ROCM_PATH}/llvm/bin/clang" \
+        -DCMAKE_CXX_COMPILER=$(set_build_variables __CLANG++__) \
+        -DCMAKE_C_COMPILER=$(set_build_variables __CLANG__) \
         -DCMAKE_PREFIX_PATH="${ROCM_PATH};${ROCM_PATH}/hip;${HOME}/miopen-deps" \
         -DHIP_OC_COMPILER="${ROCM_PATH}/bin/clang-ocl" \
         -DMIOPEN_TEST_DISCRETE=OFF \
-        "$COMPONENT_SRC"
+	"$COMPONENT_SRC"
 
     cmake --build "$BUILD_DIR" -- -j${PROC}
     cmake --build "$BUILD_DIR" -- install
     cmake --build "$BUILD_DIR" -- package
 
     rm -rf $BUILD_DIR/_CPack_Packages/ && find $BUILD_DIR -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -62,7 +64,7 @@ checkout_lfs() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_miopen_hip ;;
+    build) build_miopen_hip; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_miopen_hip ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_mivisionx.sh

@@ -1,39 +1,44 @@
 #!/bin/bash
 
 set -ex
-source "$(dirname "${BASH_SOURCE[0]}")/compute_helper.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/compute_utils.sh"
 
 set_component_src MIVisionX
 BUILD_DEV=ON
 
 build_mivisionx() {
+
+    if [ "$DISTRO_ID" = "mariner-2.0" ] || [ "$DISTRO_ID" = "azurelinux-3.0" ] ; then
+        echo "Not building mivisionx for ${DISTRO_ID}. Exiting..."
+        return 0
+    fi
+
     echo "Start build"
 
     if [ "${ENABLE_STATIC_BUILDS}" == "true" ]; then
         ack_and_skip_static
     fi
 
+    CXX=$(set_build_variables __AMD_CLANG_++__)
     mkdir -p $BUILD_DIR && cd $BUILD_DIR
     if [ "${ENABLE_ADDRESS_SANITIZER}" == "true" ]; then
        set_asan_env_vars
        set_address_sanitizer_on
+       # Setting BUILD_DEV to OFF. This will prevent the installation of
+       # header files, other files in share,libexec folder. ASAN pkg doesn't need this
        BUILD_DEV=OFF
     fi
 
-    init_rocm_common_cmake_params
+    echo "C compiler: $CC"
+    echo "CXX compiler: $CXX"
 
-    if [ -n "$GPU_ARCHS" ]; then
-        GPU_TARGETS="$GPU_ARCHS"
-    else
-        GPU_TARGETS="gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201"
-    fi
+    init_rocm_common_cmake_params
 
     cmake \
         "${rocm_math_common_cmake_params[@]}" \
         -DROCM_PATH="$ROCM_PATH" \
         -DBUILD_DEV=$BUILD_DEV \
         -DCMAKE_INSTALL_LIBDIR=$(getInstallLibDir) \
-        -DAMDGPU_TARGETS=${GPU_TARGETS} \
         -DROCM_DEP_ROCMCORE=ON \
         -DROCAL_PYTHON=OFF \
         ${LAUNCHER_FLAGS} \
@@ -44,7 +49,7 @@ build_mivisionx() {
     cpack -G ${PKGTYPE^^}
 
     rm -rf _CPack_Packages/ && find -name '*.o' -delete
-    mkdir -p $PACKAGE_DIR && cp ${BUILD_DIR}/*.${PKGTYPE} $PACKAGE_DIR
+    copy_if "${PKGTYPE}" "${CPACKGEN:-"DEB;RPM"}" "${PACKAGE_DIR}" "${BUILD_DIR}"/*."${PKGTYPE}"
 
     show_build_cache_stats
 }
@@ -58,7 +63,7 @@ clean_mivisionx() {
 stage2_command_args "$@"
 
 case $TARGET in
-    build) build_mivisionx ;;
+    build) build_mivisionx; build_wheel ;;
     outdir) print_output_directory ;;
     clean) clean_mivisionx ;;
     *) die "Invalid target $TARGET" ;;

tools/rocm-build/build_omniperf.sh

@@ -26,6 +26,7 @@ printUsage() {
     return 0
 }
 
+## Build environment variables
 API_NAME="omniperf"
 PROJ_NAME="$API_NAME"
 LIB_NAME="lib${API_NAME}"
@@ -33,18 +34,16 @@ TARGET="build"
 MAKETARGET="deb"
 PACKAGE_ROOT="$(getPackageRoot)"
 PACKAGE_LIB="$(getLibPath)"
+# PACKAGE_INCLUDE="$(getIncludePath)"
 BUILD_DIR="$(getBuildPath $API_NAME)"
 PACKAGE_DEB="$(getPackageRoot)/deb/$API_NAME"
 PACKAGE_RPM="$(getPackageRoot)/rpm/$API_NAME"
-ROCM_WHEEL_DIR="${BUILD_DIR}/_wheel"
 BUILD_TYPE="Debug"
 MAKE_OPTS="$DASH_JAY -C $BUILD_DIR"
 SHARED_LIBS="ON"
 CLEAN_OR_OUT=0;
 MAKETARGET="deb"
 PKGTYPE="deb"
-WHEEL_PACKAGE=false
-
 
 #parse the arguments
 VALID_STR=$(getopt -o hcraso:p:w --long help,clean,release,static,address_sanitizer,outdir:,package:,wheel -- "$@")
@@ -52,6 +51,7 @@ eval set -- "$VALID_STR"
 
 while true ;
 do
+    #echo "parocessing $1"
     case "$1" in
         -h | --help)
                 printUsage ; exit 0;;
@@ -86,7 +86,6 @@ fi
 
 clean() {
     echo "Cleaning $PROJ_NAME"
-    rm -rf "$ROCM_WHEEL_DIR"
     rm -rf "$BUILD_DIR"
     rm -rf "$PACKAGE_DEB"
     rm -rf "$PACKAGE_RPM"
@@ -100,7 +99,6 @@ build() {
         echo "Skip make and uploading packages for Omniperf on Centos7 distro, due to python dependency"
         exit 0
     fi
-
     if [ ! -d "$BUILD_DIR" ]; then
         mkdir -p "$BUILD_DIR"
         pushd "$BUILD_DIR" || exit
@@ -108,6 +106,8 @@ build() {
         echo "ROCm CMake Params: $(rocm_cmake_params)"
         echo "ROCm Common CMake Params: $(rocm_common_cmake_params)"
 
+        #install python deps
+        #python3 -m pip install -t ${BUILD_DIR}/python-libs -r ${OMNIPERF_ROOT}/requirements.txt
         print_lib_type $SHARED_LIBS
         cmake \
             $(rocm_cmake_params) \
@@ -117,7 +117,6 @@ build() {
             -DMOD_INSTALL_PATH=${BUILD_DIR}/modulefiles \
             "$OMNIPERF_ROOT"
     fi
-
     make $MAKE_OPTS
     make $MAKE_OPTS install
     make $MAKE_OPTS package
@@ -126,22 +125,6 @@ build() {
     copy_if RPM "${CPACKGEN:-"DEB;RPM"}" "$PACKAGE_RPM" "$BUILD_DIR/${API_NAME}"*.rpm
 }
 
-create_wheel_package() {
-    echo "Creating Omniperf wheel package"
-
-    # Copy the setup.py generator to build folder
-    mkdir -p "$ROCM_WHEEL_DIR"
-    cp -f "$SCRIPT_ROOT"/generate_setup_py.py "$ROCM_WHEEL_DIR"
-    cp -f "$SCRIPT_ROOT"/repackage_wheel.sh "$ROCM_WHEEL_DIR"
-    cd "$ROCM_WHEEL_DIR" || exit
-
-    # Currently only supports python3.6
-    ./repackage_wheel.sh "$BUILD_DIR"/*.rpm python3.6
-
-    # Copy the wheel created to RPM folder which will be uploaded to artifactory
-    copy_if WHL "WHL" "$PACKAGE_RPM" "$ROCM_WHEEL_DIR"/dist/*.whl
-}
-
 print_output_directory() {
     case ${PKGTYPE} in
         ("deb")
@@ -158,14 +141,9 @@ verifyEnvSetup
 
 case "$TARGET" in
     (clean) clean ;;
-    (build) build ;;
+    (build) build; build_wheel "$BUILD_DIR" "$PROJ_NAME" ;;
     (outdir) print_output_directory ;;
     (*) die "Invalid target $TARGET" ;;
 esac
 
-if [[ $WHEEL_PACKAGE == true ]]; then
-    echo "Wheel Package build started !!!!"
-    create_wheel_package
-fi
-
-echo "Operation complete"
+echo "Operation complete"

tools/rocm-build/build_omnitrace.sh

@@ -26,6 +26,7 @@ printUsage() {
     return 0
 }
 
+## Build environment variables
 API_NAME="omnitrace"
 PROJ_NAME="$API_NAME"
 LIB_NAME="lib${API_NAME}"
@@ -33,9 +34,11 @@ TARGET="build"
 MAKETARGET="deb"
 PACKAGE_ROOT="$(getPackageRoot)"
 PACKAGE_LIB="$(getLibPath)"
+# PACKAGE_INCLUDE="$(getIncludePath)"
 BUILD_DIR="$(getBuildPath $API_NAME)"
 PACKAGE_DEB="$(getPackageRoot)/deb/$API_NAME"
 PACKAGE_RPM="$(getPackageRoot)/rpm/$API_NAME"
+# PACKAGE_PREFIX="$ROCM_INSTALL_PATH"
 BUILD_TYPE="Debug"
 MAKE_OPTS="-j 8"
 SHARED_LIBS="ON"
@@ -49,6 +52,7 @@ VALID_STR=$(getopt -o hcraso:p:w --long help,clean,release,address_sanitizer,sta
 eval set -- "$VALID_STR"
 
 while true; do
+    #echo "parocessing $1"
     case "$1" in
     -h | --help)
         printUsage
@@ -65,6 +69,8 @@ while true; do
         ;;
     -a | --address_sanitizer)
         ack_and_ignore_asan
+        # set_asan_env_vars
+        # set_address_sanitizer_on
 
         ASAN=1
         shift
@@ -76,6 +82,7 @@ while true; do
     -o | --outdir)
         TARGET="outdir"
         PKGTYPE=$2
+        # OUT_DIR_SPECIFIED=1
         ((CLEAN_OR_OUT |= 2))
         shift 2
         ;;
@@ -84,13 +91,13 @@ while true; do
         shift 2
         ;;
     -w | --wheel)
-	   echo "omnitrace: wheel build option accepted and ignored"
+	   WHEEL_PACKAGE=true
        shift
        ;;
     --)
         shift
         break
-        ;;
+        ;; # end delimiter
     *)
         echo " This should never come but just incase : UNEXPECTED ERROR Parm : [$1] " >&2
         exit 20
@@ -131,23 +138,68 @@ build_omnitrace() {
         echo "Created build directory: $BUILD_DIR"
     fi
 
+    cd $OMNITRACE_ROOT || exit
+
+    echo "Current submodule status"
+    git submodule status
+    echo "Cached (old) submodule status"
+    git submodule status --cached
+    cat .git/config
+
+    echo "Updating submodules"
+    git submodule init
+    # copy the new URL to your local config
+    git submodule sync --recursive
+    # force update the submodule from the new URL
+    git submodule update --init --recursive --force
+
+    echo "Updated submodule status"
+    git submodule status
+    cat .git/config
+
     echo "Build directory: $BUILD_DIR"
     pushd "$BUILD_DIR" || exit
     print_lib_type $SHARED_LIBS
 
+    ELFUTIL_URL="https://compute-artifactory.amd.com/artifactory/rocm-generic-local/dev-tools/omnitrace/elfutils-0.188.tar.bz2"
+    BINUTIL_URL="https://compute-artifactory.amd.com/artifactory/rocm-generic-local/dev-tools/omnitrace/binutils-2.40.tar.gz"
+
     echo "ROCm CMake Params: $(rocm_cmake_params)"
     echo "ROCm Common CMake Params: $(rocm_common_cmake_params)"
-
+    echo "ELFUTIL_URL=$ELFUTIL_URL, BINUTIL_URL=$BINUTIL_URL"
 
     if [ $ASAN == 1 ]; then
         echo "Address Sanitizer path"
 
+        # Commenting out the below cmake command as it is not working as expected
+        # LD_LIBRARY_PATH=$ROCM_INSTALL_PATH/lib/asan:$LD_LIBRARY_PATH
+        # cmake \
+        #     $(rocm_cmake_params) \
+        #     $(rocm_common_cmake_params) \
+        #     -DOMNITRACE_BUILD_{LIBUNWIND,DYNINST}=ON \
+        #     -DDYNINST_BUILD_{TBB,BOOST,ELFUTILS,LIBIBERTY}=ON \
+        #     -DAMDDeviceLibs_DIR="${ROCM_INSTALL_PATH}/lib/asan/cmake/AMDDeviceLibs" \
+        #     -Dhip_DIR="${ROCM_INSTALL_PATH}/lib/asan/cmake/hip" \
+        #     -Dhip-lang_DIR="${ROCM_INSTALL_PATH}/lib/asan/cmake/hip-lang" \
+        #     -Damd_comgr_DIR="${ROCM_INSTALL_PATH}/lib/asan/cmake/amd_comgr" \
+        #     -Dhsa-runtime64_DIR="${ROCM_INSTALL_PATH}/lib/asan/cmake/hsa-runtime64" \
+        #     -Dhsakmt_DIR="${ROCM_INSTALL_PATH}/lib/asan/cmake/hsakmt" \
+        #     -DROCM_PATH="${ROCM_INSTALL_PATH}/lib/asan" \
+        #     -Drocprofiler_ROOT_DIR="${ROCM_INSTALL_PATH}/lib/asan" \
+        #     -DCMAKE_HIP_COMPILER_ROCM_ROOT="${ROCM_INSTALL_PATH}" \
+        #     -DCMAKE_PREFIX_PATH="${ROCM_INSTALL_PATH};${ROCM_INSTALL_PATH}/lib/asan" \
+        #     -DCMAKE_LIBRARY_PATH="${ROCM_INSTALL_PATH}/lib/asan" \
+        #     -DCPACK_DEBIAN_PACKAGE_SHLIBDEPS=OFF \
+        #     "$OMNITRACE_ROOT"
+
     else
         cmake \
             $(rocm_cmake_params) \
             $(rocm_common_cmake_params) \
             -DOMNITRACE_BUILD_{LIBUNWIND,DYNINST}=ON \
             -DDYNINST_BUILD_{TBB,BOOST,ELFUTILS,LIBIBERTY}=ON \
+            -DElfUtils_DOWNLOAD_URL="$ELFUTIL_URL" \
+            -D{DYNINST,TIMEMORY}_BINUTILS_DOWNLOAD_URL="$BINUTIL_URL" \
             "$OMNITRACE_ROOT"
     fi
 
@@ -182,10 +234,10 @@ print_output_directory() {
 verifyEnvSetup
 
 case "$TARGET" in
-clean) clean ;;
-build) build_omnitrace ;;
-outdir) print_output_directory ;;
-*) die "Invalid target $TARGET" ;;
+    clean) clean ;;
+    build) build_omnitrace; build_wheel "$BUILD_DIR" "$PROJ_NAME" ;;
+    outdir) print_output_directory ;;
+    *) die "Invalid target $TARGET" ;;
 esac
 
 echo "Operation complete"

tools/rocm-build/build_opencl_cts.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+source "$(dirname "${BASH_SOURCE}")/compute_utils.sh"
+PROJ_NAME=opencl-cts
+TARGET="build"
+MAKEOPTS="$DASH_JAY"
+BUILD_TYPE="Debug"
+OPENCL_ICD_LOADER_BUILD_DIR="$(getBuildPath)/OpenCL-ICD-Loader"
+OPENCL_CTS_BUILD_DIR="$(getBuildPath)/OpenCL-CTS"
+PACKAGE_ROOT="$(getPackageRoot)"
+PACKAGE_DEB="$PACKAGE_ROOT/deb/$PROJ_NAME"
+PACKAGE_RPM="$PACKAGE_ROOT/rpm/$PROJ_NAME"
+CLEAN_OR_OUT=0;
+PKGTYPE="deb"
+MAKETARGET="deb"
+
+printUsage() {
+    echo
+    echo "Usage: $(basename "${BASH_SOURCE}") [options ...]"
+    echo
+    echo "Options:"
+    echo "  -c,  --clean              Clean output and delete all intermediate work"
+    echo "  -p,  --package <type>     Specify packaging format"
+    echo "  -r,  --release            Make a release build instead of a debug build"
+    echo "  -w,  --wheel              Creates python wheel package of opencl-cts. 
+                                      It needs to be used along with -r option"
+    echo "  -h,  --help               Prints this help"
+    echo "  -o,  --outdir             Print path of output directory containing packages"
+    echo "  -s,  --static             Component/Build does not support static builds just accepting this param & ignore. No effect of the param on this build"
+    echo
+    echo "Possible values for <type>:"
+    echo "  deb -> Debian format (default)"
+    echo "  rpm -> RPM format"
+    echo
+    return 0
+}
+
+RET_CONFLICT=1
+check_conflicting_options $CLEAN_OR_OUT $PKGTYPE $MAKETARGET
+if [ $RET_CONFLICT -ge 30 ]; then
+   print_vars $TARGET $BUILD_TYPE $CLEAN_OR_OUT $PKGTYPE $MAKETARGET
+   exit $RET_CONFLICT
+fi
+
+clean_opencl_cts() {
+    echo "Cleaning $PROJ_NAME"
+    rm -rf "$OPENCL_CTS_BUILD_DIR"
+    rm -rf "$PACKAGE_DEB"
+    rm -rf "$PACKAGE_RPM"
+    rm -rf "$PACKAGE_ROOT/opencl-cts"
+}
+
+build_opencl_cts() {
+    echo "Building $PROJ_NAME"
+    mkdir -p "$OPENCL_CTS_BUILD_DIR"
+    pushd "$OPENCL_CTS_BUILD_DIR"
+    if [ ! -e Makefile ]; then
+        cmake \
+            -S "$OPENCL_CTS_ROOT" \
+            $(rocm_cmake_params) \
+            $(rocm_common_cmake_params) \
+            -DCL_INCLUDE_DIR="$OPENCL_HEADERS_ROOT" \
+            -DCL_LIB_DIR="$OPENCL_ICD_LOADER_BUILD_DIR" \
+            -DOPENCL_LIBRARIES="-lOpenCL -lpthread" \
+            -DGL_IS_SUPPORTED="ON"
+    fi
+    cmake --build . -- $MAKEOPTS
+    popd
+}
+
+package_opencl_cts() {
+    echo "Packaging $PROJ_NAME"
+    pushd "$OPENCL_CTS_BUILD_DIR"
+    cmake --build . -- package
+    mkdir -p $PACKAGE_DEB
+    mkdir -p $PACKAGE_RPM
+    copy_if DEB "${CPACKGEN:-"DEB;RPM"}" "$PACKAGE_DEB" *.deb
+    copy_if RPM "${CPACKGEN:-"DEB;RPM"}" "$PACKAGE_RPM" *.rpm
+    popd
+}
+
+print_output_directory() {
+    case ${PKGTYPE} in
+        ("deb")
+            echo ${PACKAGE_DEB};;
+        ("rpm")
+            echo ${PACKAGE_RPM};;
+        (*)
+            echo "Invalid package type \"${PKGTYPE}\" provided for -o" >&2; exit 1;;
+    esac
+    exit
+}
+
+#parse the arguments
+VALID_STR=`getopt -o hcraswlo:p: --long help,clean,release,static,outdir:,package: -- "$@"`
+eval set -- "$VALID_STR"
+while true ;
+do
+    case "$1" in
+        (-c  | --clean )
+            TARGET="clean" ; ((CLEAN_OR_OUT|=1)) ; shift ;;
+        (-r  | --release )
+            BUILD_TYPE="RelWithDebInfo" ; shift ;;
+        (-h  | --help )
+            printUsage ; exit 0 ;;
+        (-a  | --address_sanitizer)
+            ack_and_ignore_asan ; shift ;;
+        (-w | --wheel)
+                WHEEL_PACKAGE=true ; shift ;;
+        (-o  | --outdir)
+            TARGET="outdir"; PKGTYPE=$2 ; OUT_DIR_SPECIFIED=1 ; ((CLEAN_OR_OUT|=2)) ; shift 2 ;;
+        (-p | --package)
+            MAKETARGET="$2" ; shift 2;;
+        (-s | --static)
+            echo "-s parameter accepted but ignored" ; shift ;;
+        --)     shift; break;; # end delimiter
+        (*)
+            echo " This should never come but just incase : UNEXPECTED ERROR Parm : [$1] ">&2 ; exit 20;;
+    esac
+done
+
+case $TARGET in
+    (clean) clean_opencl_cts ;;
+    (build) build_opencl_cts ; package_opencl_cts; build_wheel "$OPENCL_ICD_LOADER_BUILD_DIR" "$PROJ_NAME";;
+    (outdir) print_output_directory ;;
+    (*) die "Invalid target $TARGET" ;;
+esac
+
+echo "Operation complete"