github/ROCm
1
1
Fork 0
mirror of https://github.com/ROCm/ROCm.git synced 2026-01-08 22:28:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

[External CI] rccl triggers rocprofiler-sdk downstream (#5420)

Browse Source 
- Update rccl component pipeline to include new additions made to projects already in super repos.
- Also update rccl to trigger rocproifler-sdk job upon completion.
- rocprofiler-sdk pipeline updated to include os parameter to enable future almalinux 8 job.
This commit is contained in:
Joseph Macaranas
2025-10-27 12:14:30 -04:00
committed by GitHub
parent 4132a2609c
commit 248cbf8bc1
2 changed files with 152 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
.azuredevops/components/rccl.yml
View File

@@ -1,10 +1,35 @@
parameters: parameters:
- name: componentName
type: string
default: rccl
- name: checkoutRepo - name: checkoutRepo
type: string type: string
default: 'self' default: 'self'
- name: checkoutRef - name: checkoutRef
type: string type: string
default: '' default: ''
- name: systemsRepo
type: string
default: systems_repo
- name: systemsSparseCheckoutDir
type: string
default: 'projects/rocprofiler-sdk'
# monorepo related parameters
- name: sparseCheckoutDir
type: string
default: ''
- name: triggerDownstreamJobs
type: boolean
default: false
- name: downstreamAggregateNames
type: string
default: ''
- name: buildDependsOn
type: object
default: null
- name: unifiedBuild
type: boolean
default: false
# set to true if doing full build of ROCm stack # set to true if doing full build of ROCm stack
# and dependencies are pulled from same pipeline # and dependencies are pulled from same pipeline
- name: aggregatePipeline - name: aggregatePipeline
@@ -57,19 +82,28 @@ parameters:
type: object type: object
default: default:
buildJobs: buildJobs:
- gfx942: - { os: ubuntu2204, packageManager: apt, target: gfx942 }
target: gfx942 - { os: ubuntu2204, packageManager: apt, target: gfx90a }
- gfx90a:
target: gfx90a
testJobs: testJobs:
- gfx942: - { os: ubuntu2204, packageManager: apt, target: gfx942 }
target: gfx942 - { os: ubuntu2204, packageManager: apt, target: gfx90a }
- gfx90a: - name: downstreamComponentMatrix
target: gfx90a type: object
default:
- rocprofiler-sdk:
name: rocprofiler-sdk
sparseCheckoutDir: ''
skipUnifiedBuild: 'false'
buildDependsOn:
- rccl_build
jobs: jobs:
- ${{ each job in parameters.jobMatrix.buildJobs }}: - ${{ each job in parameters.jobMatrix.buildJobs }}:
- job: rccl_build_${{ job.target }} - job: ${{ parameters.componentName }}_build_${{ job.os }}_${{ job.target }}
${{ if parameters.buildDependsOn }}:
dependsOn:
- ${{ each build in parameters.buildDependsOn }}:
- ${{ build }}_${{ job.os }}_${{ job.target }}
timeoutInMinutes: 120 timeoutInMinutes: 120
variables: variables:
- group: common - group: common
@@ -77,17 +111,23 @@ jobs:
- name: HIP_ROCCLR_HOME - name: HIP_ROCCLR_HOME
value: $(Build.BinariesDirectory)/rocm value: $(Build.BinariesDirectory)/rocm
pool: ${{ variables.MEDIUM_BUILD_POOL }} pool: ${{ variables.MEDIUM_BUILD_POOL }}
${{ if eq(job.os, 'almalinux8') }}:
container:
image: rocmexternalcicd.azurecr.io/manylinux228:latest
endpoint: ContainerService3
workspace: workspace:
clean: all clean: all
steps: steps:
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
parameters: parameters:
aptPackages: ${{ parameters.aptPackages }} aptPackages: ${{ parameters.aptPackages }}
packageManager: ${{ job.packageManager }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-custom.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-custom.yml
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
parameters: parameters:
checkoutRepo: ${{ parameters.checkoutRepo }} checkoutRepo: ${{ parameters.checkoutRepo }}
sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }}
submoduleBehaviour: recursive submoduleBehaviour: recursive
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-vendor.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-vendor.yml
parameters: parameters:
@@ -97,10 +137,14 @@ jobs:
parameters: parameters:
checkoutRef: ${{ parameters.checkoutRef }} checkoutRef: ${{ parameters.checkoutRef }}
dependencyList: ${{ parameters.rocmDependencies }} dependencyList: ${{ parameters.rocmDependencies }}
os: ${{ job.os }}
gpuTarget: ${{ job.target }} gpuTarget: ${{ job.target }}
aggregatePipeline: ${{ parameters.aggregatePipeline }} aggregatePipeline: ${{ parameters.aggregatePipeline }}
${{ if parameters.triggerDownstreamJobs }}:
downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
parameters: parameters:
os: ${{ job.os }}
extraBuildFlags: >- extraBuildFlags: >-
-DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
-DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc
@@ -112,58 +156,87 @@ jobs:
-GNinja -GNinja
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
parameters: parameters:
componentName: ${{ parameters.componentName }}
sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }}
os: ${{ job.os }}
gpuTarget: ${{ job.target }} gpuTarget: ${{ job.target }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
parameters: parameters:
componentName: ${{ parameters.componentName }}
os: ${{ job.os }}
gpuTarget: ${{ job.target }} gpuTarget: ${{ job.target }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml - ${{ if eq(job.os, 'ubuntu2204') }}:
parameters: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
aptPackages: ${{ parameters.aptPackages }} parameters:
gpuTarget: ${{ job.target }} aptPackages: ${{ parameters.aptPackages }}
extraEnvVars: gpuTarget: ${{ job.target }}
- HIP_ROCCLR_HOME:::/home/user/workspace/rocm extraEnvVars:
installLatestCMake: true - HIP_ROCCLR_HOME:::/home/user/workspace/rocm
installLatestCMake: true
- ${{ each job in parameters.jobMatrix.testJobs }}: - ${{ if eq(parameters.unifiedBuild, False) }}:
- job: rccl_test_${{ job.target }} - ${{ each job in parameters.jobMatrix.testJobs }}:
timeoutInMinutes: 120 - job: ${{ parameters.componentName }}_test_${{ job.os }}_${{ job.target }}
dependsOn: rccl_build_${{ job.target }} timeoutInMinutes: 120
condition: dependsOn: ${{ parameters.componentName }}_build_${{ job.os }}_${{ job.target }}
and(succeeded(), condition:
eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), and(succeeded(),
not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])), eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
eq(${{ parameters.aggregatePipeline }}, False) not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), '${{ parameters.componentName }}')),
) eq(${{ parameters.aggregatePipeline }}, False)
variables: )
- group: common variables:
- template: /.azuredevops/variables-global.yml - group: common
pool: ${{ job.target }}_test_pool - template: /.azuredevops/variables-global.yml
workspace: pool: ${{ job.target }}_test_pool
clean: all workspace:
steps: clean: all
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml steps:
parameters: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml
aptPackages: ${{ parameters.aptPackages }} parameters:
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml aptPackages: ${{ parameters.aptPackages }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
parameters: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml
gpuTarget: ${{ job.target }} parameters:
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml preTargetFilter: ${{ parameters.componentName }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml os: ${{ job.os }}
parameters: gpuTarget: ${{ job.target }}
checkoutRef: ${{ parameters.checkoutRef }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
dependencyList: ${{ parameters.rocmTestDependencies }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
gpuTarget: ${{ job.target }} parameters:
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml checkoutRef: ${{ parameters.checkoutRef }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml dependencyList: ${{ parameters.rocmTestDependencies }}
parameters: os: ${{ job.os }}
componentName: rccl gpuTarget: ${{ job.target }}
testDir: '$(Agent.BuildDirectory)/rocm/bin' ${{ if parameters.triggerDownstreamJobs }}:
testExecutable: './rccl-UnitTests' downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}
testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes' - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
parameters: parameters:
aptPackages: ${{ parameters.aptPackages }} componentName: ${{ parameters.componentName }}
environment: test os: ${{ job.os }}
gpuTarget: ${{ job.target }} testDir: '$(Agent.BuildDirectory)/rocm/bin'
testExecutable: './rccl-UnitTests'
testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes'
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
parameters:
aptPackages: ${{ parameters.aptPackages }}
environment: test
gpuTarget: ${{ job.target }}
- ${{ if parameters.triggerDownstreamJobs }}:
- ${{ each component in parameters.downstreamComponentMatrix }}:
- ${{ if not(and(parameters.unifiedBuild, eq(component.skipUnifiedBuild, 'true'))) }}:
- template: /.azuredevops/components/${{ component.name }}.yml@pipelines_repo
parameters:
checkoutRepo: ${{ parameters.systemsRepo }}
sparseCheckoutDir: ${{ parameters.systemsSparseCheckoutDir }}
triggerDownstreamJobs: true
unifiedBuild: ${{ parameters.unifiedBuild }}
${{ if parameters.unifiedBuild }}:
buildDependsOn: ${{ component.unifiedBuild.buildDependsOn }}
downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ component.unifiedBuild.downstreamAggregateNames }}
${{ else }}:
buildDependsOn: ${{ component.buildDependsOn }}
downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ parameters.componentName }}

34
.azuredevops/components/rocprofiler-sdk.yml
View File

@@ -79,27 +79,27 @@ parameters:
type: object type: object
default: default:
buildJobs: buildJobs:
- gfx942: - { os: ubuntu2204, packageManager: apt, target: gfx942 }
target: gfx942 - { os: ubuntu2204, packageManager: apt, target: gfx90a }
- gfx90a:
target: gfx90a
testJobs: testJobs:
- gfx942: - { os: ubuntu2204, packageManager: apt, target: gfx942 }
target: gfx942 - { os: ubuntu2204, packageManager: apt, target: gfx90a }
- gfx90a:
target: gfx90a
jobs: jobs:
- ${{ each job in parameters.jobMatrix.buildJobs }}: - ${{ each job in parameters.jobMatrix.buildJobs }}:
- job: rocprofiler_sdk_build_${{ job.target }} - job: rocprofiler_sdk_build_${{ job.os }}_${{ job.target }}
${{ if parameters.buildDependsOn }}: ${{ if parameters.buildDependsOn }}:
dependsOn: dependsOn:
- ${{ each build in parameters.buildDependsOn }}: - ${{ each build in parameters.buildDependsOn }}:
- ${{ build }}_${{ job.target }} - ${{ build }}_${{ job.os}}_${{ job.target }}
variables: variables:
- group: common - group: common
- template: /.azuredevops/variables-global.yml - template: /.azuredevops/variables-global.yml
pool: ${{ variables.MEDIUM_BUILD_POOL }} pool: ${{ variables.MEDIUM_BUILD_POOL }}
${{ if eq(job.os, 'almalinux8') }}:
container:
image: rocmexternalcicd.azurecr.io/manylinux228:latest
endpoint: ContainerService3
workspace: workspace:
clean: all clean: all
steps: steps:
@@ -107,6 +107,7 @@ jobs:
parameters: parameters:
aptPackages: ${{ parameters.aptPackages }} aptPackages: ${{ parameters.aptPackages }}
pipModules: ${{ parameters.pipModules }} pipModules: ${{ parameters.pipModules }}
packageManager: ${{ job.packageManager }}
registerROCmPackages: true registerROCmPackages: true
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
@@ -118,6 +119,7 @@ jobs:
parameters: parameters:
checkoutRef: ${{ parameters.checkoutRef }} checkoutRef: ${{ parameters.checkoutRef }}
dependencyList: ${{ parameters.rocmDependencies }} dependencyList: ${{ parameters.rocmDependencies }}
os: ${{ job.os }}
gpuTarget: ${{ job.target }} gpuTarget: ${{ job.target }}
aggregatePipeline: ${{ parameters.aggregatePipeline }} aggregatePipeline: ${{ parameters.aggregatePipeline }}
${{ if parameters.triggerDownstreamJobs }}: ${{ if parameters.triggerDownstreamJobs }}:
@@ -132,6 +134,7 @@ jobs:
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
parameters: parameters:
componentName: ${{ parameters.componentName }} componentName: ${{ parameters.componentName }}
os: ${{ job.os }}
extraBuildFlags: >- extraBuildFlags: >-
-DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-DROCPROFILER_BUILD_TESTS=ON -DROCPROFILER_BUILD_TESTS=ON
@@ -143,6 +146,7 @@ jobs:
parameters: parameters:
componentName: ${{ parameters.componentName }} componentName: ${{ parameters.componentName }}
sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }} sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }}
os: ${{ job.os }}
gpuTarget: ${{ job.target }} gpuTarget: ${{ job.target }}
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml
parameters: parameters:
@@ -158,8 +162,8 @@ jobs:
- ${{ if eq(parameters.unifiedBuild, False) }}: - ${{ if eq(parameters.unifiedBuild, False) }}:
- ${{ each job in parameters.jobMatrix.testJobs }}: - ${{ each job in parameters.jobMatrix.testJobs }}:
- job: rocprofiler_sdk_test_${{ job.target }} - job: rocprofiler_sdk_test_${{ job.os }}_${{ job.target }}
dependsOn: rocprofiler_sdk_build_${{ job.target }} dependsOn: rocprofiler_sdk_build_${{ job.os }}_${{ job.target }}
condition: condition:
and(succeeded(), and(succeeded(),
eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'),
@@ -177,6 +181,7 @@ jobs:
parameters: parameters:
aptPackages: ${{ parameters.aptPackages }} aptPackages: ${{ parameters.aptPackages }}
pipModules: ${{ parameters.pipModules }} pipModules: ${{ parameters.pipModules }}
packageManager: ${{ job.packageManager }}
registerROCmPackages: true registerROCmPackages: true
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
@@ -188,6 +193,7 @@ jobs:
parameters: parameters:
checkoutRef: ${{ parameters.checkoutRef }} checkoutRef: ${{ parameters.checkoutRef }}
dependencyList: ${{ parameters.rocmDependencies }} dependencyList: ${{ parameters.rocmDependencies }}
os: ${{ job.os }}
gpuTarget: ${{ job.target }} gpuTarget: ${{ job.target }}
${{ if parameters.triggerDownstreamJobs }}: ${{ if parameters.triggerDownstreamJobs }}:
downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}
@@ -202,6 +208,7 @@ jobs:
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml
parameters: parameters:
componentName: ${{ parameters.componentName }} componentName: ${{ parameters.componentName }}
os: ${{ job.os }}
extraBuildFlags: >- extraBuildFlags: >-
-DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm
-DROCPROFILER_BUILD_TESTS=ON -DROCPROFILER_BUILD_TESTS=ON
@@ -213,7 +220,8 @@ jobs:
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml
parameters: parameters:
componentName: ${{ parameters.componentName }} componentName: ${{ parameters.componentName }}
testDir: $(Agent.BuildDirectory)/s/build os: ${{ job.os }}
testDir: $(Agent.BuildDirectory)/build
- template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml
parameters: parameters:
aptPackages: ${{ parameters.aptPackages }} aptPackages: ${{ parameters.aptPackages }}