diff --git a/.azuredevops/components/rccl.yml b/.azuredevops/components/rccl.yml index 02af4ed68..b8bfd8c4a 100644 --- a/.azuredevops/components/rccl.yml +++ b/.azuredevops/components/rccl.yml @@ -1,10 +1,35 @@ parameters: +- name: componentName + type: string + default: rccl - name: checkoutRepo type: string default: 'self' - name: checkoutRef type: string default: '' +- name: systemsRepo + type: string + default: systems_repo +- name: systemsSparseCheckoutDir + type: string + default: 'projects/rocprofiler-sdk' +# monorepo related parameters +- name: sparseCheckoutDir + type: string + default: '' +- name: triggerDownstreamJobs + type: boolean + default: false +- name: downstreamAggregateNames + type: string + default: '' +- name: buildDependsOn + type: object + default: null +- name: unifiedBuild + type: boolean + default: false # set to true if doing full build of ROCm stack # and dependencies are pulled from same pipeline - name: aggregatePipeline @@ -57,19 +82,28 @@ parameters: type: object default: buildJobs: - - gfx942: - target: gfx942 - - gfx90a: - target: gfx90a + - { os: ubuntu2204, packageManager: apt, target: gfx942 } + - { os: ubuntu2204, packageManager: apt, target: gfx90a } testJobs: - - gfx942: - target: gfx942 - - gfx90a: - target: gfx90a + - { os: ubuntu2204, packageManager: apt, target: gfx942 } + - { os: ubuntu2204, packageManager: apt, target: gfx90a } +- name: downstreamComponentMatrix + type: object + default: + - rocprofiler-sdk: + name: rocprofiler-sdk + sparseCheckoutDir: '' + skipUnifiedBuild: 'false' + buildDependsOn: + - rccl_build jobs: - ${{ each job in parameters.jobMatrix.buildJobs }}: - - job: rccl_build_${{ job.target }} + - job: ${{ parameters.componentName }}_build_${{ job.os }}_${{ job.target }} + ${{ if parameters.buildDependsOn }}: + dependsOn: + - ${{ each build in parameters.buildDependsOn }}: + - ${{ build }}_${{ job.os }}_${{ job.target }} timeoutInMinutes: 120 variables: - group: common @@ -77,17 +111,23 @@ jobs: - name: HIP_ROCCLR_HOME value: $(Build.BinariesDirectory)/rocm pool: ${{ variables.MEDIUM_BUILD_POOL }} + ${{ if eq(job.os, 'almalinux8') }}: + container: + image: rocmexternalcicd.azurecr.io/manylinux228:latest + endpoint: ContainerService3 workspace: clean: all steps: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml parameters: aptPackages: ${{ parameters.aptPackages }} + packageManager: ${{ job.packageManager }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-custom.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml parameters: checkoutRepo: ${{ parameters.checkoutRepo }} + sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }} submoduleBehaviour: recursive - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-vendor.yml parameters: @@ -97,10 +137,14 @@ jobs: parameters: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmDependencies }} + os: ${{ job.os }} gpuTarget: ${{ job.target }} aggregatePipeline: ${{ parameters.aggregatePipeline }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml parameters: + os: ${{ job.os }} extraBuildFlags: >- -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/bin/hipcc @@ -112,58 +156,87 @@ jobs: -GNinja - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml parameters: + componentName: ${{ parameters.componentName }} + sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }} + os: ${{ job.os }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml parameters: + componentName: ${{ parameters.componentName }} + os: ${{ job.os }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml - parameters: - aptPackages: ${{ parameters.aptPackages }} - gpuTarget: ${{ job.target }} - extraEnvVars: - - HIP_ROCCLR_HOME:::/home/user/workspace/rocm - installLatestCMake: true + - ${{ if eq(job.os, 'ubuntu2204') }}: + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml + parameters: + aptPackages: ${{ parameters.aptPackages }} + gpuTarget: ${{ job.target }} + extraEnvVars: + - HIP_ROCCLR_HOME:::/home/user/workspace/rocm + installLatestCMake: true -- ${{ each job in parameters.jobMatrix.testJobs }}: - - job: rccl_test_${{ job.target }} - timeoutInMinutes: 120 - dependsOn: rccl_build_${{ job.target }} - condition: - and(succeeded(), - eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), - not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])), - eq(${{ parameters.aggregatePipeline }}, False) - ) - variables: - - group: common - - template: /.azuredevops/variables-global.yml - pool: ${{ job.target }}_test_pool - workspace: - clean: all - steps: - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml - parameters: - aptPackages: ${{ parameters.aptPackages }} - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml - parameters: - gpuTarget: ${{ job.target }} - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml - parameters: - checkoutRef: ${{ parameters.checkoutRef }} - dependencyList: ${{ parameters.rocmTestDependencies }} - gpuTarget: ${{ job.target }} - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml - parameters: - componentName: rccl - testDir: '$(Agent.BuildDirectory)/rocm/bin' - testExecutable: './rccl-UnitTests' - testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes' - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml - parameters: - aptPackages: ${{ parameters.aptPackages }} - environment: test - gpuTarget: ${{ job.target }} +- ${{ if eq(parameters.unifiedBuild, False) }}: + - ${{ each job in parameters.jobMatrix.testJobs }}: + - job: ${{ parameters.componentName }}_test_${{ job.os }}_${{ job.target }} + timeoutInMinutes: 120 + dependsOn: ${{ parameters.componentName }}_build_${{ job.os }}_${{ job.target }} + condition: + and(succeeded(), + eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), + not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), '${{ parameters.componentName }}')), + eq(${{ parameters.aggregatePipeline }}, False) + ) + variables: + - group: common + - template: /.azuredevops/variables-global.yml + pool: ${{ job.target }}_test_pool + workspace: + clean: all + steps: + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-other.yml + parameters: + aptPackages: ${{ parameters.aptPackages }} + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml + parameters: + preTargetFilter: ${{ parameters.componentName }} + os: ${{ job.os }} + gpuTarget: ${{ job.target }} + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml + parameters: + checkoutRef: ${{ parameters.checkoutRef }} + dependencyList: ${{ parameters.rocmTestDependencies }} + os: ${{ job.os }} + gpuTarget: ${{ job.target }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml + parameters: + componentName: ${{ parameters.componentName }} + os: ${{ job.os }} + testDir: '$(Agent.BuildDirectory)/rocm/bin' + testExecutable: './rccl-UnitTests' + testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes' + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml + parameters: + aptPackages: ${{ parameters.aptPackages }} + environment: test + gpuTarget: ${{ job.target }} + +- ${{ if parameters.triggerDownstreamJobs }}: + - ${{ each component in parameters.downstreamComponentMatrix }}: + - ${{ if not(and(parameters.unifiedBuild, eq(component.skipUnifiedBuild, 'true'))) }}: + - template: /.azuredevops/components/${{ component.name }}.yml@pipelines_repo + parameters: + checkoutRepo: ${{ parameters.systemsRepo }} + sparseCheckoutDir: ${{ parameters.systemsSparseCheckoutDir }} + triggerDownstreamJobs: true + unifiedBuild: ${{ parameters.unifiedBuild }} + ${{ if parameters.unifiedBuild }}: + buildDependsOn: ${{ component.unifiedBuild.buildDependsOn }} + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ component.unifiedBuild.downstreamAggregateNames }} + ${{ else }}: + buildDependsOn: ${{ component.buildDependsOn }} + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ parameters.componentName }} diff --git a/.azuredevops/components/rocprofiler-sdk.yml b/.azuredevops/components/rocprofiler-sdk.yml index 596400efb..b551f46fb 100644 --- a/.azuredevops/components/rocprofiler-sdk.yml +++ b/.azuredevops/components/rocprofiler-sdk.yml @@ -79,27 +79,27 @@ parameters: type: object default: buildJobs: - - gfx942: - target: gfx942 - - gfx90a: - target: gfx90a + - { os: ubuntu2204, packageManager: apt, target: gfx942 } + - { os: ubuntu2204, packageManager: apt, target: gfx90a } testJobs: - - gfx942: - target: gfx942 - - gfx90a: - target: gfx90a + - { os: ubuntu2204, packageManager: apt, target: gfx942 } + - { os: ubuntu2204, packageManager: apt, target: gfx90a } jobs: - ${{ each job in parameters.jobMatrix.buildJobs }}: - - job: rocprofiler_sdk_build_${{ job.target }} + - job: rocprofiler_sdk_build_${{ job.os }}_${{ job.target }} ${{ if parameters.buildDependsOn }}: dependsOn: - ${{ each build in parameters.buildDependsOn }}: - - ${{ build }}_${{ job.target }} + - ${{ build }}_${{ job.os}}_${{ job.target }} variables: - group: common - template: /.azuredevops/variables-global.yml pool: ${{ variables.MEDIUM_BUILD_POOL }} + ${{ if eq(job.os, 'almalinux8') }}: + container: + image: rocmexternalcicd.azurecr.io/manylinux228:latest + endpoint: ContainerService3 workspace: clean: all steps: @@ -107,6 +107,7 @@ jobs: parameters: aptPackages: ${{ parameters.aptPackages }} pipModules: ${{ parameters.pipModules }} + packageManager: ${{ job.packageManager }} registerROCmPackages: true - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml @@ -118,6 +119,7 @@ jobs: parameters: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmDependencies }} + os: ${{ job.os }} gpuTarget: ${{ job.target }} aggregatePipeline: ${{ parameters.aggregatePipeline }} ${{ if parameters.triggerDownstreamJobs }}: @@ -132,6 +134,7 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml parameters: componentName: ${{ parameters.componentName }} + os: ${{ job.os }} extraBuildFlags: >- -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm -DROCPROFILER_BUILD_TESTS=ON @@ -143,6 +146,7 @@ jobs: parameters: componentName: ${{ parameters.componentName }} sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }} + os: ${{ job.os }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml parameters: @@ -158,8 +162,8 @@ jobs: - ${{ if eq(parameters.unifiedBuild, False) }}: - ${{ each job in parameters.jobMatrix.testJobs }}: - - job: rocprofiler_sdk_test_${{ job.target }} - dependsOn: rocprofiler_sdk_build_${{ job.target }} + - job: rocprofiler_sdk_test_${{ job.os }}_${{ job.target }} + dependsOn: rocprofiler_sdk_build_${{ job.os }}_${{ job.target }} condition: and(succeeded(), eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), @@ -177,6 +181,7 @@ jobs: parameters: aptPackages: ${{ parameters.aptPackages }} pipModules: ${{ parameters.pipModules }} + packageManager: ${{ job.packageManager }} registerROCmPackages: true - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml @@ -188,6 +193,7 @@ jobs: parameters: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmDependencies }} + os: ${{ job.os }} gpuTarget: ${{ job.target }} ${{ if parameters.triggerDownstreamJobs }}: downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} @@ -202,6 +208,7 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml parameters: componentName: ${{ parameters.componentName }} + os: ${{ job.os }} extraBuildFlags: >- -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm -DROCPROFILER_BUILD_TESTS=ON @@ -213,7 +220,8 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml parameters: componentName: ${{ parameters.componentName }} - testDir: $(Agent.BuildDirectory)/s/build + os: ${{ job.os }} + testDir: $(Agent.BuildDirectory)/build - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml parameters: aptPackages: ${{ parameters.aptPackages }}