From a62560342eea248e10633a9b4a7aff675423d33e Mon Sep 17 00:00:00 2001 From: Daniel Su Date: Thu, 29 May 2025 10:07:22 -0400 Subject: [PATCH] Ex CI: unified mathlibs build (up to rocSOLVER) --- .azuredevops/ci-builds/mathlibs-trigger.yml | 30 ++++++++ .azuredevops/ci-builds/mathlibs.yml | 38 ++++++++++ .azuredevops/components/hipBLAS-common.yml | 51 +++++++++++++ .azuredevops/components/hipBLASLt.yml | 66 +++++++++++++++-- .azuredevops/components/hipCUB.yml | 8 +- .azuredevops/components/hipRAND.yml | 8 +- .azuredevops/components/rocBLAS.yml | 82 +++++++++++++++++++-- .azuredevops/components/rocPRIM.yml | 36 +++++++-- .azuredevops/components/rocRAND.yml | 24 ++++-- .azuredevops/components/rocSOLVER.yml | 42 +++++++++-- .azuredevops/components/rocThrust.yml | 8 +- 11 files changed, 358 insertions(+), 35 deletions(-) create mode 100644 .azuredevops/ci-builds/mathlibs-trigger.yml create mode 100644 .azuredevops/ci-builds/mathlibs.yml diff --git a/.azuredevops/ci-builds/mathlibs-trigger.yml b/.azuredevops/ci-builds/mathlibs-trigger.yml new file mode 100644 index 000000000..41e4e018f --- /dev/null +++ b/.azuredevops/ci-builds/mathlibs-trigger.yml @@ -0,0 +1,30 @@ +variables: +- group: common +- template: /.azuredevops/variables-global.yml@pipelines_repo + +parameters: +- name: pipelinesRepoRef + type: string + default: refs/heads/develop +- name: librariesRepoRef + type: string + default: refs/heads/develop + +resources: + repositories: + - repository: pipelines_repo + type: github + endpoint: ROCm + name: ROCm/ROCm + ref: ${{ parameters.pipelinesRepoRef }} + - repository: libraries_repo + type: github + endpoint: ROCm + name: ROCm/rocm-libraries + ref: ${{ parameters.librariesRepoRef }} + +jobs: + - template: /.azuredevops/nightly/mathlibs.yml@pipelines_repo + parameters: + checkoutRepo: libraries_repo + buildDependsOn: false diff --git a/.azuredevops/ci-builds/mathlibs.yml b/.azuredevops/ci-builds/mathlibs.yml new file mode 100644 index 000000000..6e75345f6 --- /dev/null +++ b/.azuredevops/ci-builds/mathlibs.yml @@ -0,0 +1,38 @@ +# entrypoint for kicking off a unified build of the mathlibs +# this template is designed to be called by another pipeline (llvm, clr, etc.) +# `buildDependsOn` will need to be set when calling this template +# passes a `unifiedBuild` param to downstream pipelines, which will prevent duplicate jobs +# logic needs to be added in individual mathlib pipelines for handling `unifiedBuild` + +parameters: +- name: checkoutRepo + type: string + default: monorepo +- name: buildDependsOn + type: object + default: false +- name: downstreamComponentMatrix + type: object + default: + - rocRAND: + name: rocRAND + sparseCheckoutDir: projects/rocrand + - rocPRIM: + name: rocPRIM + sparseCheckoutDir: projects/rocprim + - hipBLAS-common: + name: hipBLAS-common + sparseCheckoutDir: projects/hipblas-common + # - composable_kernel: + # name: composable_kernel + # sparseCheckoutDir: projects/composablekernel + +jobs: +- ${{ each component in parameters.downstreamComponentMatrix }}: + - template: /.azuredevops/components/${{ component.name }}.yml@pipelines_repo + parameters: + checkoutRepo: ${{ parameters.checkoutRepo }} + sparseCheckoutDir: ${{ component.sparseCheckoutDir }} + buildDependsOn: ${{ parameters.buildDependsOn }} + triggerDownstreamJobs: true + unifiedBuild: true diff --git a/.azuredevops/components/hipBLAS-common.yml b/.azuredevops/components/hipBLAS-common.yml index 6caa998ee..d8eabe01c 100644 --- a/.azuredevops/components/hipBLAS-common.yml +++ b/.azuredevops/components/hipBLAS-common.yml @@ -1,10 +1,29 @@ parameters: +- name: componentName + type: string + default: hipBLAS-common - name: checkoutRepo type: string default: 'self' - name: checkoutRef type: string default: '' +# monorepo related parameters +- name: sparseCheckoutDir + type: string + default: '' +- name: triggerDownstreamJobs + type: boolean + default: false +- name: downstreamAggregateNames + type: string + default: '' +- name: buildDependsOn + type: object + default: null +- name: unifiedBuild + type: boolean + default: false # set to true if doing full build of ROCm stack # and dependencies are pulled from same pipeline - name: aggregatePipeline @@ -27,6 +46,19 @@ parameters: - clr - rocminfo +- name: downstreamComponentMatrix + type: object + default: + - hipBLASLt: + name: hipBLASLt + sparseCheckoutDir: projects/hipblaslt + skipUnifiedBuild: 'false' + buildDependsOn: + gfx942: + - hipBLAS_common + gfx90a: + - hipBLAS_common + jobs: - job: hipBLAS_common variables: @@ -46,11 +78,14 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml parameters: checkoutRepo: ${{ parameters.checkoutRepo }} + sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml parameters: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmDependencies }} aggregatePipeline: ${{ parameters.aggregatePipeline }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml parameters: extraBuildFlags: >- @@ -58,10 +93,26 @@ jobs: -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ -GNinja - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml + parameters: + componentName: ${{ parameters.componentName }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml + parameters: + componentName: ${{ parameters.componentName }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml # - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml # parameters: # aptPackages: ${{ parameters.aptPackages }} # extraEnvVars: # - ROCM_PATH:::/home/user/workspace/rocm + +- ${{ if parameters.triggerDownstreamJobs }}: + - ${{ each component in parameters.downstreamComponentMatrix }}: + - ${{ if not(and(parameters.unifiedBuild, eq(component.skipUnifiedBuild, 'true'))) }}: + - template: /.azuredevops/components/${{ component.name }}.yml@pipelines_repo + parameters: + checkoutRepo: ${{ parameters.checkoutRepo }} + sparseCheckoutDir: ${{ component.sparseCheckoutDir }} + buildDependsOn: ${{ component.buildDependsOn }} + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ parameters.componentName }} + triggerDownstreamJobs: true + unifiedBuild: ${{ parameters.unifiedBuild }} diff --git a/.azuredevops/components/hipBLASLt.yml b/.azuredevops/components/hipBLASLt.yml index 418b466c7..c92c96a45 100644 --- a/.azuredevops/components/hipBLASLt.yml +++ b/.azuredevops/components/hipBLASLt.yml @@ -1,10 +1,29 @@ parameters: +- name: componentName + type: string + default: hipBLASLt - name: checkoutRepo type: string default: 'self' - name: checkoutRef type: string default: '' +# monorepo related parameters +- name: sparseCheckoutDir + type: string + default: '' +- name: triggerDownstreamJobs + type: boolean + default: false +- name: downstreamAggregateNames + type: string + default: '' +- name: buildDependsOn + type: object + default: null +- name: unifiedBuild + type: boolean + default: false # set to true if doing full build of ROCm stack # and dependencies are pulled from same pipeline - name: aggregatePipeline @@ -67,11 +86,25 @@ parameters: target: gfx942 - gfx90a: target: gfx90a +- name: downstreamComponentMatrix + type: object + default: + - rocBLAS: + name: rocBLAS + sparseCheckoutDir: projects/rocblas + skipUnifiedBuild: 'false' + buildDependsOn: + gfx942: + - hipBLASLt_build_gfx942 + gfx90a: + - hipBLASLt_build_gfx90a jobs: - ${{ each job in parameters.jobMatrix.buildJobs }}: - - job: hipBLASLt_build_${{ job.target }} + - job: ${{ parameters.componentName }}_build_${{ job.target }} timeoutInMinutes: 300 + ${{ if parameters.buildDependsOn }}: + dependsOn: ${{ parameters.buildDependsOn[job.target] }} variables: - group: common - template: /.azuredevops/variables-global.yml @@ -98,12 +131,15 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml parameters: checkoutRepo: ${{ parameters.checkoutRepo }} + sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml parameters: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmDependencies }} gpuTarget: ${{ job.target }} aggregatePipeline: ${{ parameters.aggregatePipeline }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - task: Bash@3 displayName: Add ROCm binaries to PATH inputs: @@ -158,9 +194,11 @@ jobs: -GNinja - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml parameters: + componentName: ${{ parameters.componentName }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml parameters: + componentName: ${{ parameters.componentName }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml @@ -179,14 +217,15 @@ jobs: - deps - ${{ each job in parameters.jobMatrix.testJobs }}: - - job: hipBLASLt_test_${{ job.target }} + - job: ${{ parameters.componentName }}_test_${{ job.target }} timeoutInMinutes: 300 - dependsOn: hipBLASLt_build_${{ job.target }} + dependsOn: ${{ parameters.componentName }}_build_${{ job.target }} condition: and(succeeded(), eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), - not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])), - eq(${{ parameters.aggregatePipeline }}, False) + not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), '${{ parameters.componentName }}')), + eq(${{ parameters.aggregatePipeline }}, False), + eq(${{ parameters.unifiedBuild }}, False) ) variables: - group: common @@ -204,6 +243,7 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml parameters: + preTargetFilter: ${{ parameters.componentName }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml @@ -211,10 +251,12 @@ jobs: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmTestDependencies }} gpuTarget: ${{ job.target }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml parameters: - componentName: hipBLASLt + componentName: ${{ parameters.componentName }} testDir: '$(Agent.BuildDirectory)/rocm/bin' testExecutable: './hipblaslt-test' testParameters: '--gtest_output=xml:./test_output.xml --gtest_color=yes --gtest_filter=*pre_checkin*' @@ -224,3 +266,15 @@ jobs: pipModules: ${{ parameters.pipModules }} environment: test gpuTarget: ${{ job.target }} + +- ${{ if parameters.triggerDownstreamJobs }}: + - ${{ each component in parameters.downstreamComponentMatrix }}: + - ${{ if not(and(parameters.unifiedBuild, eq(component.skipUnifiedBuild, 'true'))) }}: + - template: /.azuredevops/components/${{ component.name }}.yml@pipelines_repo + parameters: + checkoutRepo: ${{ parameters.checkoutRepo }} + sparseCheckoutDir: ${{ component.sparseCheckoutDir }} + buildDependsOn: ${{ component.buildDependsOn }} + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ parameters.componentName }} + triggerDownstreamJobs: true + unifiedBuild: ${{ parameters.unifiedBuild }} diff --git a/.azuredevops/components/hipCUB.yml b/.azuredevops/components/hipCUB.yml index f0597b387..0a224065f 100644 --- a/.azuredevops/components/hipCUB.yml +++ b/.azuredevops/components/hipCUB.yml @@ -21,6 +21,9 @@ parameters: - name: buildDependsOn type: object default: null +- name: unifiedBuild + type: boolean + default: false # set to true if doing full build of ROCm stack # and dependencies are pulled from same pipeline - name: aggregatePipeline @@ -124,7 +127,8 @@ jobs: and(succeeded(), eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), '${{ parameters.componentName }}')), - eq(${{ parameters.aggregatePipeline }}, False) + eq(${{ parameters.aggregatePipeline }}, False), + eq(${{ parameters.unifiedBuild }}, False) ) variables: - group: common @@ -148,6 +152,8 @@ jobs: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmTestDependencies }} gpuTarget: ${{ job.target }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml parameters: diff --git a/.azuredevops/components/hipRAND.yml b/.azuredevops/components/hipRAND.yml index da77ccb86..f1bfa12a2 100644 --- a/.azuredevops/components/hipRAND.yml +++ b/.azuredevops/components/hipRAND.yml @@ -21,6 +21,9 @@ parameters: - name: buildDependsOn type: object default: null +- name: unifiedBuild + type: boolean + default: false # set to true if doing full build of ROCm stack # and dependencies are pulled from same pipeline - name: aggregatePipeline @@ -131,7 +134,8 @@ jobs: and(succeeded(), eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), '${{ parameters.componentName }}')), - eq(${{ parameters.aggregatePipeline }}, False) + eq(${{ parameters.aggregatePipeline }}, False), + eq(${{ parameters.unifiedBuild }}, False) ) variables: - group: common @@ -155,6 +159,8 @@ jobs: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmTestDependencies }} gpuTarget: ${{ job.target }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml parameters: diff --git a/.azuredevops/components/rocBLAS.yml b/.azuredevops/components/rocBLAS.yml index 1de90494c..a8b50905f 100644 --- a/.azuredevops/components/rocBLAS.yml +++ b/.azuredevops/components/rocBLAS.yml @@ -1,10 +1,29 @@ parameters: +- name: componentName + type: string + default: rocBLAS - name: checkoutRepo type: string default: 'self' - name: checkoutRef type: string default: '' +# monorepo related parameters +- name: sparseCheckoutDir + type: string + default: '' +- name: triggerDownstreamJobs + type: boolean + default: false +- name: downstreamAggregateNames + type: string + default: '' +- name: buildDependsOn + type: object + default: null +- name: unifiedBuild + type: boolean + default: false # set to true if doing full build of ROCm stack # and dependencies are pulled from same pipeline - name: aggregatePipeline @@ -73,10 +92,35 @@ parameters: target: gfx942 - gfx90a: target: gfx90a +- name: downstreamComponentMatrix + type: object + default: + # rocSOLVER depends on both rocBLAS and rocPRIM + # for a unified build, rocBLAS will be the one to call rocSOLVER + - rocSOLVER: + name: rocSOLVER + sparseCheckoutDir: projects/rocsolver + skipUnifiedBuild: 'false' + buildDependsOn: + gfx942: + - rocBLAS_build_gfx942 + gfx90a: + - rocBLAS_build_gfx90a + unifiedBuild: + downstreamAggregateNames: rocBLAS+rocPRIM + buildDependsOn: + gfx942: + - rocBLAS_build_gfx942 + - rocPRIM_build_gfx942 + gfx90a: + - rocBLAS_build_gfx90a + - rocPRIM_build_gfx90a jobs: - ${{ each job in parameters.jobMatrix.buildJobs }}: - - job: rocBLAS_build_${{ job.target }} + - job: ${{ parameters.componentName }}_build_${{ job.target }} + ${{ if parameters.buildDependsOn }}: + dependsOn: ${{ parameters.buildDependsOn[job.target] }} variables: - group: common - template: /.azuredevops/variables-global.yml @@ -96,10 +140,12 @@ jobs: parameters: aptPackages: ${{ parameters.aptPackages }} pipModules: ${{ parameters.pipModules }} + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml parameters: checkoutRepo: ${{ parameters.checkoutRepo }} + sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aocl.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml parameters: @@ -107,6 +153,8 @@ jobs: dependencyList: ${{ parameters.rocmDependencies }} gpuTarget: ${{ job.target }} aggregatePipeline: ${{ parameters.aggregatePipeline }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml parameters: extraBuildFlags: >- @@ -128,9 +176,11 @@ jobs: -GNinja - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml parameters: + componentName: ${{ parameters.componentName }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml parameters: + componentName: ${{ parameters.componentName }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml @@ -146,13 +196,14 @@ jobs: - ROCM_PATH:::/home/user/workspace/rocm - ${{ each job in parameters.jobMatrix.testJobs }}: - - job: rocBLAS_test_${{ job.target }} - dependsOn: rocBLAS_build_${{ job.target }} + - job: ${{ parameters.componentName }}_test_${{ job.target }} + dependsOn: ${{ parameters.componentName }}_build_${{ job.target }} condition: and(succeeded(), eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), - not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])), - eq(${{ parameters.aggregatePipeline }}, False) + not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), '${{ parameters.componentName }}')), + eq(${{ parameters.aggregatePipeline }}, False), + eq(${{ parameters.unifiedBuild }}, False) ) variables: - group: common @@ -168,6 +219,7 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml parameters: + preTargetFilter: ${{ parameters.componentName }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml @@ -175,10 +227,12 @@ jobs: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmTestDependencies }} gpuTarget: ${{ job.target }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml parameters: - componentName: rocBLAS + componentName: ${{ parameters.componentName }} testDir: '$(Agent.BuildDirectory)/rocm/bin' testExecutable: './rocblas-test' testParameters: '--yaml rocblas_smoke.yaml --gtest_output=xml:./test_output.xml --gtest_color=yes' @@ -188,3 +242,19 @@ jobs: pipModules: ${{ parameters.pipModules }} environment: test gpuTarget: ${{ job.target }} + +- ${{ if parameters.triggerDownstreamJobs }}: + - ${{ each component in parameters.downstreamComponentMatrix }}: + - ${{ if not(and(parameters.unifiedBuild, eq(component.skipUnifiedBuild, 'true'))) }}: + - template: /.azuredevops/components/${{ component.name }}.yml@pipelines_repo + parameters: + checkoutRepo: ${{ parameters.checkoutRepo }} + sparseCheckoutDir: ${{ component.sparseCheckoutDir }} + ${{ if parameters.unifiedBuild }}: + buildDependsOn: ${{ component.unifiedBuild.buildDependsOn }} + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ component.unifiedBuild.downstreamAggregateNames }} + ${{ else }}: + buildDependsOn: ${{ component.buildDependsOn }} + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ parameters.componentName }} + triggerDownstreamJobs: true + unifiedBuild: ${{ parameters.unifiedBuild }} diff --git a/.azuredevops/components/rocPRIM.yml b/.azuredevops/components/rocPRIM.yml index 82d8989eb..fa273439f 100644 --- a/.azuredevops/components/rocPRIM.yml +++ b/.azuredevops/components/rocPRIM.yml @@ -21,6 +21,9 @@ parameters: - name: buildDependsOn type: object default: null +- name: unifiedBuild + type: boolean + default: false # set to true if doing full build of ROCm stack # and dependencies are pulled from same pipeline - name: aggregatePipeline @@ -70,6 +73,7 @@ parameters: - rocThrust: name: rocThrust sparseCheckoutDir: projects/rocthrust + skipUnifiedBuild: 'false' buildDependsOn: gfx942: - rocPRIM_build_gfx942 @@ -78,6 +82,18 @@ parameters: - hipCUB: name: hipCUB sparseCheckoutDir: projects/hipcub + skipUnifiedBuild: 'false' + buildDependsOn: + gfx942: + - rocPRIM_build_gfx942 + gfx90a: + - rocPRIM_build_gfx90a + # rocSOLVER depends on both rocBLAS and rocPRIM + # for a unified build, rocBLAS will be the one to call rocSOLVER + - rocSOLVER: + name: rocSOLVER + sparseCheckoutDir: projects/rocsolver + skipUnifiedBuild: 'true' buildDependsOn: gfx942: - rocPRIM_build_gfx942 @@ -142,7 +158,8 @@ jobs: and(succeeded(), eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), '${{ parameters.componentName }}')), - eq(${{ parameters.aggregatePipeline }}, False) + eq(${{ parameters.aggregatePipeline }}, False), + eq(${{ parameters.unifiedBuild }}, False) ) variables: - group: common @@ -166,6 +183,8 @@ jobs: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmTestDependencies }} gpuTarget: ${{ job.target }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml parameters: @@ -179,9 +198,12 @@ jobs: - ${{ if parameters.triggerDownstreamJobs }}: - ${{ each component in parameters.downstreamComponentMatrix }}: - - template: /.azuredevops/components/${{ component.name }}.yml@pipelines_repo - parameters: - sparseCheckoutDir: ${{ component.sparseCheckoutDir }} - buildDependsOn: ${{ component.buildDependsOn }} - downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ parameters.componentName }} - triggerDownstreamJobs: true + - ${{ if not(and(parameters.unifiedBuild, eq(component.skipUnifiedBuild, 'true'))) }}: + - template: /.azuredevops/components/${{ component.name }}.yml@pipelines_repo + parameters: + checkoutRepo: ${{ parameters.checkoutRepo }} + sparseCheckoutDir: ${{ component.sparseCheckoutDir }} + buildDependsOn: ${{ component.buildDependsOn }} + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ parameters.componentName }} + triggerDownstreamJobs: true + unifiedBuild: ${{ parameters.unifiedBuild }} diff --git a/.azuredevops/components/rocRAND.yml b/.azuredevops/components/rocRAND.yml index 4dfcb1aa3..8ec7abd4c 100644 --- a/.azuredevops/components/rocRAND.yml +++ b/.azuredevops/components/rocRAND.yml @@ -21,6 +21,9 @@ parameters: - name: buildDependsOn type: object default: null +- name: unifiedBuild + type: boolean + default: false # set to true if doing full build of ROCm stack # and dependencies are pulled from same pipeline - name: aggregatePipeline @@ -71,6 +74,7 @@ parameters: - hipRAND: name: hipRAND sparseCheckoutDir: projects/hiprand + skipUnifiedBuild: 'false' buildDependsOn: gfx942: - rocRAND_build_gfx942 @@ -139,7 +143,8 @@ jobs: and(succeeded(), eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), '${{ parameters.componentName }}')), - eq(${{ parameters.aggregatePipeline }}, False) + eq(${{ parameters.aggregatePipeline }}, False), + eq(${{ parameters.unifiedBuild }}, False) ) variables: - group: common @@ -163,6 +168,8 @@ jobs: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmTestDependencies }} gpuTarget: ${{ job.target }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml parameters: @@ -176,9 +183,12 @@ jobs: - ${{ if parameters.triggerDownstreamJobs }}: - ${{ each component in parameters.downstreamComponentMatrix }}: - - template: /.azuredevops/components/${{ component.name }}.yml@pipelines_repo - parameters: - sparseCheckoutDir: ${{ component.sparseCheckoutDir }} - buildDependsOn: ${{ component.buildDependsOn }} - downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ parameters.componentName }} - triggerDownstreamJobs: true + - ${{ if not(and(parameters.unifiedBuild, eq(component.skipUnifiedBuild, 'true'))) }}: + - template: /.azuredevops/components/${{ component.name }}.yml@pipelines_repo + parameters: + checkoutRepo: ${{ parameters.checkoutRepo }} + sparseCheckoutDir: ${{ component.sparseCheckoutDir }} + buildDependsOn: ${{ component.buildDependsOn }} + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }}+${{ parameters.componentName }} + triggerDownstreamJobs: true + unifiedBuild: ${{ parameters.unifiedBuild }} diff --git a/.azuredevops/components/rocSOLVER.yml b/.azuredevops/components/rocSOLVER.yml index f21ab2670..702e1425c 100644 --- a/.azuredevops/components/rocSOLVER.yml +++ b/.azuredevops/components/rocSOLVER.yml @@ -1,10 +1,29 @@ parameters: +- name: componentName + type: string + default: rocSOLVER - name: checkoutRepo type: string default: 'self' - name: checkoutRef type: string default: '' +# monorepo related parameters +- name: sparseCheckoutDir + type: string + default: '' +- name: triggerDownstreamJobs + type: boolean + default: false +- name: downstreamAggregateNames + type: string + default: '' +- name: buildDependsOn + type: object + default: null +- name: unifiedBuild + type: boolean + default: false # set to true if doing full build of ROCm stack # and dependencies are pulled from same pipeline - name: aggregatePipeline @@ -65,7 +84,9 @@ parameters: jobs: - ${{ each job in parameters.jobMatrix.buildJobs }}: - - job: rocSOLVER_build_${{ job.target }} + - job: ${{ parameters.componentName }}_build_${{ job.target }} + ${{ if parameters.buildDependsOn }}: + dependsOn: ${{ parameters.buildDependsOn[job.target] }} variables: - group: common - template: /.azuredevops/variables-global.yml @@ -80,6 +101,7 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml parameters: checkoutRepo: ${{ parameters.checkoutRepo }} + sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }} - task: Bash@3 displayName: 'Clone lapack' inputs: @@ -92,6 +114,8 @@ jobs: dependencyList: ${{ parameters.rocmDependencies }} gpuTarget: ${{ job.target }} aggregatePipeline: ${{ parameters.aggregatePipeline }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml parameters: componentName: lapack @@ -118,9 +142,11 @@ jobs: -GNinja - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml parameters: + componentName: ${{ parameters.componentName }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-upload.yml parameters: + componentName: ${{ parameters.componentName }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/docker-container.yml @@ -131,13 +157,14 @@ jobs: - deps-install - ${{ each job in parameters.jobMatrix.testJobs }}: - - job: rocSOLVER_test_${{ job.target }} - dependsOn: rocSOLVER_build_${{ job.target }} + - job: ${{ parameters.componentName }}_test_${{ job.target }} + dependsOn: ${{ parameters.componentName }}_build_${{ job.target }} condition: and(succeeded(), eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), - not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), variables['Build.DefinitionName'])), - eq(${{ parameters.aggregatePipeline }}, False) + not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), '${{ parameters.componentName }}')), + eq(${{ parameters.aggregatePipeline }}, False), + eq(${{ parameters.unifiedBuild }}, False) ) variables: - group: common @@ -152,6 +179,7 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/local-artifact-download.yml parameters: + preTargetFilter: ${{ parameters.componentName }} gpuTarget: ${{ job.target }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml @@ -159,10 +187,12 @@ jobs: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmTestDependencies }} gpuTarget: ${{ job.target }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml parameters: - componentName: rocSOLVER + componentName: ${{ parameters.componentName }} testDir: '$(Agent.BuildDirectory)/rocm/bin' testExecutable: './rocsolver-test' testParameters: '--gtest_filter="*checkin*" --gtest_output=xml:./test_output.xml --gtest_color=yes' diff --git a/.azuredevops/components/rocThrust.yml b/.azuredevops/components/rocThrust.yml index 54d944da1..2c893ab0b 100644 --- a/.azuredevops/components/rocThrust.yml +++ b/.azuredevops/components/rocThrust.yml @@ -21,6 +21,9 @@ parameters: - name: buildDependsOn type: object default: null +- name: unifiedBuild + type: boolean + default: false # set to true if doing full build of ROCm stack # and dependencies are pulled from same pipeline - name: aggregatePipeline @@ -126,7 +129,8 @@ jobs: and(succeeded(), eq(variables['ENABLE_${{ upper(job.target) }}_TESTS'], 'true'), not(containsValue(split(variables['DISABLED_${{ upper(job.target) }}_TESTS'], ','), '${{ parameters.componentName }}')), - eq(${{ parameters.aggregatePipeline }}, False) + eq(${{ parameters.aggregatePipeline }}, False), + eq(${{ parameters.unifiedBuild }}, False) ) variables: - group: common @@ -150,6 +154,8 @@ jobs: checkoutRef: ${{ parameters.checkoutRef }} dependencyList: ${{ parameters.rocmTestDependencies }} gpuTarget: ${{ job.target }} + ${{ if parameters.triggerDownstreamJobs }}: + downstreamAggregateNames: ${{ parameters.downstreamAggregateNames }} - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/gpu-diagnostics.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/test.yml parameters: