mirror of
https://github.com/ROCm/ROCm.git
synced 2026-01-08 22:28:06 -05:00
401 lines
12 KiB
YAML
401 lines
12 KiB
YAML
# download and install rocm dependencies through pipeline builds in the project
|
|
parameters:
|
|
- name: checkoutRef
|
|
type: string
|
|
default: ''
|
|
- name: dependencyList
|
|
type: object
|
|
default: []
|
|
- name: os
|
|
type: string
|
|
default: 'ubuntu2204'
|
|
- name: gpuTarget
|
|
type: string
|
|
default: ''
|
|
# set to true if dlopen calls for HIP libraries are causing failures
|
|
# because they do not follow shared library symlink convention
|
|
- name: setupHIPLibrarySymlinks
|
|
type: boolean
|
|
default: false
|
|
# set to true if doing full build of ROCm stack
|
|
# and dependencies are pulled from same pipeline
|
|
- name: aggregatePipeline
|
|
type: boolean
|
|
default: false
|
|
# monorepo related parameters
|
|
- name: downstreamAggregateNames
|
|
type: string
|
|
default: ''
|
|
|
|
- name: componentVarList
|
|
type: object
|
|
default:
|
|
AMDMIGraphX:
|
|
pipelineId: 113
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
amdsmi:
|
|
pipelineId: 376
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
aomp-extras:
|
|
pipelineId: 111
|
|
developBranch: aomp-dev
|
|
hasGpuTarget: false
|
|
aomp:
|
|
pipelineId: 115
|
|
developBranch: aomp-dev
|
|
hasGpuTarget: false
|
|
aqlprofile:
|
|
pipelineId: 365
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
clr:
|
|
pipelineId: 335
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
composable_kernel:
|
|
pipelineId: 86
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
half:
|
|
pipelineId: 101
|
|
developBranch: rocm
|
|
hasGpuTarget: false
|
|
HIP:
|
|
pipelineId: 335
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
hip-tests:
|
|
pipelineId: 362
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
hipBLAS:
|
|
pipelineId: 317
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
hipBLASLt:
|
|
pipelineId: 301
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
hipBLAS-common:
|
|
pipelineId: 300
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
hipCUB:
|
|
pipelineId: 277
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
hipFFT:
|
|
pipelineId: 283
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
hipfort:
|
|
pipelineId: 102
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
HIPIFY:
|
|
pipelineId: 92
|
|
developBranch: amd-staging
|
|
hasGpuTarget: false
|
|
hipRAND:
|
|
pipelineId: 275
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
hipSOLVER:
|
|
pipelineId: 84
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
hipSPARSE:
|
|
pipelineId: 315
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
hipSPARSELt:
|
|
pipelineId: 309
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
hipTensor:
|
|
pipelineId: 374
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
llvm-project:
|
|
pipelineId: 2
|
|
developBranch: amd-staging
|
|
hasGpuTarget: false
|
|
MIOpen:
|
|
pipelineId: 320
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
MIVisionX:
|
|
pipelineId: 80
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
origami:
|
|
pipelineId: 364
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rccl:
|
|
pipelineId: 107
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rdc:
|
|
pipelineId: 360
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
rocAL:
|
|
pipelineId: 151
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rocALUTION:
|
|
pipelineId: 89
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rocBLAS:
|
|
pipelineId: 302
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
ROCdbgapi:
|
|
pipelineId: 135
|
|
developBranch: amd-staging
|
|
hasGpuTarget: false
|
|
rocDecode:
|
|
pipelineId: 79
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
rocFFT:
|
|
pipelineId: 282
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
ROCgdb:
|
|
pipelineId: 134
|
|
developBranch: amd-staging
|
|
hasGpuTarget: false
|
|
rocJPEG:
|
|
pipelineId: 262
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
rocm-cmake:
|
|
pipelineId: 6
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
rocm-core:
|
|
pipelineId: 349
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
rocm-examples:
|
|
pipelineId: 216
|
|
developBranch: amd-staging
|
|
hasGpuTarget: true
|
|
rocminfo:
|
|
pipelineId: 356
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
rocMLIR:
|
|
pipelineId: 229
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
ROCmValidationSuite:
|
|
pipelineId: 106
|
|
developBranch: master
|
|
hasGpuTarget: true
|
|
rocm_bandwidth_test:
|
|
pipelineId: 88
|
|
developBranch: master
|
|
hasGpuTarget: false
|
|
rocm_smi_lib:
|
|
pipelineId: 358
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
rocPRIM:
|
|
pipelineId: 273
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rocprofiler:
|
|
pipelineId: 329
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rocprofiler-compute:
|
|
pipelineId: 344
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rocprofiler-register:
|
|
pipelineId: 327
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
rocprofiler-sdk:
|
|
pipelineId: 347
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rocprofiler-systems:
|
|
pipelineId: 345
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rocPyDecode:
|
|
pipelineId: 239
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
ROCR-Runtime:
|
|
pipelineId: 354
|
|
developBranch: develop
|
|
hasGpuTarget: false
|
|
rocRAND:
|
|
pipelineId: 274
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rocr_debug_agent:
|
|
pipelineId: 136
|
|
developBranch: amd-staging
|
|
hasGpuTarget: false
|
|
rocSOLVER:
|
|
pipelineId: 81
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rocSPARSE:
|
|
pipelineId: 314
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rocThrust:
|
|
pipelineId: 276
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
roctracer:
|
|
pipelineId: 331
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rocWMMA:
|
|
pipelineId: 370
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
rpp:
|
|
pipelineId: 78
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
TransferBench:
|
|
pipelineId: 265
|
|
developBranch: develop
|
|
hasGpuTarget: true
|
|
|
|
steps:
|
|
# assuming artifact-download.yml template file in same directory
|
|
# for the case where rocm dependency item in list has a colon (:)
|
|
# assume it is of the format of componentName:fileFilter
|
|
# fileFilter could contain both a subcomponent name or gpu name separated by asterisks
|
|
# gpu name will be specified by parameters.gpuTarget for components that are in componentsWithGPUTarget
|
|
# e.g., gfx942 to only download artifacts from component for this gpu if applicable
|
|
- ${{ each dependency in parameters.dependencyList }}:
|
|
- ${{ if contains(dependency, ':') }}:
|
|
- template: artifact-download.yml
|
|
parameters:
|
|
componentName: ${{ split(dependency, ':')[0] }}
|
|
pipelineId: ${{ parameters.componentVarList[split(dependency, ':')[0]].pipelineId }}
|
|
branchName: ${{ parameters.componentVarList[split(dependency, ':')[0]].developBranch }}
|
|
aggregatePipeline: ${{ parameters.aggregatePipeline }}
|
|
extractAndDeleteFiles: false
|
|
# no colon (:) found in this item in the list
|
|
- ${{ elseif containsValue(split(parameters.downstreamAggregateNames, '+'), dependency) }}:
|
|
- template: local-artifact-download.yml
|
|
parameters:
|
|
buildType: current
|
|
preTargetFilter: ${{ dependency }}
|
|
os: ${{ parameters.os }}
|
|
${{ if parameters.componentVarList[dependency].hasGpuTarget }}:
|
|
gpuTarget: ${{ parameters.gpuTarget }}
|
|
- ${{ else }}:
|
|
- template: artifact-download.yml
|
|
parameters:
|
|
componentName: ${{ dependency }}
|
|
pipelineId: ${{ parameters.componentVarList[dependency].pipelineId }}
|
|
branchName: ${{ parameters.componentVarList[dependency].developBranch }}
|
|
aggregatePipeline: ${{ parameters.aggregatePipeline }}
|
|
extractAndDeleteFiles: false
|
|
${{ if parameters.componentVarList[dependency].hasGpuTarget }}:
|
|
fileFilter: ${{ parameters.os }}_${{ parameters.gpuTarget }}
|
|
${{ else }}:
|
|
fileFilter: ${{ parameters.os }}
|
|
- task: ExtractFiles@1
|
|
displayName: Extract ROCm artifacts
|
|
inputs:
|
|
archiveFilePatterns: $(Pipeline.Workspace)/d/**/*.tar.gz
|
|
destinationFolder: $(Agent.BuildDirectory)/rocm
|
|
cleanDestinationFolder: false
|
|
overwriteExistingFiles: true
|
|
- task: DeleteFiles@1
|
|
displayName: Clean up ROCm artifacts
|
|
inputs:
|
|
SourceFolder: $(Pipeline.Workspace)/d
|
|
Contents: '**/*.tar.gz'
|
|
RemoveDotFiles: true
|
|
- ${{ if containsValue(parameters.dependencyList, 'llvm-project') }}:
|
|
- task: Bash@3
|
|
displayName: Symlink from rocm/llvm to rocm/lib/llvm
|
|
inputs:
|
|
targetType: inline
|
|
script: |
|
|
sudo mkdir -p $(Agent.BuildDirectory)/rocm/lib
|
|
sudo ln -sr $(Agent.BuildDirectory)/rocm/llvm $(Agent.BuildDirectory)/rocm/lib/llvm
|
|
echo "Created symlink from rocm/llvm to rocm/lib/llvm"
|
|
- task: Bash@3
|
|
displayName: Symlink executables from rocm/llvm/bin to rocm/bin
|
|
inputs:
|
|
targetType: inline
|
|
script: |
|
|
for file in amdclang amdclang++ amdclang-cl amdclang-cpp amdflang amdlld aompcc mygpu mycpu offload-arch; do
|
|
sudo ln -sr $(Agent.BuildDirectory)/rocm/llvm/bin/$file $(Agent.BuildDirectory)/rocm/bin/$file
|
|
echo "Created symlink from rocm/llvm/bin/$file to rocm/bin/$file"
|
|
done
|
|
- ${{ if containsValue(parameters.dependencyList, 'rocm-core') }}:
|
|
- task: Bash@3
|
|
displayName: Print rocm/.info/version
|
|
inputs:
|
|
targetType: inline
|
|
script: cat $(Agent.BuildDirectory)/rocm/.info/version
|
|
# dlopen calls within a ctest or pytest sequence runs into issues when shared library symlink convention is not followed
|
|
# the convention is as follows:
|
|
# unversioned .so is a symlink to major version .so
|
|
# major version .so is a symlink to detailed version .so
|
|
# HIP libraries do not follow this convention, and each .so is a copy of each other
|
|
# changing the library structure to follow the symlink convention resolves some test failures
|
|
- ${{ if eq(parameters.setupHIPLibrarySymlinks, true) }}:
|
|
- task: Bash@3
|
|
displayName: Setup symlinks for hip libraries
|
|
inputs:
|
|
targetType: inline
|
|
workingDirectory: $(Agent.BuildDirectory)/rocm/lib
|
|
script: |
|
|
LIBRARIES=("libamdhip64" "libhiprtc-builtins" "libhiprtc")
|
|
for LIB_NAME in "${LIBRARIES[@]}"; do
|
|
VERSIONED_SO=$(ls ${LIB_NAME}.so.* 2>/dev/null | grep -E "${LIB_NAME}\.so\.[0-9]+\.[0-9]+\.[0-9]+(-.*)?" | sort -V | tail -n 1)
|
|
if [[ -z "$VERSIONED_SO" ]]; then
|
|
continue
|
|
fi
|
|
MAJOR_VERSION=$(echo "$VERSIONED_SO" | grep -oP "${LIB_NAME}\.so\.\K[0-9]+")
|
|
if [[ -e "${LIB_NAME}.so.${MAJOR_VERSION}" && ! -L "${LIB_NAME}.so.${MAJOR_VERSION}" ]]; then
|
|
rm -f "${LIB_NAME}.so.${MAJOR_VERSION}"
|
|
fi
|
|
if [[ -e "${LIB_NAME}.so" && ! -L "${LIB_NAME}.so" ]]; then
|
|
rm -f "${LIB_NAME}.so"
|
|
fi
|
|
ln -sf "$VERSIONED_SO" "${LIB_NAME}.so.${MAJOR_VERSION}"
|
|
ln -sf "${LIB_NAME}.so.${MAJOR_VERSION}" "${LIB_NAME}.so"
|
|
echo "Symlinks created for $LIB_NAME:"
|
|
ls -l ${LIB_NAME}.so*
|
|
done
|
|
- task: Bash@3
|
|
displayName: 'List downloaded ROCm files'
|
|
inputs:
|
|
targetType: inline
|
|
script: ls -la1R $(Agent.BuildDirectory)/rocm
|
|
- task: Bash@3
|
|
displayName: 'Link ROCm shared libraries'
|
|
inputs:
|
|
targetType: inline
|
|
# OS ignores if the ROCm lib folder shows up more than once
|
|
script: |
|
|
echo $(Agent.BuildDirectory)/rocm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
|
|
echo $(Agent.BuildDirectory)/rocm/llvm/lib | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
|
|
echo $(Agent.BuildDirectory)/rocm/lib64 | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
|
|
echo $(Agent.BuildDirectory)/rocm/llvm/lib64 | sudo tee -a /etc/ld.so.conf.d/rocm-ci.conf
|
|
sudo cat /etc/ld.so.conf.d/rocm-ci.conf
|
|
sudo ldconfig -v
|
|
ldconfig -p
|