diff --git a/.azuredevops/components/MIOpen.yml b/.azuredevops/components/MIOpen.yml index e87cd0233..4896c6991 100644 --- a/.azuredevops/components/MIOpen.yml +++ b/.azuredevops/components/MIOpen.yml @@ -9,19 +9,14 @@ parameters: type: object default: - cmake - - libbz2-dev + - jq - libdrm-dev - - libeigen3-dev - - libgmock-dev - - libgtest-dev - - libsqlite3-dev - libstdc++-12-dev - - libzstd-dev - ninja-build - - nlohmann-json3-dev - python3-pip + - python3-venv - software-properties-common - - zstd + - zip - name: pipModules type: object default: @@ -29,14 +24,12 @@ parameters: - name: rocmDependencies type: object default: - - rocMLIR + - half - rocRAND - rocBLAS - hipBLAS - hipBLASLt - hipBLAS-common - - half - - composable_kernel - rocm-cmake - llvm-project - ROCR-Runtime @@ -48,7 +41,6 @@ parameters: type: object default: - clr - - composable_kernel - half - hipBLAS - hipBLAS-common @@ -57,7 +49,6 @@ parameters: - rocBLAS - rocm-cmake - rocminfo - - rocMLIR - ROCR-Runtime - rocprofiler-register - rocRAND @@ -84,8 +75,9 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml parameters: checkoutRepo: ${{ parameters.checkoutRepo }} - # The default boost library from apt is 1.74, which does not satisfy MIOpen's build requirement (1.79+) - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-boost.yml + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml + parameters: + gpuTarget: $(JOB_GPU_TARGET) - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml parameters: dependencyList: ${{ parameters.rocmDependencies }} @@ -96,12 +88,22 @@ jobs: # manual build case: triggered by ROCm/ROCm repo ${{ elseif ne(parameters.checkoutRef, '') }}: dependencySource: tag-builds + - task: Bash@3 + displayName: Build and install other dependencies + inputs: + targetType: inline + workingDirectory: $(Build.SourcesDirectory) + script: | + sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm + sed -i '/composable_kernel/d' requirements.txt + sudo cmake -P install_deps.cmake + sudo rm -rf /opt/rocm - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml parameters: extraBuildFlags: >- -DMIOPEN_BACKEND=HIP -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ - -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Agent.BuildDirectory)/boost + -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm -DAMDGPU_TARGETS=$(JOB_GPU_TARGET) -DMIOPEN_ENABLE_AI_KERNEL_TUNING=OFF -DMIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK=OFF @@ -113,6 +115,7 @@ jobs: gpuTarget: $(JOB_GPU_TARGET) - job: MIOpen_testing + timeoutInMinutes: 90 dependsOn: MIOpen condition: and(succeeded(), eq(variables.ENABLE_GFX942_TESTS, 'true'), not(containsValue(split(variables.DISABLED_GFX942_TESTS, ','), variables['Build.DefinitionName']))) variables: @@ -135,14 +138,15 @@ jobs: - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml parameters: checkoutRepo: ${{ parameters.checkoutRepo }} - # The default boost library from apt is 1.74, which does not satisfy MIOpen's build requirement (1.79+) - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-boost.yml - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml parameters: ${{ if eq(parameters.checkoutRef, '') }}: dependencySource: staging ${{ elseif ne(parameters.checkoutRef, '') }}: dependencySource: tag-builds + - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml + parameters: + gpuTarget: $(JOB_GPU_TARGET) - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml parameters: dependencyList: ${{ parameters.rocmTestDependencies }} @@ -151,44 +155,21 @@ jobs: dependencySource: staging ${{ elseif ne(parameters.checkoutRef, '') }}: dependencySource: tag-builds -# MIOpen depends on a specific version of frugally-deep which is forked here: https://github.com/ROCm/frugally-deep -# https://github.com/ROCm/frugally-deep/blob/master/INSTALL.md - task: Bash@3 - displayName: Add Python site-packages binaries to path + displayName: Build and install other dependencies inputs: targetType: inline - script: | - USER_BASE=$(python3 -m site --user-base) - echo "##vso[task.prependpath]$USER_BASE/bin" - - task: Bash@3 - displayName: Install FunctionalPlus - inputs: - targetType: inline - script: cget install Dobiasd/FunctionalPlus - - task: Bash@3 - displayName: Remove Python site-packages binaries from path - inputs: - targetType: inline - script: | - USER_BASE=$(python3 -m site --user-base) - echo "##vso[task.setvariable variable=PATH]$(echo $PATH | sed -e 's;:$USER_BASE/bin;;' -e 's;^/;;' -e 's;/$;;')" - - task: Bash@3 - displayName: git clone frugally-deep - inputs: - targetType: inline - script: git clone https://github.com/ROCm/frugally-deep --depth=1 --shallow-submodules workingDirectory: $(Build.SourcesDirectory) - - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/build-cmake.yml - parameters: - componentName: frugally-deep - cmakeBuildDir: $(Build.SourcesDirectory)/frugally-deep/build - installDir: $(Build.SourcesDirectory)/bin - extraBuildFlags: -DCMAKE_PREFIX_PATH=$(Build.SourcesDirectory)/cget/cget/pkg/Dobiasd__FunctionalPlus/install + script: | + sudo ln -s $(Agent.BuildDirectory)/rocm /opt/rocm + sed -i '/composable_kernel/d' requirements.txt + sudo cmake -P install_deps.cmake + sudo rm -rf /opt/rocm - task: CMake@1 displayName: 'MIOpen Test CMake Flags' inputs: cmakeArgs: >- - -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Build.SourcesDirectory)/bin;$(Build.SourcesDirectory)/cget/cget/pkg/Dobiasd__FunctionalPlus/install;$(Agent.BuildDirectory)/boost + -DCMAKE_PREFIX_PATH=$(Agent.BuildDirectory)/rocm;$(Build.SourcesDirectory)/bin -DCMAKE_INSTALL_PREFIX=$(Agent.BuildDirectory)/rocm -DCMAKE_CXX_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ -DCMAKE_C_COMPILER=$(Agent.BuildDirectory)/rocm/llvm/bin/amdclang diff --git a/.azuredevops/templates/steps/miopen-get-ck-build.yml b/.azuredevops/templates/steps/miopen-get-ck-build.yml new file mode 100644 index 000000000..6ea77a51f --- /dev/null +++ b/.azuredevops/templates/steps/miopen-get-ck-build.yml @@ -0,0 +1,72 @@ +parameters: +- name: gpuTarget + type: string + default: '' + +steps: +- task: Bash@3 + name: downloadCKBuild + displayName: Download specific CK build + continueOnError: true + env: + CXX: $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang++ + CC: $(Agent.BuildDirectory)/rocm/llvm/bin/amdclang + inputs: + targetType: inline + workingDirectory: $(Build.SourcesDirectory) + script: | + AZ_API="https://dev.azure.com/ROCm-CI/ROCm-CI/_apis" + GH_API="https://api.github.com/repos/ROCm" + ARTIFACT_NAME="composablekernel.${{ parameters.gpuTarget }}" + EXIT_CODE=0 + + # The commits that MIOpen reference are all merge commits from CK/develop to CK/amd-develop + # These commits are present on CK/amd-develop but not on CK/develop + # Ex-CI only builds CK/develop, so we need to find a commit present on both CK/develop and CK/amd-develop + + CK_COMMIT=$(grep 'ROCm/composable_kernel' requirements.txt | sed -E 's/.*@([a-f0-9]{40}).*/\1/') + echo "Fetching CK build ID for commit $CK_COMMIT" + CK_COMMIT_URL="$GH_API/composable_kernel/commits/${CK_COMMIT}" + PARENT_COMMIT=$(curl -s $CK_COMMIT_URL | jq '.parents[1].sha' | tr -d '"') + echo "Found parent commit: $PARENT_COMMIT" + PARENT_CHECKS_URL="$GH_API/composable_kernel/commits/${PARENT_COMMIT}/check-runs" + CK_BUILD_ID=$(curl -s $PARENT_CHECKS_URL | \ + jq '.check_runs[] | select(.name == "composable_kernel" and .app.slug == "azure-pipelines") | .details_url' | \ + tr -d '"' | grep -oP 'buildId=\K\d+') + + if [ -z "$CK_BUILD_ID" ]; then + echo "Did not find specific CK build ID" + LATEST_BUILD_URL="$AZ_API/build/builds?definitions=$(COMPOSABLE_KERNEL_PIPELINE_ID)&status=completed&result=succeeded&\$top=1&api-version=7.1" + CK_BUILD_ID=$(curl -s $LATEST_BUILD_URL | jq '.value[0].id') + echo "Found latest CK build ID: $CK_BUILD_ID" + EXIT_CODE=1 + fi + + AZURE_URL="$AZ_API/build/builds/$CK_BUILD_ID/artifacts?artifactName=$ARTIFACT_NAME&api-version=7.1" + ARTIFACT_URL=$(curl -s $AZURE_URL | jq '.resource.downloadUrl' | tr -d '"') + + if [ -z "$ARTIFACT_URL" ]; then + echo "Did not find specific CK build artifact" + LATEST_BUILD_URL="$AZ_API/build/builds?definitions=$(COMPOSABLE_KERNEL_PIPELINE_ID)&status=completed&result=succeeded&\$top=1&api-version=7.1" + CK_BUILD_ID=$(curl -s $LATEST_BUILD_URL | jq '.value[0].id') + echo "Found latest CK build ID: $CK_BUILD_ID" + AZURE_URL="$AZ_API/build/builds/$CK_BUILD_ID/artifacts?artifactName=$ARTIFACT_NAME&api-version=7.1" + ARTIFACT_URL=$(curl -s $AZURE_URL | jq '.resource.downloadUrl' | tr -d '"') + EXIT_CODE=2 + elif [ $EXIT_CODE -eq 0 ]; then + echo "Found specific CK build ID: $CK_BUILD_ID" + fi + + echo "Downloading CK artifact from $ARTIFACT_URL" + wget -nv $ARTIFACT_URL -O $(System.ArtifactsDirectory)/ck.zip + unzip $(System.ArtifactsDirectory)/ck.zip -d $(System.ArtifactsDirectory) + mkdir -p $(Agent.BuildDirectory)/rocm + tar -zxvf $(System.ArtifactsDirectory)/$ARTIFACT_NAME/*.tar.gz -C $(Agent.BuildDirectory)/rocm + rm -r $(System.ArtifactsDirectory)/ck.zip $(System.ArtifactsDirectory)/$ARTIFACT_NAME + + if [ $EXIT_CODE -ne 0 ]; then + BUILD_COMMIT=$(curl -s $AZ_API/build/builds/$CK_BUILD_ID | jq '.sourceVersion' | tr -d '"') + echo "WARNING: couldn't find a CK build for commit $CK_COMMIT" + echo "Instead used latest CK build $CK_BUILD_ID for commit $BUILD_COMMIT" + fi + exit $EXIT_CODE