ROCm/.azuredevops/nightly/rocm-nightly.yml

parameters:
# using a parameter instead of a strategy matrix allows job variables to be eval'd at compile-time rather than runtime
- name: jobList
  type: object
  default:
    - { os: ubuntu2204, packageManager: apt, target: gfx942 }
    - { os: ubuntu2204, packageManager: apt, target: gfx90a }
    - { os: ubuntu2204, packageManager: apt, target: gfx1201 }
    - { os: ubuntu2204, packageManager: apt, target: gfx1100 }
    - { os: ubuntu2204, packageManager: apt, target: gfx1030 }
    - { os: ubuntu2404, packageManager: apt, target: gfx942 }
    - { os: ubuntu2404, packageManager: apt, target: gfx90a }
    - { os: ubuntu2404, packageManager: apt, target: gfx1201 }
    - { os: ubuntu2404, packageManager: apt, target: gfx1100 }
    - { os: ubuntu2404, packageManager: apt, target: gfx1030 }
    - { os: almalinux8, packageManager: dnf, target: gfx942 }
    - { os: almalinux8, packageManager: dnf, target: gfx90a }
    - { os: almalinux8, packageManager: dnf, target: gfx1201 }
    - { os: almalinux8, packageManager: dnf, target: gfx1100 }
    - { os: almalinux8, packageManager: dnf, target: gfx1030 }
- name: rocmDependencies
  type: object
  default:
    - AMDMIGraphX
    - amdsmi
    - aomp-extras
    - aomp
    - clr
    - composable_kernel
    - half
    - hip-tests
    - hipBLAS
    - hipBLAS-common
    - hipBLASLt
    - hipCUB
    - hipFFT
    - hipfort
    - HIPIFY
    - hipRAND
    - hipSOLVER
    - hipSPARSE
    - hipSPARSELt
    - hipTensor
    - llvm-project
    - MIOpen
    - MIVisionX
    - rccl
    - rdc
    - rocAL
    - rocALUTION
    - rocBLAS
    - ROCdbgapi
    - rocDecode
    - rocFFT
    - ROCgdb
    - rocJPEG
    - rocm-cmake
    - rocm-core
    - rocm-examples
    - rocminfo
    - rocMLIR
    - ROCmValidationSuite
    - rocm_bandwidth_test
    - rocm_smi_lib
    - rocPRIM
    - rocprofiler-compute
    - rocprofiler-register
    - rocprofiler-sdk
    - rocprofiler-systems
    - rocprofiler
    - rocPyDecode
    - ROCR-Runtime
    - rocRAND
    - rocr_debug_agent
    - rocSOLVER
    - rocSPARSE
    - rocThrust
    - roctracer
    - rocWMMA
    - rpp
    - TransferBench

trigger: none
pr: none
schedules:
- cron: '30 7 * * *'
  displayName: Nightly build
  branches:
    include:
    - develop
  always: true

jobs:
- ${{ each job in parameters.jobList }}:
  - job: nightly_${{ job.os }}_${{ job.target }}
    timeoutInMinutes: 120
    variables:
    - group: common
    - template: /.azuredevops/variables-global.yml
    pool: ${{ variables.MEDIUM_BUILD_POOL }}
    workspace:
      clean: all
    steps:
    - task: DeleteFiles@1
      displayName: 'Cleanup checkout space'
      inputs:
        SourceFolder: '$(Agent.BuildDirectory)/s'
        Contents: '**/*'
    - task: DeleteFiles@1
      displayName: 'Cleanup Staging Area'
      inputs:
        SourceFolder: '$(Build.ArtifactStagingDirectory)'
        Contents: '/**/*'
        RemoveDotFiles: true
    - script: df -h
      displayName: System disk space before ROCm
    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-rocm.yml
      parameters:
        dependencyList: ${{ parameters.rocmDependencies }}
        os: ${{ job.os }}
        gpuTarget: ${{ job.target }}
    - script: df -h
      displayName: System disk space after ROCm
    - script: du -sh $(Agent.BuildDirectory)/rocm
      displayName: Uncompressed ROCm size
    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/manifest.yml
    - task: ArchiveFiles@2
      displayName: Compress rocm-nightly
      inputs:
        rootFolderOrFile: $(Agent.BuildDirectory)/rocm
        includeRootFolder: false
        archiveType: tar
        tarCompression: gz
        archiveFile: $(Build.ArtifactStagingDirectory)/$(Build.DefinitionName)_$(Build.BuildNumber)_${{ job.os }}_${{ job.target }}.tar.gz
    - script: du -sh $(Build.ArtifactStagingDirectory)
      displayName: Compressed ROCm size
    - task: PublishPipelineArtifact@1
      displayName: 'Publish ROCm Nightly Artifact'
      retryCountOnTaskFailure: 3
      inputs:
        targetPath: '$(Build.ArtifactStagingDirectory)'
    - task: Bash@3
      displayName: Save pipeline artifact file name
      inputs:
        workingDirectory: $(Pipeline.Workspace)
        targetType: inline
        script: echo "$(Build.DefinitionName)_$(Build.BuildNumber)_${{ job.os }}_${{ job.target }}.tar.gz" >> pipelineArtifacts.txt
    - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/artifact-links.yml
    - ${{ if eq(job.packageManager, 'apt') }}:
      - task: Bash@3
        displayName: Create Dockerfile
        inputs:
          workingDirectory: $(Agent.BuildDirectory)
          targetType: inline
          script: |
            cat <<'EOF' > Dockerfile
              ${{ iif(eq(job.os, 'ubuntu2204'), 'FROM ubuntu:22.04', '') }}
              ${{ iif(eq(job.os, 'ubuntu2404'), 'FROM ubuntu:24.04', '') }}

              WORKDIR /root
              RUN mkdir rocm

              RUN apt update \
                && apt upgrade -y \
                && apt install -y cmake curl git gcc g++ gpg lsb-release lsof ninja-build pkg-config python3 python3-pip wget zip libdrm-dev libelf-dev libgtest-dev libhsakmt-dev libhwloc-dev libnuma-dev libstdc++-12-dev libtbb-dev jq \
                && apt clean all

              RUN PACKAGE_NAME=$(curl -s https://repo.radeon.com/rocm/apt/latest/pool/main/h/hsa-amd-aqlprofile/ | grep -oP "href=\"\K[^\"]*$(lsb_release -rs)[^\"]*\.deb") \
                && wget -nv --retry-connrefused https://repo.radeon.com/rocm/apt/latest/pool/main/h/hsa-amd-aqlprofile/$PACKAGE_NAME \
                && mkdir hsa-amd-aqlprofile \
                && dpkg-deb -R $PACKAGE_NAME hsa-amd-aqlprofile \
                && cp -R hsa-amd-aqlprofile/opt/rocm-*/* rocm

              RUN ARTIFACT_URL="https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds/$(Build.BuildId)/artifacts?artifactName=nightly${{ job.os }}${{ job.target }}&api-version=7.1" \
                && DOWNLOAD_URL=$(curl -s $ARTIFACT_URL | jq ".resource.downloadUrl" | tr -d '"') \
                && wget -nv --retry-connrefused $DOWNLOAD_URL -O nightly.zip \
                && unzip nightly.zip \
                && tar -xf nightly${{ job.os }}${{ job.target }}/rocm-nightly*${{ job.os }}*${{ job.target }}*.tar.gz -C rocm

              RUN echo /root/rocm/lib | tee /etc/ld.so.conf.d/rocm-ci.conf
              RUN echo /root/rocm/llvm/lib | tee -a /etc/ld.so.conf.d/rocm-ci.conf
              RUN echo /root/rocm/lib64 | tee -a /etc/ld.so.conf.d/rocm-ci.conf
              RUN echo /root/rocm/llvm/lib64 | tee -a /etc/ld.so.conf.d/rocm-ci.conf
              RUN ldconfig -v
              ENV PATH="$PATH:/root/rocm/bin"
              ENTRYPOINT ["/bin/bash"]
            EOF
            cat Dockerfile
    - ${{ elseif eq(job.packageManager, 'dnf') }}:
      - task: Bash@3
        displayName: Create Dockerfile
        inputs:
          workingDirectory: $(Agent.BuildDirectory)
          targetType: inline
          script: |
            cat <<'EOF' > Dockerfile
              ${{ iif(eq(job.os, 'almalinux8'), 'FROM almalinux:8', '') }}

              WORKDIR /root
              RUN mkdir rocm

              RUN dnf install -y cmake curl git gcc gcc-c++ gnupg2 redhat-lsb-core lsof pkgconf python3 python3-pip wget zip libdrm-devel elfutils-libelf-devel numactl-devel libstdc++-devel tbb-devel jq \
                && dnf clean all

              RUN PACKAGE_NAME=$(curl -s https://repo.radeon.com/rocm/rhel8/$(REPO_RADEON_VERSION)/main/ | grep -oP "hsa-amd-aqlprofile-[^\"]+\.rpm" | head -n1) \
                && wget -nv --retry-connrefused https://repo.radeon.com/rocm/rhel8/$(REPO_RADEON_VERSION)/main/$PACKAGE_NAME \
                && mkdir hsa-amd-aqlprofile \
                && dnf -y install rpm-build cpio \
                && rpm2cpio $PACKAGE_NAME | (cd hsa-amd-aqlprofile && cpio -idmv) \
                && cp -R hsa-amd-aqlprofile/opt/rocm-*/* rocm

              RUN ARTIFACT_URL="https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds/$(Build.BuildId)/artifacts?artifactName=nightly${{ job.os }}${{ job.target }}&api-version=7.1" \
                && DOWNLOAD_URL=$(curl -s $ARTIFACT_URL | jq ".resource.downloadUrl" | tr -d '"') \
                && wget -nv --retry-connrefused $DOWNLOAD_URL -O nightly.zip \
                && UNZIP_DISABLE_ZIPBOMB_DETECTION=TRUE unzip nightly.zip \
                && tar -xf nightly${{ job.os }}${{ job.target }}/rocm-nightly*${{ job.os }}*${{ job.target }}*.tar.gz -C rocm

              RUN echo /root/rocm/lib | tee /etc/ld.so.conf.d/rocm-ci.conf
              RUN echo /root/rocm/llvm/lib | tee -a /etc/ld.so.conf.d/rocm-ci.conf
              RUN echo /root/rocm/lib64 | tee -a /etc/ld.so.conf.d/rocm-ci.conf
              RUN echo /root/rocm/llvm/lib64 | tee -a /etc/ld.so.conf.d/rocm-ci.conf
              RUN ldconfig -v
              ENV PATH="$PATH:/root/rocm/bin"
              ENTRYPOINT ["/bin/bash"]
            EOF
            cat Dockerfile
    - task: Docker@2
      displayName: Build and upload Docker image
      retryCountOnTaskFailure: 3
      inputs:
        containerRegistry: ContainerService3
        repository: 'nightly-${{ job.os }}-${{ job.target }}'
        Dockerfile: '$(Agent.BuildDirectory)/Dockerfile'
        buildContext: '$(Agent.BuildDirectory)'
    - task: Bash@3
      displayName: '!! Docker Run Command !!'
      inputs:
        targetType: inline
        script: echo "docker run -it --network=host --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined rocmexternalcicd.azurecr.io/nightly-${{ job.os }}-${{ job.target }}:$(Build.BuildId)" | tr '[:upper:]' '[:lower:]'