Update checkout.yml

.azuredevops/components/MIOpen.yml

@@ -42,11 +42,15 @@ parameters:
     - python3-pip
     - python3-venv
     - software-properties-common
+    - wget
     - zip
 - name: pipModules
   type: object
   default:
     - cget
+- name: largeFiles
+  type: boolean
+  default: true
 - name: rocmDependencies
   type: object
   default:
@@ -131,12 +135,14 @@ jobs:
       parameters:
         aptPackages: ${{ parameters.aptPackages }}
         pipModules: ${{ parameters.pipModules }}
+        largeFiles: ${{ parameters.largeFiles }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
       parameters:
         checkoutRepo: ${{ parameters.checkoutRepo }}
         sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }}
+        largeFiles: ${{ parameters.largeFiles }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml
       parameters:
         gpuTarget: ${{ job.target }}
@@ -212,12 +218,14 @@ jobs:
       parameters:
         aptPackages: ${{ parameters.aptPackages }}
         pipModules: ${{ parameters.pipModules }}
+        largeFiles: ${{ parameters.largeFiles }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-cmake-latest.yml
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/preamble.yml
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/checkout.yml
       parameters:
         checkoutRepo: ${{ parameters.checkoutRepo }}
         sparseCheckoutDir: ${{ parameters.sparseCheckoutDir }}
+        largeFiles: ${{ parameters.largeFiles }}
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/dependencies-aqlprofile.yml
     - template: ${{ variables.CI_TEMPLATE_PATH }}/steps/miopen-get-ck-build.yml
       parameters:

.azuredevops/templates/steps/checkout.yml

@@ -12,6 +12,10 @@ parameters:
 - name: submoduleBehaviour
   type: string
   default: 'true'
+# determines if dvc should be used or not
+- name: largeFiles
+  type: boolean
+  default: true
 
 steps:
   - checkout: ${{ parameters.checkoutRepo }}
@@ -20,7 +24,7 @@ steps:
     retryCountOnTaskFailure: 3
     fetchFilter: blob:none
     ${{ if ne(parameters.sparseCheckoutDir, '') }}:
-      sparseCheckoutDirectories: ${{ parameters.sparseCheckoutDir }} shared
+      sparseCheckoutDirectories: ${{ parameters.sparseCheckoutDir }} shared .dvc .dvcignore .gitignore
       path: sparse
   - ${{ if ne(parameters.sparseCheckoutDir, '') }}:
     - task: Bash@3
@@ -28,3 +32,13 @@ steps:
       inputs:
         targetType: inline
         script: ln -s $(Agent.BuildDirectory)/sparse/${{ parameters.sparseCheckoutDir }} $(Agent.BuildDirectory)/s
+  - ${{ if eq(parameters.largeFiles, true) }}:
+    - task: Bash@3
+      displayName: dvc pull
+      inputs:
+        targetType: inline
+        script: dvc pull
+        ${{ if eq(parameters.sparseCheckoutDir, '') }}:
+          workingDirectory: $(Agent.BuildDirectory)/s
+        ${{ else }}:
+          workingDirectory: $(Agent.BuildDirectory)/sparse/${{ parameters.sparseCheckoutDir }}

.azuredevops/templates/steps/dependencies-other.yml

@@ -1,5 +1,8 @@
 # download and install non-rocm dependencies through apt and pip
 parameters:
+- name: os
+  type: string
+  default: 'ubuntu2204'
 - name: aptPackages
   type: object
   default: []
@@ -12,6 +15,9 @@ parameters:
 - name: packageManager
   type: string
   default: apt
+- name: largeFiles
+  type: boolean
+  default: true
 
 steps:
 - ${{ if eq(parameters.packageManager, 'apt') }}:
@@ -31,3 +37,18 @@ steps:
     inputs:
       targetType: inline
       script: python3 -m pip install -v --force-reinstall ${{ join(' ', parameters.pipModules) }}
+- ${{ if eq(parameters.largeFiles, true) }}:
+  - task: Bash@3
+    displayName: install dvc
+    inputs:
+      targetType: inline
+      ${{ if ne(parameters.os, 'almalinux8') }}:
+        script: |
+          wget https://dvc.org/download/linux-deb/dvc-3.62.0 -O ./dvc_3.62.0_amd64.deb
+          sudo DEBIAN_FRONTEND=noninteractive apt-get --yes --fix-missing install ./dvc_3.62.0_amd64.deb
+          sudo rm -f ./dvc_3.62.0_amd64.deb
+      ${{ else }}:
+        script: |
+          wget https://dvc.org/download/linux-rpm/dvc-3.62.0 -O ./dvc-3.62.0-1.x86_64.rpm
+          sudo DEBIAN_FRONTEND=noninteractive apt-get --yes --fix-missing install ./dvc-3.62.0-1.x86_64.rpm
+          sudo rm -f ./dvc-3.62.0-1.x86_64.rpm