From 37e1ef1be365ce2c2127aa538a39bc8e1da7339b Mon Sep 17 00:00:00 2001
From: wozeparrot <wozeparrot@gmail.com>
Date: Thu, 5 Jun 2025 15:41:00 -0700
Subject: [PATCH] feat: cleanup old AM processes (#10653)

---
 .github/workflows/benchmark.yml        | 6 ++++++
 .github/workflows/benchmark_search.yml | 2 ++
 .github/workflows/mlperf.yml           | 2 ++
 3 files changed, 10 insertions(+)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 95bddd99ff..40098291fd 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -354,6 +354,8 @@ jobs:
       uses: actions/checkout@v4
     - name: Remove amdgpu
       run: sudo rmmod amdgpu || true
+    - name: Cleanup running AM processes
+      run: python extra/amdpci/am_smi.py --pids --kill
     #- name: Insert amdgpu
     #  run: sudo modprobe amdgpu
     - name: Symlink models and datasets
@@ -482,6 +484,8 @@ jobs:
       uses: actions/checkout@v4
     - name: Remove amdgpu
       run: sudo rmmod amdgpu || true
+    - name: Cleanup running AM processes
+      run: python extra/amdpci/am_smi.py --pids --kill
     - name: Symlink models and datasets
       run: |
         mkdir -p weights
@@ -543,6 +547,8 @@ jobs:
       uses: actions/checkout@v4
     - name: Remove amdgpu
       run: sudo rmmod amdgpu || true
+    - name: Cleanup running AM processes
+      run: python extra/amdpci/am_smi.py --pids --kill
     - name: Symlink models and datasets
       run: |
         mkdir -p weights
diff --git a/.github/workflows/benchmark_search.yml b/.github/workflows/benchmark_search.yml
index e4ee07e699..670fa6448d 100644
--- a/.github/workflows/benchmark_search.yml
+++ b/.github/workflows/benchmark_search.yml
@@ -16,6 +16,8 @@ jobs:
       uses: actions/checkout@v4
     - name: Remove amdgpu
       run: sudo rmmod amdgpu || true
+    - name: Cleanup running AM processes
+      run: python extra/amdpci/am_smi.py --pids --kill
     - name: Run SDXL with new search
       run: |
         BENCHMARK_LOG=search_sdxl PYTHONPATH=. AMD=1 JITBEAM=2 IGNORE_BEAM_CACHE=1 DISABLE_COMPILER_CACHE=1 python examples/sdxl.py --noshow --timing --seed 0
diff --git a/.github/workflows/mlperf.yml b/.github/workflows/mlperf.yml
index 8575db1239..83876e6a96 100644
--- a/.github/workflows/mlperf.yml
+++ b/.github/workflows/mlperf.yml
@@ -16,6 +16,8 @@ jobs:
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
+    - name: Cleanup running AM processes
+      run: python extra/amdpci/am_smi.py --pids --kill
     - name: Symlink datasets
       run: |
         mkdir -p extra/datasets