test(gossipsub): Performance tests - plot docker stats (#1597)

.github/actions/add_comment/action.yml

@@ -1,12 +1,5 @@
 name: Add Comment
 description: "Add or update comment in the PR"
-inputs:
-  marker:
-    description: "Text used to find the comment to update"
-    required: true
-  markdown_path:
-    description: "Path to the file containing markdown"
-    required: true
 
 runs:
   using: "composite"
@@ -16,8 +9,8 @@ runs:
       with:
         script: |
           const fs = require('fs');
-          const marker = "${{ inputs.marker }}";
-          const body = fs.readFileSync("${{ inputs.markdown_path }}", 'utf8');
+          const marker = "${{ env.MARKER }}";
+          const body = fs.readFileSync("${{ env.COMMENT_SUMMARY_PATH }}", 'utf8');
           const { data: comments } = await github.rest.issues.listComments({
             owner: context.repo.owner,
             repo: context.repo.repo,

.github/actions/generate_plots/action.yml

@@ -0,0 +1,20 @@
+name: Generate Plots
+description: "Set up Python and run script to generate plots with Docker Stats"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+
+    - name: Install Python dependencies
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        pip install matplotlib
+
+    - name: Run plot_metrics.py
+      shell: bash
+      run: python performance/scripts/plot_docker_stats.py

.github/actions/process_stats/action.yml

@@ -0,0 +1,21 @@
+name: Process Stats
+description: "Set up Nim and run scripts to aggregate latency and process raw docker stats"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Nim
+      uses: jiro4989/setup-nim-action@v2
+      with:
+        nim-version: "2.x"
+        repo-token: ${{ env.GITHUB_TOKEN }}
+
+    - name: Aggregate latency stats and prepare markdown for comment and summary 
+      shell: bash
+      run: |
+        nim c -r -d:release -o:/tmp/process_latency_stats ./performance/scripts/process_latency_stats.nim
+
+    - name: Process raw docker stats to csv files
+      shell: bash
+      run: |
+        nim c -r -d:release -o:/tmp/process_docker_stats ./performance/scripts/process_docker_stats.nim

.github/actions/publish_plots/action.yml

@@ -0,0 +1,41 @@
+name: Publish Plots
+description: "Publish plots in performance_plots branch and add to the workflow summary"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Clone the performance_plots branch
+      uses: actions/checkout@v4
+      with:
+        repository: ${{ github.repository }}
+        ref: ${{ env.PUBLISH_BRANCH_NAME }}
+        path: ${{ env.CHECKOUT_SUBFOLDER_SUBPLOTS }}
+        fetch-depth: 0
+
+    - name: Commit & push plots
+      shell: bash
+      run: |
+        cd $CHECKOUT_SUBFOLDER_SUBPLOTS
+
+        # Remove any branch folder older than two weeks
+        find $PUBLISH_DIR_PLOTS/ -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} +
+
+        rm -rf $PUBLISH_DIR_PLOTS/$BRANCH_NAME
+        mkdir -p $PUBLISH_DIR_PLOTS/$BRANCH_NAME
+
+        cp ../$SHARED_VOLUME_PATH/*.png $PUBLISH_DIR_PLOTS/$BRANCH_NAME/
+        git add $PUBLISH_DIR_PLOTS/$BRANCH_NAME
+
+        if git diff-index --quiet HEAD --; then
+          echo "No changes to commit"
+        else
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
+          git commit -m "Update performance plots for $BRANCH_NAME"
+          git push origin $PUBLISH_BRANCH_NAME
+        fi
+
+    - name: Add plots to GitHub Actions summary
+      shell: bash
+      run: |
+        nim c -r -d:release -o:/tmp/add_plots_to_summary ./performance/scripts/add_plots_to_summary.nim

.github/workflows/performance.yml

@@ -14,7 +14,7 @@ concurrency:
 
 jobs:
   examples:
-    timeout-minutes: 10
+    timeout-minutes: 20
     strategy:
       fail-fast: false
 
@@ -22,6 +22,19 @@ jobs:
       run:
         shell: bash
 
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+      PR_NUMBER: ${{ github.event.number }}
+      BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+      MARKER: "<!-- perf-summary-marker -->"
+      COMMENT_SUMMARY_PATH: "/tmp/perf-summary.md"
+      SHARED_VOLUME_PATH: "performance/output"
+      DOCKER_STATS_PREFIX: "docker_stats_"
+      PUBLISH_BRANCH_NAME: "performance_plots"
+      CHECKOUT_SUBFOLDER_SUBPLOTS: "subplots"
+      PUBLISH_DIR_PLOTS: "plots"
+
     name: "Performance"
     runs-on: ubuntu-22.04
     steps:
@@ -47,26 +60,15 @@ jobs:
         run: |
           ./performance/runner.sh
 
-      - name: Set up Nim for aggragate script
-        uses: jiro4989/setup-nim-action@v2
-        with:
-          nim-version: "2.x"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Process latency and docker stats
+        uses: ./.github/actions/process_stats
 
-      - name: Process stats and display summary
-        env:
-          MARKER: "<!-- perf-summary-marker -->"
-          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-          COMMENT_SUMMARY_PATH: "/tmp/perf-summary.md"
-          DOCKER_STATS_DIR: "performance/output"
-          DOCKER_STATS_PREFIX: "docker_stats_"
-        run: |
-          nim c -r -d:release -o:/tmp/process_latency_stats ./performance/scripts/process_latency_stats.nim
-          nim c -r -d:release -o:/tmp/process_docker_stats ./performance/scripts/process_docker_stats.nim
+      - name: Generate plots
+        uses: ./.github/actions/generate_plots
 
-      - name: Post/Update PR Performance Comment
+      - name: Publish plots and add to summary
+        uses: ./.github/actions/publish_plots
+
+      - name: Post/Update PR comment
         if: github.event_name == 'pull_request'
         uses: ./.github/actions/add_comment
-        with:
-          marker: "<!-- perf-summary-marker -->"
-          markdown_path: "/tmp/perf-summary.md"

performance/scripts/add_plots_to_summary.nim

@@ -0,0 +1,36 @@
+import os
+import algorithm
+import sequtils
+import strformat
+import strutils
+import tables
+
+let summaryPath = getEnv("GITHUB_STEP_SUMMARY", "step_summary.md")
+let repo = getEnv("GITHUB_REPOSITORY", "vacp2p/nim-libp2p")
+let branchName = getEnv("BRANCH_NAME", "")
+let plotDir = &"subplots/plots/{branchName}"
+
+proc extractTestName(base: string): string =
+  let parts = base.split("_")
+  return parts[^2]
+
+proc makeImgTag(base: string): string =
+  &"<img src=\"https://raw.githubusercontent.com/{repo}/refs/heads/performance_plots/plots/{branchName}/{base}\" width=\"450\" style=\"margin-right:10px;\" />"
+
+var grouped: Table[string, seq[string]]
+for path in walkFiles(fmt"{plotDir}/*.png"):
+  let base = path.splitPath.tail
+  let testName = extractTestName(base)
+  let imgTag = makeImgTag(base)
+
+  discard grouped.hasKeyOrPut(testName, @[])
+  grouped[testName].add(imgTag)
+
+var summary = &"## Performance Plots for {branchName}\n"
+for test in grouped.keys.toSeq().sorted():
+  let imgs = grouped[test]
+  summary &= &"### {test}\n"
+  summary &= imgs.join(" ") & "<br>\n"
+
+writeFile(summaryPath, summary)
+echo summary

performance/scripts/plot_docker_stats.py

@@ -0,0 +1,126 @@
+import os
+import glob
+import csv
+import statistics
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+
+def parse_csv(filepath):
+    timestamps = []
+    cpu_percent = []
+    mem_usage_mb = []
+    download_MBps = []
+    upload_MBps = []
+    download_MB = []
+    upload_MB = []
+    with open(filepath, "r") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            timestamps.append(float(row["timestamp"]))
+            cpu_percent.append(float(row["cpu_percent"]))
+            mem_usage_mb.append(float(row["mem_usage_mb"]))
+            download_MBps.append(float(row["download_MBps"]))
+            upload_MBps.append(float(row["upload_MBps"]))
+            download_MB.append(float(row["download_MB"]))
+            upload_MB.append(float(row["upload_MB"]))
+    return {
+        "timestamps": timestamps,
+        "cpu_percent": cpu_percent,
+        "mem_usage_mb": mem_usage_mb,
+        "download_MBps": download_MBps,
+        "upload_MBps": upload_MBps,
+        "download_MB": download_MB,
+        "upload_MB": upload_MB,
+    }
+
+
+def plot_metrics(data, title, output_path):
+    timestamps = data["timestamps"]
+    time_points = [t - timestamps[0] for t in timestamps]
+    cpu = data["cpu_percent"]
+    mem = data["mem_usage_mb"]
+    download_MBps = data["download_MBps"]
+    upload_MBps = data["upload_MBps"]
+    download_MB = data["download_MB"]
+    upload_MB = data["upload_MB"]
+
+    cpu_median = statistics.median(cpu)
+    cpu_max = max(cpu)
+    mem_median = statistics.median(mem)
+    mem_max = max(mem)
+    download_MBps_median = statistics.median(download_MBps)
+    download_MBps_max = max(download_MBps)
+    upload_MBps_median = statistics.median(upload_MBps)
+    upload_MBps_max = max(upload_MBps)
+    download_MB_total = download_MB[-1]
+    upload_MB_total = upload_MB[-1]
+
+    fig, (ax1, ax2, ax3, ax4) = plt.subplots(4, 1, figsize=(12, 16), sharex=True)
+    fig.suptitle(title, fontsize=16)
+
+    # CPU Usage
+    ax1.plot(time_points, cpu, "b-", label=f"CPU Usage (%)\nmedian = {cpu_median:.2f}\nmax = {cpu_max:.2f}")
+    ax1.set_ylabel("CPU Usage (%)")
+    ax1.set_title("CPU Usage Over Time")
+    ax1.grid(True)
+    ax1.set_xlim(left=0)
+    ax1.set_ylim(bottom=0)
+    ax1.legend(loc="best")
+
+    # Memory Usage
+    ax2.plot(time_points, mem, "m-", label=f"Memory Usage (MB)\nmedian = {mem_median:.2f} MB\nmax = {mem_max:.2f} MB")
+    ax2.set_ylabel("Memory Usage (MB)")
+    ax2.set_title("Memory Usage Over Time")
+    ax2.grid(True)
+    ax2.set_xlim(left=0)
+    ax2.set_ylim(bottom=0)
+    ax2.legend(loc="best")
+
+    # Network Throughput
+    ax3.plot(
+        time_points,
+        download_MBps,
+        "c-",
+        label=f"Download (MB/s)\nmedian = {download_MBps_median:.2f} MB/s\nmax = {download_MBps_max:.2f} MB/s",
+        linewidth=2,
+    )
+    ax3.plot(
+        time_points, upload_MBps, "r-", label=f"Upload (MB/s)\nmedian = {upload_MBps_median:.2f} MB/s\nmax = {upload_MBps_max:.2f} MB/s", linewidth=2
+    )
+    ax3.set_ylabel("Network Throughput (MB/s)")
+    ax3.set_title("Network Activity Over Time")
+    ax3.grid(True)
+    ax3.set_xlim(left=0)
+    ax3.set_ylim(bottom=0)
+    ax3.legend(loc="best", labelspacing=2)
+
+    # Accumulated Network Data
+    ax4.plot(time_points, download_MB, "c-", label=f"Download (MB), total: {download_MB_total:.2f} MB", linewidth=2)
+    ax4.plot(time_points, upload_MB, "r-", label=f"Upload (MB), total: {upload_MB_total:.2f} MB", linewidth=2)
+    ax4.set_xlabel("Time (seconds)")
+    ax4.set_ylabel("Total Data Transferred (MB)")
+    ax4.set_title("Accumulated Network Data Over Time")
+    ax4.grid(True)
+    ax4.set_xlim(left=0)
+    ax4.set_ylim(bottom=0)
+    ax4.legend(loc="best")
+
+    plt.tight_layout(rect=(0, 0, 1, 1))
+    os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True)
+    plt.savefig(output_path, dpi=100, bbox_inches="tight")
+    plt.close(fig)
+    print(f"Saved plot to {output_path}")
+
+
+if __name__ == "__main__":
+    docker_stats_dir = os.environ.get("SHARED_VOLUME_PATH", "performance/output")
+    docker_stats_prefix = os.environ.get("DOCKER_STATS_PREFIX", "docker_stats_")
+    glob_pattern = os.path.join(docker_stats_dir, f"{docker_stats_prefix}*.csv")
+    csv_files = glob.glob(glob_pattern)
+    for csv_file in csv_files:
+        file_name = os.path.splitext(os.path.basename(csv_file))[0]
+        data = parse_csv(csv_file)
+        plot_metrics(data, title=file_name, output_path=os.path.join(docker_stats_dir, f"{file_name}.png"))

performance/scripts/process_docker_stats.nim

@@ -138,7 +138,7 @@ proc findInputFiles(dir: string, prefix: string): seq[string] =
   return files
 
 proc main() =
-  let dir = getEnv("DOCKER_STATS_DIR", "performance/output")
+  let dir = getEnv("SHARED_VOLUME_PATH", "performance/output")
   let prefix = getEnv("DOCKER_STATS_PREFIX", "docker_stats_")
 
   let inputFiles = findInputFiles(dir, prefix)

performance/scripts/process_latency_stats.nim

@@ -93,7 +93,8 @@ proc getMarkdownReport*(
   output.add marker & "\n"
   output.add "# 🏁 **Performance Summary**\n"
 
-  output.add fmt"**Commit:** `{commitSha}`"
+  let commitUrl = fmt"https://github.com/vacp2p/nim-libp2p/commit/{commitSha}"
+  output.add fmt"**Commit:** [`{commitSha}`]({commitUrl})"
 
   output.add "| Scenario | Nodes | Total messages sent | Total messages received | Latency min (ms) | Latency max (ms) | Latency avg (ms) |"
   output.add "|:---:|:---:|:---:|:---:|:---:|:---:|:---:|"
@@ -102,8 +103,13 @@ proc getMarkdownReport*(
     let nodes = validNodes[scenarioName]
     output.add fmt"| {stats.scenarioName} | {nodes} | {stats.totalSent} | {stats.totalReceived} | {stats.latency.minLatencyMs:.3f} | {stats.latency.maxLatencyMs:.3f} | {stats.latency.avgLatencyMs:.3f} |"
 
-  let markdown = output.join("\n")
+  let runId = getEnv("GITHUB_RUN_ID", "")
+  let summaryUrl = fmt"https://github.com/vacp2p/nim-libp2p/actions/runs/{runId}"
+  output.add(
+    fmt"### 📊 View full Container Resources stats in the [Workflow Summary]({summaryUrl})"
+  )
 
+  let markdown = output.join("\n")
   return markdown
 
 proc main() =