Merge branch 'main' into ray/enforce-coverage-soft

.github/workflows/py-coverage-comment.yml

@@ -0,0 +1,66 @@
+name: Python coverage comment
+# This allows tests to post coverage comments when the PR is from a fork.
+# It uses the artifact `python-coverage-comment-action`.
+
+on:
+  workflow_run:
+    workflows: ["Run Python Tests"]
+    types:
+      - completed
+
+jobs:
+  test:
+    name: Run tests & display coverage
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
+    permissions:
+      # Permission to publish new comments in PRs.
+      pull-requests: write
+      # Permissions to edit existing comments.
+      contents: write
+      # Permission to download the artifact.
+      actions: read
+    steps:
+      # DO NOT run actions/checkout here, for security reasons
+      # For details, refer to https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+      - name: Post comment
+        uses: py-cov-action/python-coverage-comment-action@v3
+        with:
+          GITHUB_TOKEN: ${{ github.token }}
+          GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }}
+          ANNOTATE_MISSING_LINES: true
+
+      - name: Download low-coverage-python artifact if present
+        id: download
+        uses: actions/download-artifact@v5
+        with:
+          name: low-coverage-python
+          run-id: ${{ github.event.workflow_run.id }}
+          path: ./low-coverage-python
+          github-token: ${{ github.token }}
+        continue-on-error: true
+
+      - name: Check if low-coverage-python found
+        id: check_low_coverage_artifact
+        run: |
+          if [ -f low-coverage-python/coverage-stats.txt ]; then
+            echo "found=true" >> $GITHUB_OUTPUT
+          else
+            echo "found=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Add label if uncovered code was modified
+        if: ${{ steps.check_low_coverage_artifact.outputs.found == 'true' && github.event.workflow_run.pull_requests[0] }}
+        uses: actions-ecosystem/action-add-labels@v1
+        with:
+          github_token: ${{ github.token }}
+          labels: low-coverage-python
+          number: ${{ github.event.workflow_run.pull_requests[0].number }}
+
+      - name: Remove label if uncovered code was not modified
+        if: ${{ steps.check_low_coverage_artifact.outputs.found == 'false' && github.event.workflow_run.pull_requests[0] }}
+        uses: actions-ecosystem/action-remove-labels@v1
+        with:
+          github_token: ${{ github.token }}
+          labels: low-coverage-python
+          number: ${{ github.event.workflow_run.pull_requests[0].number }}

.github/workflows/py-tests.yml

@@ -173,10 +173,8 @@ jobs:
           path: ".coverage.openhands-cli.${{ matrix.python-version }}"
           include-hidden-files: true
 
-
   coverage-comment:
     name: Coverage Comment
-    if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     needs: [test-on-linux, test-enterprise, test-cli-python]
 
@@ -196,8 +194,32 @@ jobs:
         run: ln -sf openhands-cli/openhands_cli openhands_cli
 
       - name: Coverage comment
+        # In PR mode leaves a comment or file of one.
+        # On non-PR records coverage in branch python-coverage-comment-action-data.
         id: coverage_comment
         uses: py-cov-action/python-coverage-comment-action@v3
         with:
           GITHUB_TOKEN: ${{ github.token }}
           MERGE_COVERAGE_FILES: true
+
+      - name: Store Pull Request comment to be posted
+        uses: actions/upload-artifact@v4
+        if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
+        with:
+          name: python-coverage-comment-action
+          path: python-coverage-comment-action.txt
+
+      - name: Record stats on coverage change
+        shell: bash
+        run: |
+          # Docs: https://github.com/marketplace/actions/python-coverage-comment
+          echo "diff_total_num_violations: ${{ steps.coverage_comment.outputs.diff_total_num_violations }}" >> coverage-stats.txt
+          echo "diff_total_percent_covered: ${{ steps.coverage_comment.outputs.diff_total_percent_covered}}" >> coverage-stats.txt
+          echo "new_percent_covered: ${{ steps.coverage_comment.outputs.new_percent_covered}}" >> coverage-stats.txt
+
+      - name: Store artifact to mark potential untested code modified
+        uses: actions/upload-artifact@v4
+        if: ${{ github.event_name == 'pull_request' && fromJSON(steps.coverage_comment.outputs.diff_total_num_violations) > 0 && steps.coverage_comment.outputs.diff_total_percent_covered < steps.coverage_comment.outputs.new_percent_covered }}
+        with:
+          name: low-coverage-python
+          path: coverage-stats.txt