AutoGPT/.github/workflows/arena-intake.yml

name: Arena intake

on:
  # We recommend `pull_request_target` so that github secrets are available.
  # In `pull_request` we wouldn't be able to change labels of fork PRs
  pull_request_target:
    types: [ opened, synchronize ]
    paths:
      - 'arena/**'

jobs:
  check:
    permissions:
      pull-requests: write
    runs-on: ubuntu-latest
    steps:
    - name: Checkout PR
      uses: actions/checkout@v4
      with:
        ref: ${{ github.event.pull_request.head.sha }}

    - name: Check Arena entry
      uses: actions/github-script@v7
      with:
        script: |
          console.log('⚙️ Setting up...');

          const fs = require('fs');
          const path = require('path');

          const pr = context.payload.pull_request;
          const isFork = pr.head.repo.fork;

          console.log('🔄️ Fetching PR diff metadata...');
          const prFilesChanged = (await github.rest.pulls.listFiles({
            owner: context.repo.owner,
            repo: context.repo.repo,
            pull_number: pr.number,
          })).data;
          console.debug(prFilesChanged);
          const arenaFilesChanged = prFilesChanged.filter(
            ({ filename: file }) => file.startsWith('arena/') && file.endsWith('.json')
          );
          const hasChangesInAutogptsFolder = prFilesChanged.some(
            ({ filename }) => filename.startsWith('autogpts/')
          );

          console.log(`🗒️ ${arenaFilesChanged.length} arena entries affected`);
          console.debug(arenaFilesChanged);
          if (arenaFilesChanged.length === 0) {
            // If no files in `arena/` are changed, this job does not need to run.
            return;
          }

          let close = false;
          let flagForManualCheck = false;
          let issues = [];

          if (isFork) {
            if (arenaFilesChanged.length > 1) {
              // Impacting multiple entries in `arena/` is not allowed
              issues.push('This pull request impacts multiple arena entries');
            }
            if (hasChangesInAutogptsFolder) {
              // PRs that include the custom agent are generally not allowed
              issues.push(
                'This pull request includes changes in `autogpts/`.\n'
                + 'Please make sure to only submit your arena entry (`arena/*.json`), '
                + 'and not to accidentally include your custom agent itself.'
              );
            }
          }

          if (arenaFilesChanged.length === 1) {
            const newArenaFile = arenaFilesChanged[0]
            const newArenaFileName = path.basename(newArenaFile.filename)
            console.log(`🗒️ Arena entry in PR: ${newArenaFile}`);

            if (newArenaFile.status != 'added') {
              flagForManualCheck = true;
            }

            if (pr.mergeable != false) {
              const newArenaEntry = JSON.parse(fs.readFileSync(newArenaFile.filename));
              const allArenaFiles = await (await glob.create('arena/*.json')).glob();
              console.debug(newArenaEntry);

              console.log(`➡️ Checking ${newArenaFileName} against existing entries...`);
              for (const file of allArenaFiles) {
                const existingEntryName = path.basename(file);

                if (existingEntryName === newArenaFileName) {
                  continue;
                }

                console.debug(`Checking against ${existingEntryName}...`);

                const arenaEntry = JSON.parse(fs.readFileSync(file));
                if (arenaEntry.github_repo_url === newArenaEntry.github_repo_url) {
                  console.log(`⚠️ Duplicate detected: ${existingEntryName}`);
                  issues.push(
                    `The \`github_repo_url\` specified in __${newArenaFileName}__ `
                    + `already exists in __${existingEntryName}__. `
                    + `This PR will be closed as duplicate.`
                  )
                  close = true;
                }
              }
            } else {
              console.log('⚠️ PR has conflicts');
              issues.push(
                `__${newArenaFileName}__ conflicts with existing entry with the same name`
              )
              close = true;
            }
          }  // end if (arenaFilesChanged.length === 1)

          console.log('🏁 Finished checking against existing entries');

          if (issues.length == 0) {
            console.log('✅ No issues detected');
            if (flagForManualCheck) {
              console.log('🤔 Requesting review from maintainers...');
              await github.rest.pulls.requestReviewers({
                owner: context.repo.owner,
                repo: context.repo.repo,
                pull_number: pr.number,
                reviewers: ['Pwuts'],
                // team_reviewers: ['maintainers'],  // doesn't work: https://stackoverflow.com/a/64977184/4751645
              });
            } else {
              console.log('➡️ Approving PR...');
              await github.rest.pulls.createReview({
                owner: context.repo.owner,
                repo: context.repo.repo,
                pull_number: pr.number,
                event: 'APPROVE',
              });
            }
          } else {
            console.log(`⚠️ ${issues.length} issues detected`);

            console.log('➡️ Posting comment indicating issues...');
            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: pr.number,
              body: `Our automation found one or more issues with this submission:\n`
                + issues.map(i => `- ${i.replace('\n', '\n  ')}`).join('\n'),
            });

            console.log("➡️ Applying label 'invalid'...");
            await github.rest.issues.addLabels({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: pr.number,
              labels: ['invalid'],
            });

            if (close) {
              console.log('➡️ Auto-closing PR...');
              await github.rest.pulls.update({
                owner: context.repo.owner,
                repo: context.repo.repo,
                pull_number: pr.number,
                state: 'closed',
              });
            }
          }