From b8f6c36fad3d5a94ce156b0dc5eeb924adc34d26 Mon Sep 17 00:00:00 2001 From: Arunavo Ray Date: Wed, 2 Apr 2025 13:49:50 +0530 Subject: [PATCH] add support for mirroring GitHub issues, starred repositories, and organization repositories --- README.md | 22 +++- src/configuration.mjs | 11 ++ src/get-github-repositories.mjs | 59 +++++++++- src/index.mjs | 192 +++++++++++++++++++++++--------- 4 files changed, 229 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 9153bbd..39f63c2 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,11 @@ The mirror settings are default by your gitea instance. It is also possible to mirror private repos, which can be configred here in [#parameters](#parameters). When mirroring private repos, they will be created as private repos on your gitea server. +Additionally, you can now mirror: +- Issues from GitHub repositories (including labels) +- Starred repositories from your GitHub account +- Repositories from organizations you belong to + ## Prerequisites - A github user or organization with repositories @@ -39,8 +44,11 @@ All configuration is performed through environment variables. Flags are consider | GITHUB_USERNAME | yes | string | - | The name of the GitHub user or organisation to mirror. | | GITEA_URL | yes | string | - | The url of your Gitea server. | | GITEA_TOKEN | yes | string | - | The token for your gitea user (Settings -> Applications -> Generate New Token). **Attention: if this is set, the token will be transmitted to your specified Gitea instance!** | -| GITHUB_TOKEN | no* | string | - | GitHub token (PAT). Is mandatory in combination with `MIRROR_PRIVATE_REPOSITORIES`. | +| GITHUB_TOKEN | no* | string | - | GitHub token (PAT). Is mandatory in combination with `MIRROR_PRIVATE_REPOSITORIES`, `MIRROR_ISSUES`, `MIRROR_STARRED`, or `MIRROR_ORGANIZATIONS`. | | MIRROR_PRIVATE_REPOSITORIES | no | bool | FALSE | If set to `true` your private GitHub Repositories will be mirrored to Gitea. Requires `GITHUB_TOKEN`. | +| MIRROR_ISSUES | no | bool | FALSE | If set to `true` the issues of your GitHub repositories will be mirrored to Gitea. Requires `GITHUB_TOKEN`. | +| MIRROR_STARRED | no | bool | FALSE | If set to `true` repositories you've starred on GitHub will be mirrored to Gitea. Requires `GITHUB_TOKEN`. | +| MIRROR_ORGANIZATIONS | no | bool | FALSE | If set to `true` repositories from organizations you belong to will be mirrored to Gitea. Requires `GITHUB_TOKEN`. | | SKIP_FORKS | no | bool | FALSE | If set to `true` will disable the mirroring of forks from your GitHub User / Organisation. | | DELAY | no | int | 3600 | Number of seconds between program executions. Setting this will only affect how soon after a new repo was created a mirror may appar on Gitea, but has no affect on the ongoing replication. | | DRY_RUN | no | bool | FALSE | If set to `true` will perform no writing changes to your Gitea instance, but log the planned actions. | @@ -57,6 +65,10 @@ docker container run \ -e GITHUB_USERNAME=github-user \ -e GITEA_URL=https://your-gitea.url \ -e GITEA_TOKEN=please-exchange-with-token \ + -e GITHUB_TOKEN=your-github-token \ + -e MIRROR_ISSUES=true \ + -e MIRROR_STARRED=true \ + -e MIRROR_ORGANIZATIONS=true \ jaedle/mirror-to-gitea:latest ``` @@ -76,6 +88,10 @@ services: - GITHUB_USERNAME=github-user - GITEA_URL=https://your-gitea.url - GITEA_TOKEN=please-exchange-with-token + - GITHUB_TOKEN=your-github-token + - MIRROR_ISSUES=true + - MIRROR_STARRED=true + - MIRROR_ORGANIZATIONS=true ``` ## Development @@ -100,6 +116,10 @@ Create `.secrets.rc` containing at least the following variables: export GITHUB_USERNAME='...' export GITHUB_TOKEN='...' export GITEA_URL='...' +export GITEA_TOKEN='...' +export MIRROR_ISSUES='true' +export MIRROR_STARRED='true' +export MIRROR_ORGANIZATIONS='true' ``` Execute the script in foreground: diff --git a/src/configuration.mjs b/src/configuration.mjs index ab2a9c1..4e46eb9 100644 --- a/src/configuration.mjs +++ b/src/configuration.mjs @@ -35,6 +35,9 @@ export function configuration() { token: process.env.GITHUB_TOKEN, skipForks: readBoolean("SKIP_FORKS"), privateRepositories: readBoolean("MIRROR_PRIVATE_REPOSITORIES"), + mirrorIssues: readBoolean("MIRROR_ISSUES"), + mirrorStarred: readBoolean("MIRROR_STARRED"), + mirrorOrganizations: readBoolean("MIRROR_ORGANIZATIONS"), }, gitea: { url: mustReadEnv("GITEA_URL"), @@ -57,5 +60,13 @@ export function configuration() { ); } + // GitHub token is required for mirroring issues, starred repos, and orgs + if ((config.github.mirrorIssues || config.github.mirrorStarred || config.github.mirrorOrganizations) + && config.github.token === undefined) { + throw new Error( + "invalid configuration, mirroring issues, starred repositories, or organizations requires setting GITHUB_TOKEN", + ); + } + return config; } diff --git a/src/get-github-repositories.mjs b/src/get-github-repositories.mjs index 5106088..fe93859 100644 --- a/src/get-github-repositories.mjs +++ b/src/get-github-repositories.mjs @@ -6,7 +6,24 @@ async function getRepositories(octokit, mirrorOptions) { const privateRepos = mirrorOptions.privateRepositories ? await fetchPrivateRepositories(octokit) : []; - const repos = [...publicRepositories, ...privateRepos]; + + // Fetch starred repos if the option is enabled + const starredRepos = mirrorOptions.mirrorStarred + ? await fetchStarredRepositories(octokit) + : []; + + // Fetch organization repos if the option is enabled + const orgRepos = mirrorOptions.mirrorOrganizations + ? await fetchOrganizationRepositories(octokit) + : []; + + // Combine all repositories and filter duplicates + const repos = filterDuplicates([ + ...publicRepositories, + ...privateRepos, + ...starredRepos, + ...orgRepos + ]); return mirrorOptions.skipForks ? withoutForks(repos) : repos; } @@ -26,10 +43,47 @@ async function fetchPrivateRepositories(octokit) { .then(toRepositoryList); } +async function fetchStarredRepositories(octokit) { + return octokit + .paginate("GET /user/starred") + .then(toRepositoryList); +} + +async function fetchOrganizationRepositories(octokit) { + // First get all organizations the user belongs to + const orgs = await octokit.paginate("GET /user/orgs"); + + // Then fetch repositories for each organization + const orgRepoPromises = orgs.map(org => + octokit.paginate("GET /orgs/{org}/repos", { org: org.login }) + ); + + // Wait for all requests to complete and flatten the results + const orgRepos = await Promise.all(orgRepoPromises) + .then(repoArrays => repoArrays.flat()) + .then(toRepositoryList); + + return orgRepos; +} + function withoutForks(repositories) { return repositories.filter((repo) => !repo.fork); } +function filterDuplicates(repositories) { + const unique = []; + const seen = new Set(); + + for (const repo of repositories) { + if (!seen.has(repo.url)) { + seen.add(repo.url); + unique.push(repo); + } + } + + return unique; +} + function toRepositoryList(repositories) { return repositories.map((repository) => { return { @@ -37,6 +91,9 @@ function toRepositoryList(repositories) { url: repository.clone_url, private: repository.private, fork: repository.fork, + owner: repository.owner?.login, + full_name: repository.full_name, + has_issues: repository.has_issues, }; }); } diff --git a/src/index.mjs b/src/index.mjs index c9888d0..bec4b64 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -4,12 +4,15 @@ import PQueue from "p-queue"; import request from "superagent"; import { configuration } from "./configuration.mjs"; import { Logger } from "./logger.js"; +import getGithubRepositories from "./get-github-repositories.mjs"; async function getGithubRepositories( username, token, mirrorPrivateRepositories, mirrorForks, + mirrorStarred, + mirrorOrganizations, include, exclude, ) { @@ -17,60 +20,46 @@ async function getGithubRepositories( auth: token || null, }); - const publicRepositories = await octokit - .paginate("GET /users/:username/repos", { username: username }) - .then((repositories) => toRepositoryList(repositories)); + const repositories = await getGithubRepositories(octokit, { + username, + privateRepositories: mirrorPrivateRepositories, + skipForks: !mirrorForks, + mirrorStarred, + mirrorOrganizations, + }); - let allOwnedRepositories; - if (mirrorPrivateRepositories) { - allOwnedRepositories = await octokit - .paginate( - "GET /user/repos?visibility=public&affiliation=owner&visibility=private", - ) - .then((repositories) => toRepositoryList(repositories)); - } - - let repositories = publicRepositories; - - if (mirrorPrivateRepositories) { - repositories = filterDuplicates( - allOwnedRepositories.concat(publicRepositories), - ); - } - - if (!mirrorForks) { - repositories = repositories.filter((repository) => !repository.fork); - } - - repositories = repositories.filter( + return repositories.filter( (repository) => include.some((f) => minimatch(repository.name, f)) && !exclude.some((f) => minimatch(repository.name, f)), ); - - return repositories; } -function toRepositoryList(repositories) { - return repositories.map((repository) => { - return { - name: repository.name, - url: repository.clone_url, - private: repository.private, - fork: repository.fork, - }; - }); -} +// Fetch issues for a given repository +async function getGithubIssues(octokit, owner, repo) { + try { + const issues = await octokit.paginate("GET /repos/{owner}/{repo}/issues", { + owner, + repo, + state: "all", + per_page: 100, + }); -function filterDuplicates(array) { - const a = array.concat(); - for (let i = 0; i < a.length; ++i) { - for (let j = i + 1; j < a.length; ++j) { - if (a[i].url === a[j].url) a.splice(j--, 1); - } + return issues.map(issue => ({ + title: issue.title, + body: issue.body || "", + state: issue.state, + labels: issue.labels.map(label => label.name), + closed: issue.state === "closed", + created_at: issue.created_at, + updated_at: issue.updated_at, + number: issue.number, + user: issue.user.login, + })); + } catch (error) { + console.error(`Error fetching issues for ${owner}/${repo}:`, error.message); + return []; } - - return a; } async function getGiteaUser(gitea) { @@ -83,7 +72,10 @@ async function getGiteaUser(gitea) { } function isAlreadyMirroredOnGitea(repository, gitea, giteaUser) { - const requestUrl = `${gitea.url}/api/v1/repos/${giteaUser.name}/${repository}`; + const repoName = repository.name; + const ownerName = giteaUser.name; + const requestUrl = `${gitea.url}/api/v1/repos/${ownerName}/${repoName}`; + return request .get(requestUrl) .set("Authorization", `token ${gitea.token}`) @@ -92,7 +84,7 @@ function isAlreadyMirroredOnGitea(repository, gitea, giteaUser) { } function mirrorOnGitea(repository, gitea, giteaUser, githubToken) { - request + return request .post(`${gitea.url}/api/v1/repos/migrate`) .set("Authorization", `token ${gitea.token}`) .send({ @@ -103,16 +95,98 @@ function mirrorOnGitea(repository, gitea, giteaUser, githubToken) { uid: giteaUser.id, private: repository.private, }) - .then(() => { - console.log("Did it!"); + .then((response) => { + console.log(`Successfully mirrored: ${repository.name}`); + return response.body; }) .catch((err) => { - console.log("Failed", err); + console.log(`Failed to mirror ${repository.name}:`, err.message); + throw err; }); } -async function mirror(repository, gitea, giteaUser, githubToken, dryRun) { - if (await isAlreadyMirroredOnGitea(repository.name, gitea, giteaUser)) { +// Create an issue in a Gitea repository +async function createGiteaIssue(issue, repository, gitea, giteaUser) { + try { + const response = await request + .post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/issues`) + .set("Authorization", `token ${gitea.token}`) + .send({ + title: issue.title, + body: `*Originally created by @${issue.user} on ${new Date(issue.created_at).toLocaleDateString()}*\n\n${issue.body}`, + state: issue.state, + closed: issue.closed, + }); + + console.log(`Created issue #${response.body.number}: ${issue.title}`); + + // Add labels if the issue has any + if (issue.labels && issue.labels.length > 0) { + await Promise.all(issue.labels.map(async (label) => { + try { + // First try to create the label if it doesn't exist + await request + .post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/labels`) + .set("Authorization", `token ${gitea.token}`) + .send({ + name: label, + color: "#" + Math.floor(Math.random() * 16777215).toString(16), // Random color + }) + .catch(() => { + // Label might already exist, which is fine + }); + + // Then add the label to the issue + await request + .post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/issues/${response.body.number}/labels`) + .set("Authorization", `token ${gitea.token}`) + .send({ + labels: [label] + }); + } catch (labelError) { + console.error(`Error adding label ${label} to issue:`, labelError.message); + } + })); + } + + return response.body; + } catch (error) { + console.error(`Error creating issue "${issue.title}":`, error.message); + return null; + } +} + +async function mirrorIssues(repository, gitea, giteaUser, githubToken, dryRun) { + if (!repository.has_issues) { + console.log(`Repository ${repository.name} doesn't have issues enabled. Skipping issues mirroring.`); + return; + } + + if (dryRun) { + console.log(`DRY RUN: Would mirror issues for repository: ${repository.name}`); + return; + } + + try { + const octokit = new Octokit({ auth: githubToken }); + const owner = repository.owner || repository.full_name.split('/')[0]; + const issues = await getGithubIssues(octokit, owner, repository.name); + + console.log(`Found ${issues.length} issues for ${repository.name}`); + + // Create issues one by one to maintain order + for (const issue of issues) { + await createGiteaIssue(issue, repository, gitea, giteaUser); + } + + console.log(`Completed mirroring issues for ${repository.name}`); + } catch (error) { + console.error(`Error mirroring issues for ${repository.name}:`, error.message); + } +} + +async function mirror(repository, gitea, giteaUser, githubToken, mirrorIssues, dryRun) { + if (await isAlreadyMirroredOnGitea(repository, gitea, giteaUser)) { console.log( "Repository is already mirrored; doing nothing: ", repository.name, @@ -124,7 +198,16 @@ async function mirror(repository, gitea, giteaUser, githubToken, dryRun) { return; } console.log("Mirroring repository to gitea: ", repository.name); - await mirrorOnGitea(repository, gitea, giteaUser, githubToken); + try { + await mirrorOnGitea(repository, gitea, giteaUser, githubToken); + + // Mirror issues if requested and not in dry run mode + if (mirrorIssues && !dryRun) { + await mirrorIssues(repository, gitea, giteaUser, githubToken, dryRun); + } + } catch (error) { + console.error(`Error during mirroring of ${repository.name}:`, error.message); + } } async function main() { @@ -144,6 +227,8 @@ async function main() { config.github.token, config.github.privateRepositories, !config.github.skipForks, + config.github.mirrorStarred, + config.github.mirrorOrganizations, config.include, config.exclude, ); @@ -165,6 +250,7 @@ async function main() { gitea, giteaUser, config.github.token, + config.github.mirrorIssues, config.dryRun, ); };