github/mirror-to-gitea
1
1
Fork 0
mirror of https://github.com/jaedle/mirror-to-gitea.git synced 2026-01-08 04:23:51 -05:00
Code Issues 6 Packages Projects Releases Wiki Activity

add support for mirroring GitHub issues, starred repositories, and organization repositories

Browse Source
This commit is contained in:
Arunavo Ray
2025-04-02 13:49:50 +05:30
parent 16f2516f9f
commit b8f6c36fad
4 changed files with 229 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
README.md
View File

@@ -21,6 +21,11 @@ The mirror settings are default by your gitea instance.
It is also possible to mirror private repos, which can be configred here in [#parameters](#parameters). When mirroring
private repos, they will be created as private repos on your gitea server.
Additionally, you can now mirror:
- Issues from GitHub repositories (including labels)
- Starred repositories from your GitHub account
- Repositories from organizations you belong to
## Prerequisites
- A github user or organization with repositories
@@ -39,8 +44,11 @@ All configuration is performed through environment variables. Flags are consider
| GITHUB_USERNAME | yes | string | - | The name of the GitHub user or organisation to mirror. |
| GITEA_URL | yes | string | - | The url of your Gitea server. |
| GITEA_TOKEN | yes | string | - | The token for your gitea user (Settings -> Applications -> Generate New Token). **Attention: if this is set, the token will be transmitted to your specified Gitea instance!** |
| GITHUB_TOKEN | no* | string | - | GitHub token (PAT). Is mandatory in combination with `MIRROR_PRIVATE_REPOSITORIES`. |
| GITHUB_TOKEN | no* | string | - | GitHub token (PAT). Is mandatory in combination with `MIRROR_PRIVATE_REPOSITORIES`, `MIRROR_ISSUES`, `MIRROR_STARRED`, or `MIRROR_ORGANIZATIONS`. |
| MIRROR_PRIVATE_REPOSITORIES | no | bool | FALSE | If set to `true` your private GitHub Repositories will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| MIRROR_ISSUES | no | bool | FALSE | If set to `true` the issues of your GitHub repositories will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| MIRROR_STARRED | no | bool | FALSE | If set to `true` repositories you've starred on GitHub will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| MIRROR_ORGANIZATIONS | no | bool | FALSE | If set to `true` repositories from organizations you belong to will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| SKIP_FORKS | no | bool | FALSE | If set to `true` will disable the mirroring of forks from your GitHub User / Organisation. |
| DELAY | no | int | 3600 | Number of seconds between program executions. Setting this will only affect how soon after a new repo was created a mirror may appar on Gitea, but has no affect on the ongoing replication. |
| DRY_RUN | no | bool | FALSE | If set to `true` will perform no writing changes to your Gitea instance, but log the planned actions. |
@@ -57,6 +65,10 @@ docker container run \
-e GITHUB_USERNAME=github-user \
-e GITEA_URL=https://your-gitea.url \
-e GITEA_TOKEN=please-exchange-with-token \
-e GITHUB_TOKEN=your-github-token \
-e MIRROR_ISSUES=true \
-e MIRROR_STARRED=true \
-e MIRROR_ORGANIZATIONS=true \
jaedle/mirror-to-gitea:latest
```
@@ -76,6 +88,10 @@ services:
- GITHUB_USERNAME=github-user
- GITEA_URL=https://your-gitea.url
- GITEA_TOKEN=please-exchange-with-token
- GITHUB_TOKEN=your-github-token
- MIRROR_ISSUES=true
- MIRROR_STARRED=true
- MIRROR_ORGANIZATIONS=true
```
## Development
@@ -100,6 +116,10 @@ Create `.secrets.rc` containing at least the following variables:
export GITHUB_USERNAME='...'
export GITHUB_TOKEN='...'
export GITEA_URL='...'
export GITEA_TOKEN='...'
export MIRROR_ISSUES='true'
export MIRROR_STARRED='true'
export MIRROR_ORGANIZATIONS='true'
```
Execute the script in foreground:

11
src/configuration.mjs
View File

@@ -35,6 +35,9 @@ export function configuration() {
token: process.env.GITHUB_TOKEN,
skipForks: readBoolean("SKIP_FORKS"),
privateRepositories: readBoolean("MIRROR_PRIVATE_REPOSITORIES"),
mirrorIssues: readBoolean("MIRROR_ISSUES"),
mirrorStarred: readBoolean("MIRROR_STARRED"),
mirrorOrganizations: readBoolean("MIRROR_ORGANIZATIONS"),
},
gitea: {
url: mustReadEnv("GITEA_URL"),
@@ -57,5 +60,13 @@ export function configuration() {
);
}
// GitHub token is required for mirroring issues, starred repos, and orgs
if ((config.github.mirrorIssues || config.github.mirrorStarred || config.github.mirrorOrganizations)
&& config.github.token === undefined) {
throw new Error(
"invalid configuration, mirroring issues, starred repositories, or organizations requires setting GITHUB_TOKEN",
);
}
return config;
}

59
src/get-github-repositories.mjs
View File

@@ -6,7 +6,24 @@ async function getRepositories(octokit, mirrorOptions) {
const privateRepos = mirrorOptions.privateRepositories
? await fetchPrivateRepositories(octokit)
: [];
const repos = [...publicRepositories, ...privateRepos];
// Fetch starred repos if the option is enabled
const starredRepos = mirrorOptions.mirrorStarred
? await fetchStarredRepositories(octokit)
: [];
// Fetch organization repos if the option is enabled
const orgRepos = mirrorOptions.mirrorOrganizations
? await fetchOrganizationRepositories(octokit)
: [];
// Combine all repositories and filter duplicates
const repos = filterDuplicates([
...publicRepositories,
...privateRepos,
...starredRepos,
...orgRepos
]);
return mirrorOptions.skipForks ? withoutForks(repos) : repos;
}
@@ -26,10 +43,47 @@ async function fetchPrivateRepositories(octokit) {
.then(toRepositoryList);
}
async function fetchStarredRepositories(octokit) {
return octokit
.paginate("GET /user/starred")
.then(toRepositoryList);
}
async function fetchOrganizationRepositories(octokit) {
// First get all organizations the user belongs to
const orgs = await octokit.paginate("GET /user/orgs");
// Then fetch repositories for each organization
const orgRepoPromises = orgs.map(org =>
octokit.paginate("GET /orgs/{org}/repos", { org: org.login })
);
// Wait for all requests to complete and flatten the results
const orgRepos = await Promise.all(orgRepoPromises)
.then(repoArrays => repoArrays.flat())
.then(toRepositoryList);
return orgRepos;
}
function withoutForks(repositories) {
return repositories.filter((repo) => !repo.fork);
}
function filterDuplicates(repositories) {
const unique = [];
const seen = new Set();
for (const repo of repositories) {
if (!seen.has(repo.url)) {
seen.add(repo.url);
unique.push(repo);
}
}
return unique;
}
function toRepositoryList(repositories) {
return repositories.map((repository) => {
return {
@@ -37,6 +91,9 @@ function toRepositoryList(repositories) {
url: repository.clone_url,
private: repository.private,
fork: repository.fork,
owner: repository.owner?.login,
full_name: repository.full_name,
has_issues: repository.has_issues,
};
});
}

192
src/index.mjs
View File

@@ -4,12 +4,15 @@ import PQueue from "p-queue";
import request from "superagent";
import { configuration } from "./configuration.mjs";
import { Logger } from "./logger.js";
import getGithubRepositories from "./get-github-repositories.mjs";
async function getGithubRepositories(
username,
token,
mirrorPrivateRepositories,
mirrorForks,
mirrorStarred,
mirrorOrganizations,
include,
exclude,
) {
@@ -17,60 +20,46 @@ async function getGithubRepositories(
auth: token || null,
});
const publicRepositories = await octokit
.paginate("GET /users/:username/repos", { username: username })
.then((repositories) => toRepositoryList(repositories));
const repositories = await getGithubRepositories(octokit, {
username,
privateRepositories: mirrorPrivateRepositories,
skipForks: !mirrorForks,
mirrorStarred,
mirrorOrganizations,
});
let allOwnedRepositories;
if (mirrorPrivateRepositories) {
allOwnedRepositories = await octokit
.paginate(
"GET /user/repos?visibility=public&affiliation=owner&visibility=private",
)
.then((repositories) => toRepositoryList(repositories));
}
let repositories = publicRepositories;
if (mirrorPrivateRepositories) {
repositories = filterDuplicates(
allOwnedRepositories.concat(publicRepositories),
);
}
if (!mirrorForks) {
repositories = repositories.filter((repository) => !repository.fork);
}
repositories = repositories.filter(
return repositories.filter(
(repository) =>
include.some((f) => minimatch(repository.name, f)) &&
!exclude.some((f) => minimatch(repository.name, f)),
);
return repositories;
}
function toRepositoryList(repositories) {
return repositories.map((repository) => {
return {
name: repository.name,
url: repository.clone_url,
private: repository.private,
fork: repository.fork,
};
});
}
// Fetch issues for a given repository
async function getGithubIssues(octokit, owner, repo) {
try {
const issues = await octokit.paginate("GET /repos/{owner}/{repo}/issues", {
owner,
repo,
state: "all",
per_page: 100,
});
function filterDuplicates(array) {
const a = array.concat();
for (let i = 0; i < a.length; ++i) {
for (let j = i + 1; j < a.length; ++j) {
if (a[i].url === a[j].url) a.splice(j--, 1);
}
return issues.map(issue => ({
title: issue.title,
body: issue.body || "",
state: issue.state,
labels: issue.labels.map(label => label.name),
closed: issue.state === "closed",
created_at: issue.created_at,
updated_at: issue.updated_at,
number: issue.number,
user: issue.user.login,
}));
} catch (error) {
console.error(`Error fetching issues for ${owner}/${repo}:`, error.message);
return [];
}
return a;
}
async function getGiteaUser(gitea) {
@@ -83,7 +72,10 @@ async function getGiteaUser(gitea) {
}
function isAlreadyMirroredOnGitea(repository, gitea, giteaUser) {
const requestUrl = `${gitea.url}/api/v1/repos/${giteaUser.name}/${repository}`;
const repoName = repository.name;
const ownerName = giteaUser.name;
const requestUrl = `${gitea.url}/api/v1/repos/${ownerName}/${repoName}`;
return request
.get(requestUrl)
.set("Authorization", `token ${gitea.token}`)
@@ -92,7 +84,7 @@ function isAlreadyMirroredOnGitea(repository, gitea, giteaUser) {
}
function mirrorOnGitea(repository, gitea, giteaUser, githubToken) {
request
return request
.post(`${gitea.url}/api/v1/repos/migrate`)
.set("Authorization", `token ${gitea.token}`)
.send({
@@ -103,16 +95,98 @@ function mirrorOnGitea(repository, gitea, giteaUser, githubToken) {
uid: giteaUser.id,
private: repository.private,
})
.then(() => {
console.log("Did it!");
.then((response) => {
console.log(`Successfully mirrored: ${repository.name}`);
return response.body;
})
.catch((err) => {
console.log("Failed", err);
console.log(`Failed to mirror ${repository.name}:`, err.message);
throw err;
});
}
async function mirror(repository, gitea, giteaUser, githubToken, dryRun) {
if (await isAlreadyMirroredOnGitea(repository.name, gitea, giteaUser)) {
// Create an issue in a Gitea repository
async function createGiteaIssue(issue, repository, gitea, giteaUser) {
try {
const response = await request
.post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/issues`)
.set("Authorization", `token ${gitea.token}`)
.send({
title: issue.title,
body: `*Originally created by @${issue.user} on ${new Date(issue.created_at).toLocaleDateString()}*\n\n${issue.body}`,
state: issue.state,
closed: issue.closed,
});
console.log(`Created issue #${response.body.number}: ${issue.title}`);
// Add labels if the issue has any
if (issue.labels && issue.labels.length > 0) {
await Promise.all(issue.labels.map(async (label) => {
try {
// First try to create the label if it doesn't exist
await request
.post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/labels`)
.set("Authorization", `token ${gitea.token}`)
.send({
name: label,
color: "#" + Math.floor(Math.random() * 16777215).toString(16), // Random color
})
.catch(() => {
// Label might already exist, which is fine
});
// Then add the label to the issue
await request
.post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/issues/${response.body.number}/labels`)
.set("Authorization", `token ${gitea.token}`)
.send({
labels: [label]
});
} catch (labelError) {
console.error(`Error adding label ${label} to issue:`, labelError.message);
}
}));
}
return response.body;
} catch (error) {
console.error(`Error creating issue "${issue.title}":`, error.message);
return null;
}
}
async function mirrorIssues(repository, gitea, giteaUser, githubToken, dryRun) {
if (!repository.has_issues) {
console.log(`Repository ${repository.name} doesn't have issues enabled. Skipping issues mirroring.`);
return;
}
if (dryRun) {
console.log(`DRY RUN: Would mirror issues for repository: ${repository.name}`);
return;
}
try {
const octokit = new Octokit({ auth: githubToken });
const owner = repository.owner || repository.full_name.split('/')[0];
const issues = await getGithubIssues(octokit, owner, repository.name);
console.log(`Found ${issues.length} issues for ${repository.name}`);
// Create issues one by one to maintain order
for (const issue of issues) {
await createGiteaIssue(issue, repository, gitea, giteaUser);
}
console.log(`Completed mirroring issues for ${repository.name}`);
} catch (error) {
console.error(`Error mirroring issues for ${repository.name}:`, error.message);
}
}
async function mirror(repository, gitea, giteaUser, githubToken, mirrorIssues, dryRun) {
if (await isAlreadyMirroredOnGitea(repository, gitea, giteaUser)) {
console.log(
"Repository is already mirrored; doing nothing: ",
repository.name,
@@ -124,7 +198,16 @@ async function mirror(repository, gitea, giteaUser, githubToken, dryRun) {
return;
}
console.log("Mirroring repository to gitea: ", repository.name);
await mirrorOnGitea(repository, gitea, giteaUser, githubToken);
try {
await mirrorOnGitea(repository, gitea, giteaUser, githubToken);
// Mirror issues if requested and not in dry run mode
if (mirrorIssues && !dryRun) {
await mirrorIssues(repository, gitea, giteaUser, githubToken, dryRun);
}
} catch (error) {
console.error(`Error during mirroring of ${repository.name}:`, error.message);
}
}
async function main() {
@@ -144,6 +227,8 @@ async function main() {
config.github.token,
config.github.privateRepositories,
!config.github.skipForks,
config.github.mirrorStarred,
config.github.mirrorOrganizations,
config.include,
config.exclude,
);
@@ -165,6 +250,7 @@ async function main() {
gitea,
giteaUser,
config.github.token,
config.github.mirrorIssues,
config.dryRun,
);
};