add support for mirroring GitHub issues, starred repositories, and organization repositories

README.md

@@ -21,6 +21,11 @@ The mirror settings are default by your gitea instance.
 It is also possible to mirror private repos, which can be configred here in [#parameters](#parameters). When mirroring
 private repos, they will be created as private repos on your gitea server.
 
+Additionally, you can now mirror:
+- Issues from GitHub repositories (including labels)
+- Starred repositories from your GitHub account
+- Repositories from organizations you belong to
+
 ## Prerequisites
 
 - A github user or organization with repositories
@@ -39,8 +44,11 @@ All configuration is performed through environment variables. Flags are consider
 | GITHUB_USERNAME             | yes      | string | -       | The name of the GitHub user or organisation to mirror.                                                                                                                                                 |
 | GITEA_URL                   | yes      | string | -       | The url of your Gitea server.                                                                                                                                                                          |
 | GITEA_TOKEN                 | yes      | string | -       | The token for your gitea user (Settings -> Applications -> Generate New Token). **Attention: if this is set, the token will be transmitted to your specified Gitea instance!**                         |
-| GITHUB_TOKEN                | no*      | string | -       | GitHub token (PAT). Is mandatory in combination with `MIRROR_PRIVATE_REPOSITORIES`.                                                                                                                    |
+| GITHUB_TOKEN                | no*      | string | -       | GitHub token (PAT). Is mandatory in combination with `MIRROR_PRIVATE_REPOSITORIES`, `MIRROR_ISSUES`, `MIRROR_STARRED`, or `MIRROR_ORGANIZATIONS`.                                                      |
 | MIRROR_PRIVATE_REPOSITORIES | no       | bool   | FALSE   | If set to `true` your private GitHub Repositories will be mirrored to Gitea. Requires `GITHUB_TOKEN`.                                                                                                  |
+| MIRROR_ISSUES               | no       | bool   | FALSE   | If set to `true` the issues of your GitHub repositories will be mirrored to Gitea. Requires `GITHUB_TOKEN`.                                                                                           |
+| MIRROR_STARRED              | no       | bool   | FALSE   | If set to `true` repositories you've starred on GitHub will be mirrored to Gitea. Requires `GITHUB_TOKEN`.                                                                                             |
+| MIRROR_ORGANIZATIONS        | no       | bool   | FALSE   | If set to `true` repositories from organizations you belong to will be mirrored to Gitea. Requires `GITHUB_TOKEN`.                                                                                     |
 | SKIP_FORKS                  | no       | bool   | FALSE   | If set to `true` will disable the mirroring of forks from your GitHub User / Organisation.                                                                                                             |
 | DELAY                       | no       | int    | 3600    | Number of seconds between program executions. Setting this will only affect how soon after a new repo was created a mirror may appar on Gitea, but has no affect on the ongoing replication. |
 | DRY_RUN                     | no       | bool   | FALSE   | If set to `true` will perform no writing changes to your Gitea instance, but log the planned actions.                                                                                                  |
@@ -57,6 +65,10 @@ docker container run \
  -e GITHUB_USERNAME=github-user \
  -e GITEA_URL=https://your-gitea.url \
  -e GITEA_TOKEN=please-exchange-with-token \
+ -e GITHUB_TOKEN=your-github-token \
+ -e MIRROR_ISSUES=true \
+ -e MIRROR_STARRED=true \
+ -e MIRROR_ORGANIZATIONS=true \
  jaedle/mirror-to-gitea:latest
 ```
 
@@ -76,6 +88,10 @@ services:
       - GITHUB_USERNAME=github-user
       - GITEA_URL=https://your-gitea.url
       - GITEA_TOKEN=please-exchange-with-token
+      - GITHUB_TOKEN=your-github-token
+      - MIRROR_ISSUES=true
+      - MIRROR_STARRED=true
+      - MIRROR_ORGANIZATIONS=true
 ```
 
 ## Development
@@ -100,6 +116,10 @@ Create `.secrets.rc` containing at least the following variables:
 export GITHUB_USERNAME='...'
 export GITHUB_TOKEN='...'
 export GITEA_URL='...'
+export GITEA_TOKEN='...'
+export MIRROR_ISSUES='true'
+export MIRROR_STARRED='true'
+export MIRROR_ORGANIZATIONS='true'
 ```
 
 Execute the script in foreground:

src/configuration.mjs

@@ -35,6 +35,9 @@ export function configuration() {
 			token: process.env.GITHUB_TOKEN,
 			skipForks: readBoolean("SKIP_FORKS"),
 			privateRepositories: readBoolean("MIRROR_PRIVATE_REPOSITORIES"),
+			mirrorIssues: readBoolean("MIRROR_ISSUES"),
+			mirrorStarred: readBoolean("MIRROR_STARRED"),
+			mirrorOrganizations: readBoolean("MIRROR_ORGANIZATIONS"),
 		},
 		gitea: {
 			url: mustReadEnv("GITEA_URL"),
@@ -57,5 +60,13 @@ export function configuration() {
 		);
 	}
 
+	// GitHub token is required for mirroring issues, starred repos, and orgs
+	if ((config.github.mirrorIssues || config.github.mirrorStarred || config.github.mirrorOrganizations) 
+		&& config.github.token === undefined) {
+		throw new Error(
+			"invalid configuration, mirroring issues, starred repositories, or organizations requires setting GITHUB_TOKEN",
+		);
+	}
+
 	return config;
 }

src/get-github-repositories.mjs

@@ -6,7 +6,24 @@ async function getRepositories(octokit, mirrorOptions) {
 	const privateRepos = mirrorOptions.privateRepositories
 		? await fetchPrivateRepositories(octokit)
 		: [];
-	const repos = [...publicRepositories, ...privateRepos];
+	
+	// Fetch starred repos if the option is enabled
+	const starredRepos = mirrorOptions.mirrorStarred
+		? await fetchStarredRepositories(octokit)
+		: [];
+	
+	// Fetch organization repos if the option is enabled
+	const orgRepos = mirrorOptions.mirrorOrganizations
+		? await fetchOrganizationRepositories(octokit)
+		: [];
+	
+	// Combine all repositories and filter duplicates
+	const repos = filterDuplicates([
+		...publicRepositories, 
+		...privateRepos,
+		...starredRepos,
+		...orgRepos
+	]);
 
 	return mirrorOptions.skipForks ? withoutForks(repos) : repos;
 }
@@ -26,10 +43,47 @@ async function fetchPrivateRepositories(octokit) {
 		.then(toRepositoryList);
 }
 
+async function fetchStarredRepositories(octokit) {
+	return octokit
+		.paginate("GET /user/starred")
+		.then(toRepositoryList);
+}
+
+async function fetchOrganizationRepositories(octokit) {
+	// First get all organizations the user belongs to
+	const orgs = await octokit.paginate("GET /user/orgs");
+	
+	// Then fetch repositories for each organization
+	const orgRepoPromises = orgs.map(org => 
+		octokit.paginate("GET /orgs/{org}/repos", { org: org.login })
+	);
+	
+	// Wait for all requests to complete and flatten the results
+	const orgRepos = await Promise.all(orgRepoPromises)
+		.then(repoArrays => repoArrays.flat())
+		.then(toRepositoryList);
+	
+	return orgRepos;
+}
+
 function withoutForks(repositories) {
 	return repositories.filter((repo) => !repo.fork);
 }
 
+function filterDuplicates(repositories) {
+	const unique = [];
+	const seen = new Set();
+	
+	for (const repo of repositories) {
+		if (!seen.has(repo.url)) {
+			seen.add(repo.url);
+			unique.push(repo);
+		}
+	}
+	
+	return unique;
+}
+
 function toRepositoryList(repositories) {
 	return repositories.map((repository) => {
 		return {
@@ -37,6 +91,9 @@ function toRepositoryList(repositories) {
 			url: repository.clone_url,
 			private: repository.private,
 			fork: repository.fork,
+			owner: repository.owner?.login,
+			full_name: repository.full_name,
+			has_issues: repository.has_issues,
 		};
 	});
 }

src/index.mjs

@@ -4,12 +4,15 @@ import PQueue from "p-queue";
 import request from "superagent";
 import { configuration } from "./configuration.mjs";
 import { Logger } from "./logger.js";
+import getGithubRepositories from "./get-github-repositories.mjs";
 
 async function getGithubRepositories(
 	username,
 	token,
 	mirrorPrivateRepositories,
 	mirrorForks,
+	mirrorStarred,
+	mirrorOrganizations,
 	include,
 	exclude,
 ) {
@@ -17,60 +20,46 @@ async function getGithubRepositories(
 		auth: token || null,
 	});
 
-	const publicRepositories = await octokit
+	const repositories = await getGithubRepositories(octokit, {
-		.paginate("GET /users/:username/repos", { username: username })
+		username,
-		.then((repositories) => toRepositoryList(repositories));
+		privateRepositories: mirrorPrivateRepositories,
+		skipForks: !mirrorForks,
+		mirrorStarred,
+		mirrorOrganizations,
+	});
 
-	let allOwnedRepositories;
+	return repositories.filter(
-	if (mirrorPrivateRepositories) {
-		allOwnedRepositories = await octokit
-			.paginate(
-				"GET /user/repos?visibility=public&affiliation=owner&visibility=private",
-			)
-			.then((repositories) => toRepositoryList(repositories));
-	}
-
-	let repositories = publicRepositories;
-
-	if (mirrorPrivateRepositories) {
-		repositories = filterDuplicates(
-			allOwnedRepositories.concat(publicRepositories),
-		);
-	}
-
-	if (!mirrorForks) {
-		repositories = repositories.filter((repository) => !repository.fork);
-	}
-
-	repositories = repositories.filter(
 		(repository) =>
 			include.some((f) => minimatch(repository.name, f)) &&
 			!exclude.some((f) => minimatch(repository.name, f)),
 	);
-
-	return repositories;
 }
 
-function toRepositoryList(repositories) {
+// Fetch issues for a given repository
-	return repositories.map((repository) => {
+async function getGithubIssues(octokit, owner, repo) {
-		return {
+	try {
-			name: repository.name,
+		const issues = await octokit.paginate("GET /repos/{owner}/{repo}/issues", {
-			url: repository.clone_url,
+			owner,
-			private: repository.private,
+			repo,
-			fork: repository.fork,
+			state: "all",
-		};
+			per_page: 100,
-	});
+		});
-}
 
-function filterDuplicates(array) {
+		return issues.map(issue => ({
-	const a = array.concat();
+			title: issue.title,
-	for (let i = 0; i < a.length; ++i) {
+			body: issue.body || "",
-		for (let j = i + 1; j < a.length; ++j) {
+			state: issue.state,
-			if (a[i].url === a[j].url) a.splice(j--, 1);
+			labels: issue.labels.map(label => label.name),
-		}
+			closed: issue.state === "closed",
+			created_at: issue.created_at,
+			updated_at: issue.updated_at,
+			number: issue.number,
+			user: issue.user.login,
+		}));
+	} catch (error) {
+		console.error(`Error fetching issues for ${owner}/${repo}:`, error.message);
+		return [];
 	}
-
-	return a;
 }
 
 async function getGiteaUser(gitea) {
@@ -83,7 +72,10 @@ async function getGiteaUser(gitea) {
 }
 
 function isAlreadyMirroredOnGitea(repository, gitea, giteaUser) {
-	const requestUrl = `${gitea.url}/api/v1/repos/${giteaUser.name}/${repository}`;
+	const repoName = repository.name;
+	const ownerName = giteaUser.name;
+	const requestUrl = `${gitea.url}/api/v1/repos/${ownerName}/${repoName}`;
+	
 	return request
 		.get(requestUrl)
 		.set("Authorization", `token ${gitea.token}`)
@@ -92,7 +84,7 @@ function isAlreadyMirroredOnGitea(repository, gitea, giteaUser) {
 }
 
 function mirrorOnGitea(repository, gitea, giteaUser, githubToken) {
-	request
+	return request
 		.post(`${gitea.url}/api/v1/repos/migrate`)
 		.set("Authorization", `token ${gitea.token}`)
 		.send({
@@ -103,16 +95,98 @@ function mirrorOnGitea(repository, gitea, giteaUser, githubToken) {
 			uid: giteaUser.id,
 			private: repository.private,
 		})
-		.then(() => {
+		.then((response) => {
-			console.log("Did it!");
+			console.log(`Successfully mirrored: ${repository.name}`);
+			return response.body;
 		})
 		.catch((err) => {
-			console.log("Failed", err);
+			console.log(`Failed to mirror ${repository.name}:`, err.message);
+			throw err;
 		});
 }
 
-async function mirror(repository, gitea, giteaUser, githubToken, dryRun) {
+// Create an issue in a Gitea repository
-	if (await isAlreadyMirroredOnGitea(repository.name, gitea, giteaUser)) {
+async function createGiteaIssue(issue, repository, gitea, giteaUser) {
+	try {
+		const response = await request
+			.post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/issues`)
+			.set("Authorization", `token ${gitea.token}`)
+			.send({
+				title: issue.title,
+				body: `*Originally created by @${issue.user} on ${new Date(issue.created_at).toLocaleDateString()}*\n\n${issue.body}`,
+				state: issue.state,
+				closed: issue.closed,
+			});
+		
+		console.log(`Created issue #${response.body.number}: ${issue.title}`);
+		
+		// Add labels if the issue has any
+		if (issue.labels && issue.labels.length > 0) {
+			await Promise.all(issue.labels.map(async (label) => {
+				try {
+					// First try to create the label if it doesn't exist
+					await request
+						.post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/labels`)
+						.set("Authorization", `token ${gitea.token}`)
+						.send({
+							name: label,
+							color: "#" + Math.floor(Math.random() * 16777215).toString(16), // Random color
+						})
+						.catch(() => {
+							// Label might already exist, which is fine
+						});
+					
+					// Then add the label to the issue
+					await request
+						.post(`${gitea.url}/api/v1/repos/${giteaUser.name}/${repository.name}/issues/${response.body.number}/labels`)
+						.set("Authorization", `token ${gitea.token}`)
+						.send({
+							labels: [label]
+						});
+				} catch (labelError) {
+					console.error(`Error adding label ${label} to issue:`, labelError.message);
+				}
+			}));
+		}
+		
+		return response.body;
+	} catch (error) {
+		console.error(`Error creating issue "${issue.title}":`, error.message);
+		return null;
+	}
+}
+
+async function mirrorIssues(repository, gitea, giteaUser, githubToken, dryRun) {
+	if (!repository.has_issues) {
+		console.log(`Repository ${repository.name} doesn't have issues enabled. Skipping issues mirroring.`);
+		return;
+	}
+
+	if (dryRun) {
+		console.log(`DRY RUN: Would mirror issues for repository: ${repository.name}`);
+		return;
+	}
+
+	try {
+		const octokit = new Octokit({ auth: githubToken });
+		const owner = repository.owner || repository.full_name.split('/')[0];
+		const issues = await getGithubIssues(octokit, owner, repository.name);
+		
+		console.log(`Found ${issues.length} issues for ${repository.name}`);
+		
+		// Create issues one by one to maintain order
+		for (const issue of issues) {
+			await createGiteaIssue(issue, repository, gitea, giteaUser);
+		}
+		
+		console.log(`Completed mirroring issues for ${repository.name}`);
+	} catch (error) {
+		console.error(`Error mirroring issues for ${repository.name}:`, error.message);
+	}
+}
+
+async function mirror(repository, gitea, giteaUser, githubToken, mirrorIssues, dryRun) {
+	if (await isAlreadyMirroredOnGitea(repository, gitea, giteaUser)) {
 		console.log(
 			"Repository is already mirrored; doing nothing: ",
 			repository.name,
@@ -124,7 +198,16 @@ async function mirror(repository, gitea, giteaUser, githubToken, dryRun) {
 		return;
 	}
 	console.log("Mirroring repository to gitea: ", repository.name);
-	await mirrorOnGitea(repository, gitea, giteaUser, githubToken);
+	try {
+		await mirrorOnGitea(repository, gitea, giteaUser, githubToken);
+		
+		// Mirror issues if requested and not in dry run mode
+		if (mirrorIssues && !dryRun) {
+			await mirrorIssues(repository, gitea, giteaUser, githubToken, dryRun);
+		}
+	} catch (error) {
+		console.error(`Error during mirroring of ${repository.name}:`, error.message);
+	}
 }
 
 async function main() {
@@ -144,6 +227,8 @@ async function main() {
 		config.github.token,
 		config.github.privateRepositories,
 		!config.github.skipForks,
+		config.github.mirrorStarred,
+		config.github.mirrorOrganizations,
 		config.include,
 		config.exclude,
 	);
@@ -165,6 +250,7 @@ async function main() {
 					gitea,
 					giteaUser,
 					config.github.token,
+					config.github.mirrorIssues,
 					config.dryRun,
 				);
 			};