github/mirror-to-gitea
1
1
Fork 0
mirror of https://github.com/jaedle/mirror-to-gitea.git synced 2026-01-08 04:23:51 -05:00
Code Issues 6 Packages Projects Releases Wiki Activity

Merge pull request #30 from arunavo4/main

Browse Source 
[Feature] Starred Repo Mirror | Org Features | Mirroring Issues
This commit is contained in:
Dennis Wielepsky
2025-04-07 21:06:21 +02:00
committed by GitHub
parent 16f2516f9f 5c246b2e08
commit 63bcbe4034
7 changed files with 1047 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
Dockerfile
View File

@@ -1,14 +1,63 @@
FROM node:lts-alpine
# Build stage
FROM node:lts-alpine AS builder
# Set working directory
WORKDIR /app
# Copy package files
COPY package.json package-lock.json ./
# Install all dependencies (including dev dependencies needed for build)
RUN npm ci
COPY docker-entrypoint.sh .
COPY src ./src
# Copy source code
COPY --chown=node:node . .
RUN npm run build && rm -rf package*.json
# Build the application with minification
RUN npm run build -- --minify
# Prune dependencies stage
FROM node:lts-alpine AS deps
WORKDIR /app
# Copy package files
COPY --from=builder /app/package.json /app/package-lock.json ./
# Install production dependencies only
RUN npm install --omit=dev --production && \
# Remove unnecessary npm cache and temp files to reduce size
npm cache clean --force && \
rm -rf /tmp/* /var/cache/apk/*
# Production stage
FROM node:lts-alpine AS production
# Add Docker Alpine packages and remove cache in the same layer
RUN apk --no-cache add ca-certificates tini && \
rm -rf /var/cache/apk/*
# Set non-root user for better security
USER node
# Set working directory owned by node user
WORKDIR /app
# Copy only the built application and entry point from builder
COPY --from=builder --chown=node:node /app/dist ./dist
COPY --from=builder --chown=node:node /app/docker-entrypoint.sh .
# Copy only production node_modules
COPY --from=deps --chown=node:node /app/node_modules ./node_modules
# Make entry point executable
RUN chmod +x /app/docker-entrypoint.sh
# Set environment to production to disable development features
ENV NODE_ENV=production
# Use tini as init system to properly handle signals
ENTRYPOINT ["/sbin/tini", "--"]
# The command to run
CMD [ "/app/docker-entrypoint.sh" ]

130
README.md
View File

@@ -21,6 +21,15 @@ The mirror settings are default by your gitea instance.
It is also possible to mirror private repos, which can be configred here in [#parameters](#parameters). When mirroring
private repos, they will be created as private repos on your gitea server.
Additionally, you can now mirror:
- Issues from GitHub repositories (including labels)
- Starred repositories from your GitHub account
- Repositories from organizations you belong to
- Filter which organizations to include or exclude
- Maintain original organization structure in Gitea
- A single repository instead of all repositories
- Repositories to a specific Gitea organization
## Prerequisites
- A github user or organization with repositories
@@ -39,10 +48,22 @@ All configuration is performed through environment variables. Flags are consider
| GITHUB_USERNAME | yes | string | - | The name of the GitHub user or organisation to mirror. |
| GITEA_URL | yes | string | - | The url of your Gitea server. |
| GITEA_TOKEN | yes | string | - | The token for your gitea user (Settings -> Applications -> Generate New Token). **Attention: if this is set, the token will be transmitted to your specified Gitea instance!** |
| GITHUB_TOKEN | no* | string | - | GitHub token (PAT). Is mandatory in combination with `MIRROR_PRIVATE_REPOSITORIES`. |
| GITHUB_TOKEN | no* | string | - | GitHub token (PAT). Is mandatory in combination with `MIRROR_PRIVATE_REPOSITORIES`, `MIRROR_ISSUES`, `MIRROR_STARRED`, `MIRROR_ORGANIZATIONS`, or `SINGLE_REPO`. |
| MIRROR_PRIVATE_REPOSITORIES | no | bool | FALSE | If set to `true` your private GitHub Repositories will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| MIRROR_ISSUES | no | bool | FALSE | If set to `true` the issues of your GitHub repositories will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| MIRROR_STARRED | no | bool | FALSE | If set to `true` repositories you've starred on GitHub will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| MIRROR_ORGANIZATIONS | no | bool | FALSE | If set to `true` repositories from organizations you belong to will be mirrored to Gitea. Requires `GITHUB_TOKEN`. |
| USE_SPECIFIC_USER | no | bool | FALSE | If set to `true`, the tool will use public API endpoints to fetch starred repositories and organizations for the specified `GITHUB_USERNAME` instead of the authenticated user. |
| INCLUDE_ORGS | no | string | "" | Comma-separated list of GitHub organization names to include when mirroring organizations. If not specified, all organizations will be included. |
| EXCLUDE_ORGS | no | string | "" | Comma-separated list of GitHub organization names to exclude when mirroring organizations. Takes precedence over `INCLUDE_ORGS`. |
| PRESERVE_ORG_STRUCTURE | no | bool | FALSE | If set to `true`, each GitHub organization will be mirrored to a Gitea organization with the same name. If the organization doesn't exist, it will be created. |
| SINGLE_REPO | no | string | - | URL of a single GitHub repository to mirror (e.g., https://github.com/username/repo or username/repo). When specified, only this repository will be mirrored. Requires `GITHUB_TOKEN`. |
| GITEA_ORGANIZATION | no | string | - | Name of a Gitea organization to mirror repositories to. If doesn't exist, will be created. |
| GITEA_ORG_VISIBILITY | no | string | public | Visibility of the Gitea organization to create. Can be "public" or "private". |
| GITEA_STARRED_ORGANIZATION | no | string | github | Name of a Gitea organization to mirror starred repositories to. If doesn't exist, will be created. Defaults to "github". |
| SKIP_STARRED_ISSUES | no | bool | FALSE | If set to `true` will not mirror issues for starred repositories, even if `MIRROR_ISSUES` is enabled. |
| SKIP_FORKS | no | bool | FALSE | If set to `true` will disable the mirroring of forks from your GitHub User / Organisation. |
| DELAY | no | int | 3600 | Number of seconds between program executions. Setting this will only affect how soon after a new repo was created a mirror may appar on Gitea, but has no affect on the ongoing replication. |
| DELAY | no | int | 3600 | Number of seconds between program executions. Setting this will only affect how soon after a new repo was created a mirror may appear on Gitea, but has no effect on the ongoing replication. |
| DRY_RUN | no | bool | FALSE | If set to `true` will perform no writing changes to your Gitea instance, but log the planned actions. |
| INCLUDE | no | string | "*" | Name based repository filter (include): If any filter matches, the repository will be mirrored. It supports glob format, multiple filters can be separated with commas (`,`) |
| EXCLUDE | no | string | "" | Name based repository filter (exclude). If any filter matches, the repository will not be mirrored. It supports glob format, multiple filters can be separated with commas (`,`). `EXCLUDE` filters are applied after `INCLUDE` ones.
@@ -57,11 +78,90 @@ docker container run \
-e GITHUB_USERNAME=github-user \
-e GITEA_URL=https://your-gitea.url \
-e GITEA_TOKEN=please-exchange-with-token \
-e GITHUB_TOKEN=your-github-token \
-e MIRROR_ISSUES=true \
-e MIRROR_STARRED=true \
-e MIRROR_ORGANIZATIONS=true \
jaedle/mirror-to-gitea:latest
```
This will a spin up a docker container which will run forever, mirroring all your repositories once every hour to your
gitea server.
### Mirror Only Specific Organizations
```sh
docker container run \
-d \
--restart always \
-e GITHUB_USERNAME=github-user \
-e GITEA_URL=https://your-gitea.url \
-e GITEA_TOKEN=please-exchange-with-token \
-e GITHUB_TOKEN=your-github-token \
-e MIRROR_ORGANIZATIONS=true \
-e INCLUDE_ORGS=org1,org2,org3 \
jaedle/mirror-to-gitea:latest
```
### Mirror Organizations with Preserved Structure
```sh
docker container run \
-d \
--restart always \
-e GITHUB_USERNAME=github-user \
-e GITEA_URL=https://your-gitea.url \
-e GITEA_TOKEN=please-exchange-with-token \
-e GITHUB_TOKEN=your-github-token \
-e MIRROR_ORGANIZATIONS=true \
-e PRESERVE_ORG_STRUCTURE=true \
-e GITEA_ORG_VISIBILITY=private \
jaedle/mirror-to-gitea:latest
```
### Mirror a Single Repository
```sh
docker container run \
-d \
--restart always \
-e GITHUB_USERNAME=github-user \
-e GITEA_URL=https://your-gitea.url \
-e GITEA_TOKEN=please-exchange-with-token \
-e GITHUB_TOKEN=your-github-token \
-e SINGLE_REPO=https://github.com/organization/repository \
jaedle/mirror-to-gitea:latest
```
### Mirror to a Gitea Organization
```sh
docker container run \
-d \
--restart always \
-e GITHUB_USERNAME=github-user \
-e GITEA_URL=https://your-gitea.url \
-e GITEA_TOKEN=please-exchange-with-token \
-e GITHUB_TOKEN=your-github-token \
-e GITEA_ORGANIZATION=my-organization \
-e GITEA_ORG_VISIBILITY=private \
jaedle/mirror-to-gitea:latest
```
### Mirror Starred Repositories to a Dedicated Organization
```sh
docker container run \
-d \
--restart always \
-e GITHUB_USERNAME=github-user \
-e GITEA_URL=https://your-gitea.url \
-e GITEA_TOKEN=please-exchange-with-token \
-e GITHUB_TOKEN=your-github-token \
-e MIRROR_STARRED=true \
-e GITEA_STARRED_ORGANIZATION=github \
-e SKIP_STARRED_ISSUES=true \
jaedle/mirror-to-gitea:latest
```
This configuration will mirror all starred repositories to a Gitea organization named "github" and will not mirror issues for these starred repositories.
### Docker Compose
@@ -76,6 +176,18 @@ services:
- GITHUB_USERNAME=github-user
- GITEA_URL=https://your-gitea.url
- GITEA_TOKEN=please-exchange-with-token
- GITHUB_TOKEN=your-github-token
- MIRROR_ISSUES=true
- MIRROR_STARRED=true
- MIRROR_ORGANIZATIONS=true
# Organization specific options
# - INCLUDE_ORGS=org1,org2
# - EXCLUDE_ORGS=org3,org4
# - PRESERVE_ORG_STRUCTURE=true
# Other options
# - SINGLE_REPO=https://github.com/organization/repository
# - GITEA_ORGANIZATION=my-organization
# - GITEA_ORG_VISIBILITY=public
```
## Development
@@ -100,6 +212,16 @@ Create `.secrets.rc` containing at least the following variables:
export GITHUB_USERNAME='...'
export GITHUB_TOKEN='...'
export GITEA_URL='...'
export GITEA_TOKEN='...'
export MIRROR_ISSUES='true'
export MIRROR_STARRED='true'
export MIRROR_ORGANIZATIONS='true'
# export INCLUDE_ORGS='org1,org2'
# export EXCLUDE_ORGS='org3,org4'
# export PRESERVE_ORG_STRUCTURE='true'
# export SINGLE_REPO='https://github.com/user/repo'
# export GITEA_ORGANIZATION='my-organization'
# export GITEA_ORG_VISIBILITY='public'
```
Execute the script in foreground:

166
debug.sh Executable file
View File

@@ -0,0 +1,166 @@
#!/usr/bin/env bash
set -e
echo "Mirror to Gitea Debug Script"
echo "============================="
source .secrets.rc
# Get host machine IP address
HOST_IP=$(ipconfig getifaddr en0)
echo "Host IP: $HOST_IP"
GITEA_URL_HOST=${GITEA_URL/localhost/$HOST_IP}
echo "Gitea URL: $GITEA_URL"
echo "Gitea URL for Docker: $GITEA_URL_HOST"
echo -e "\nTesting Gitea API access directly:"
curl -s -H "Authorization: token $GITEA_TOKEN" "$GITEA_URL/api/v1/user" | jq '.'
echo -e "\nTesting GitHub token validity:"
GITHUB_USER_RESPONSE=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/user")
echo "$GITHUB_USER_RESPONSE" | jq '. | {login, name}'
# Testing access to private repositories
echo -e "\nTesting GitHub private repositories access:"
PRIVATE_REPOS_RESPONSE=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/user/repos?visibility=private&per_page=1")
PRIVATE_REPOS_COUNT=$(echo "$PRIVATE_REPOS_RESPONSE" | jq '. | length')
if [ "$PRIVATE_REPOS_COUNT" -eq 0 ]; then
echo "No private repositories found or no permission to access them."
else
echo "Found private repositories. First one: $(echo "$PRIVATE_REPOS_RESPONSE" | jq '.[0].full_name')"
fi
echo -e "\nTesting GitHub organization access:"
echo "Method 1 - Using /user/orgs endpoint (authenticated user):"
ORG_RESPONSE=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/user/orgs")
ORG_COUNT=$(echo "$ORG_RESPONSE" | jq '. | length')
if [ "$ORG_COUNT" -eq 0 ]; then
echo "No organizations found via /user/orgs endpoint."
else
echo "Organizations found via /user/orgs:"
echo "$ORG_RESPONSE" | jq '.[].login'
fi
echo -e "\nMethod 2 - Using /users/{username}/orgs endpoint (specific user):"
PUBLIC_USER_ORGS=$(curl -s -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/users/$GITHUB_USERNAME/orgs")
PUBLIC_ORG_COUNT=$(echo "$PUBLIC_USER_ORGS" | jq '. | length')
if [ "$PUBLIC_ORG_COUNT" -eq 0 ]; then
echo "No public organizations found for $GITHUB_USERNAME via /users/{username}/orgs endpoint."
else
echo "Public organizations found for $GITHUB_USERNAME:"
echo "$PUBLIC_USER_ORGS" | jq '.[].login'
fi
echo "Method 3 - Looking for specific organizations:"
for org in "Gameplex-labs" "Neucruit" "uiastra"; do
ORG_DETAILS=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/orgs/$org")
if [[ $(echo "$ORG_DETAILS" | jq 'has("login")') == "true" ]]; then
echo "Found organization: $org"
# Check if we can access the organization's repositories
ORG_REPOS=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/orgs/$org/repos?per_page=1")
REPO_COUNT=$(echo "$ORG_REPOS" | jq '. | length')
if [ "$REPO_COUNT" -gt 0 ]; then
echo " Can access repositories for $org"
echo " Example repo: $(echo "$ORG_REPOS" | jq '.[0].full_name')"
else
echo " Cannot access repositories for $org or organization has no repositories"
fi
else
echo "Could not find organization: $org (or no permission to access it)"
fi
done
echo -e "\nTesting GitHub starred repos access:"
echo "Method 1 - Using /user/starred endpoint (authenticated user):"
STARRED_RESPONSE=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/user/starred?per_page=1")
STARRED_COUNT=$(echo "$STARRED_RESPONSE" | jq '. | length')
if [ "$STARRED_COUNT" -eq 0 ]; then
echo "No starred repositories found. You may not have starred any GitHub repositories."
else
echo "First starred repo: $(echo "$STARRED_RESPONSE" | jq '.[0].full_name')"
echo "Total starred repositories accessible: $(curl -s -H "Authorization: token $GITHUB_TOKEN" -I "https://api.github.com/user/starred" | grep -i "^link:" | grep -o "page=[0-9]*" | sort -r | head -1 | cut -d= -f2 || echo "Unknown")"
fi
echo -e "\nMethod 2 - Using /users/{username}/starred endpoint (specific user):"
PUBLIC_STARRED_RESPONSE=$(curl -s -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/users/$GITHUB_USERNAME/starred?per_page=1")
PUBLIC_STARRED_COUNT=$(echo "$PUBLIC_STARRED_RESPONSE" | jq '. | length')
if [ "$PUBLIC_STARRED_COUNT" -eq 0 ]; then
echo "No public starred repositories found for $GITHUB_USERNAME."
else
echo "First public starred repo for $GITHUB_USERNAME: $(echo "$PUBLIC_STARRED_RESPONSE" | jq '.[0].full_name')"
echo "Total public starred repositories for $GITHUB_USERNAME: $(curl -s -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -I "https://api.github.com/users/$GITHUB_USERNAME/starred" | grep -i "^link:" | grep -o "page=[0-9]*" | sort -r | head -1 | cut -d= -f2 || echo "Unknown")"
fi
echo -e "\nTesting GitHub issues access:"
echo "Checking for issues in your repositories..."
# Get a list of repositories to check for issues
USER_REPOS=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/user/repos?per_page=5&sort=updated")
REPO_COUNT=$(echo "$USER_REPOS" | jq '. | length')
echo "Found $REPO_COUNT recently updated repositories to check for issues"
# Check each repository for issues
for i in $(seq 0 $(($REPO_COUNT - 1))); do
REPO=$(echo "$USER_REPOS" | jq -r ".[$i].full_name")
REPO_HAS_ISSUES=$(echo "$USER_REPOS" | jq -r ".[$i].has_issues")
if [ "$REPO_HAS_ISSUES" = "true" ]; then
ISSUES_RESPONSE=$(curl -s -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$REPO/issues?state=all&per_page=1")
ISSUES_COUNT=$(curl -s -H "Authorization: token $GITHUB_TOKEN" -I "https://api.github.com/repos/$REPO/issues?state=all" | grep -i "^link:" | grep -o "page=[0-9]*" | sort -r | head -1 | cut -d= -f2 || echo "0")
if [ -z "$ISSUES_COUNT" ]; then
# If we couldn't get the count from Link header, count the array length
ISSUES_COUNT=$(echo "$ISSUES_RESPONSE" | jq '. | length')
fi
if [ "$ISSUES_COUNT" -gt 0 ]; then
echo "Repository $REPO has approximately $ISSUES_COUNT issues"
echo "Latest issue: $(echo "$ISSUES_RESPONSE" | jq -r '.[0].title // "No title"')"
else
echo "Repository $REPO has issues enabled but no issues were found"
fi
else
echo "Repository $REPO has issues disabled"
fi
done
echo -e "\nVerifying GitHub token scopes for issues access:"
SCOPES=$(curl -s -I -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/user" | grep -i "^x-oauth-scopes:" | cut -d ":" -f 2- | tr -d '\r' || echo "None found")
if [[ "$SCOPES" == *"repo"* ]]; then
echo "Your token has the 'repo' scope, which is required for issues access"
else
echo "WARNING: Your token may not have the 'repo' scope, which is required for full issues access"
echo "Testing issues access directly: $(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/$GITHUB_USERNAME/$(echo "$USER_REPOS" | jq -r '.[0].name')/issues")"
fi
echo -e "\nVerifying GitHub token scopes:"
SCOPES=$(curl -s -I -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/user" | grep -i "^x-oauth-scopes:" | cut -d ":" -f 2- | tr -d '\r' || echo "None found")
if [ -z "$SCOPES" ] || [ "$SCOPES" = "None found" ]; then
echo "No explicit scopes found in GitHub token. This may be a personal access token (classic) with default scopes."
echo "Testing functionality directly:"
echo "- Can access user info: $(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/user")"
echo "- Can access private repos: $(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/user/repos?visibility=private&per_page=1")"
echo "- Can access organizations: $(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/user/orgs")"
echo "- Can access starred repos: $(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/user/starred?per_page=1")"
else
echo "Your token has these scopes: $SCOPES"
fi
echo -e "\nRequired scopes for full functionality:"
echo "- repo (for repositories and issues)"
echo "- read:org (for organization access)"
echo "- user (for starred repositories)"
echo -e "\nSpecific user mode tests (USE_SPECIFIC_USER=true):"
echo "This mode uses the following endpoints:"
echo "- GET /users/{username}/orgs"
echo "- GET /users/{username}/starred"
echo "These endpoints are working: $([ "$PUBLIC_ORG_COUNT" -ge 0 ] && [ "$PUBLIC_STARRED_COUNT" -ge 0 ] && echo "YES" || echo "NO")"
echo -e "\nYour environment should now be ready for testing."
echo "To test with the new USE_SPECIFIC_USER feature:"
echo "export USE_SPECIFIC_USER=true"
echo "Run ./run-local.sh to start the mirroring process."

18
run-local.sh
View File

@@ -5,12 +5,28 @@ set -ex
docker image build -t jaedle/mirror-to-gitea:development .
source .secrets.rc
# Get host IP for Mac to connect to local Gitea instance
HOST_IP=$(ipconfig getifaddr en0)
echo "Using host IP for local Gitea: $HOST_IP"
GITEA_URL_DOCKER=${GITEA_URL/localhost/$HOST_IP}
echo "Gitea URL for Docker: $GITEA_URL_DOCKER"
docker container run \
-it \
--rm \
-e GITHUB_USERNAME="$GITHUB_USERNAME" \
-e GITEA_URL="$GITEA_URL" \
-e GITEA_URL="$GITEA_URL_DOCKER" \
-e GITEA_TOKEN="$GITEA_TOKEN" \
-e GITHUB_TOKEN="$GITHUB_TOKEN" \
-e MIRROR_PRIVATE_REPOSITORIES="true" \
-e MIRROR_ISSUES="true" \
-e MIRROR_STARRED="true" \
-e MIRROR_ORGANIZATIONS="true" \
-e USE_SPECIFIC_USER="$USE_SPECIFIC_USER" \
-e INCLUDE_ORGS="$INCLUDE_ORGS" \
-e EXCLUDE_ORGS="$EXCLUDE_ORGS" \
-e PRESERVE_ORG_STRUCTURE="$PRESERVE_ORG_STRUCTURE" \
-e GITEA_STARRED_ORGANIZATION="$GITEA_STARRED_ORGANIZATION" \
-e SKIP_STARRED_ISSUES="$SKIP_STARRED_ISSUES" \
-e DRY_RUN="true" \
jaedle/mirror-to-gitea:development

26
src/configuration.mjs
View File

@@ -35,10 +35,28 @@ export function configuration() {
token: process.env.GITHUB_TOKEN,
skipForks: readBoolean("SKIP_FORKS"),
privateRepositories: readBoolean("MIRROR_PRIVATE_REPOSITORIES"),
mirrorIssues: readBoolean("MIRROR_ISSUES"),
mirrorStarred: readBoolean("MIRROR_STARRED"),
mirrorOrganizations: readBoolean("MIRROR_ORGANIZATIONS"),
useSpecificUser: readBoolean("USE_SPECIFIC_USER"),
singleRepo: readEnv("SINGLE_REPO"),
includeOrgs: (readEnv("INCLUDE_ORGS") || "")
.split(",")
.map((o) => o.trim())
.filter((o) => o.length > 0),
excludeOrgs: (readEnv("EXCLUDE_ORGS") || "")
.split(",")
.map((o) => o.trim())
.filter((o) => o.length > 0),
preserveOrgStructure: readBoolean("PRESERVE_ORG_STRUCTURE"),
skipStarredIssues: readBoolean("SKIP_STARRED_ISSUES"),
},
gitea: {
url: mustReadEnv("GITEA_URL"),
token: mustReadEnv("GITEA_TOKEN"),
organization: readEnv("GITEA_ORGANIZATION"),
visibility: readEnv("GITEA_ORG_VISIBILITY") || "public",
starredReposOrg: readEnv("GITEA_STARRED_ORGANIZATION") || "github",
},
dryRun: readBoolean("DRY_RUN"),
delay: readInt("DELAY") ?? defaultDelay,
@@ -57,5 +75,13 @@ export function configuration() {
);
}
// GitHub token is required for mirroring issues, starred repos, and orgs
if ((config.github.mirrorIssues || config.github.mirrorStarred || config.github.mirrorOrganizations || config.github.singleRepo)
&& config.github.token === undefined) {
throw new Error(
"invalid configuration, mirroring issues, starred repositories, organizations, or a single repo requires setting GITHUB_TOKEN",
);
}
return config;
}

242
src/get-github-repositories.mjs
View File

@@ -1,14 +1,92 @@
async function getRepositories(octokit, mirrorOptions) {
const publicRepositories = await fetchPublicRepositories(
octokit,
mirrorOptions.username,
);
const privateRepos = mirrorOptions.privateRepositories
? await fetchPrivateRepositories(octokit)
: [];
const repos = [...publicRepositories, ...privateRepos];
let repositories = [];
// Check if we're mirroring a single repo
if (mirrorOptions.singleRepo) {
const singleRepo = await fetchSingleRepository(octokit, mirrorOptions.singleRepo);
if (singleRepo) {
repositories.push(singleRepo);
}
} else {
// Standard mirroring logic
const publicRepositories = await fetchPublicRepositories(
octokit,
mirrorOptions.username,
);
const privateRepos = mirrorOptions.privateRepositories
? await fetchPrivateRepositories(octokit)
: [];
// Fetch starred repos if the option is enabled
const starredRepos = mirrorOptions.mirrorStarred
? await fetchStarredRepositories(octokit, {
username: mirrorOptions.useSpecificUser ? mirrorOptions.username : undefined
})
: [];
// Fetch organization repos if the option is enabled
const orgRepos = mirrorOptions.mirrorOrganizations
? await fetchOrganizationRepositories(
octokit,
mirrorOptions.includeOrgs,
mirrorOptions.excludeOrgs,
mirrorOptions.preserveOrgStructure,
{
username: mirrorOptions.useSpecificUser ? mirrorOptions.username : undefined,
privateRepositories: mirrorOptions.privateRepositories
}
)
: [];
// Combine all repositories and filter duplicates
repositories = filterDuplicates([
...publicRepositories,
...privateRepos,
...starredRepos,
...orgRepos
]);
}
return mirrorOptions.skipForks ? withoutForks(repos) : repos;
return mirrorOptions.skipForks ? withoutForks(repositories) : repositories;
}
async function fetchSingleRepository(octokit, repoUrl) {
try {
// Remove URL prefix if present and clean up
let repoPath = repoUrl;
if (repoPath.startsWith('https://github.com/')) {
repoPath = repoPath.replace('https://github.com/', '');
}
if (repoPath.endsWith('.git')) {
repoPath = repoPath.slice(0, -4);
}
// Split into owner and repo
const [owner, repo] = repoPath.split('/');
if (!owner || !repo) {
console.error(`Invalid repository URL format: ${repoUrl}`);
return null;
}
// Fetch the repository details
const response = await octokit.rest.repos.get({
owner,
repo
});
return {
name: response.data.name,
url: response.data.clone_url,
private: response.data.private,
fork: response.data.fork,
owner: response.data.owner.login,
full_name: response.data.full_name,
has_issues: response.data.has_issues,
};
} catch (error) {
console.error(`Error fetching single repository ${repoUrl}:`, error.message);
return null;
}
}
async function fetchPublicRepositories(octokit, username) {
@@ -26,18 +104,160 @@ async function fetchPrivateRepositories(octokit) {
.then(toRepositoryList);
}
async function fetchStarredRepositories(octokit, options = {}) {
// If a specific username is provided, use the user-specific endpoint
if (options.username) {
return octokit
.paginate("GET /users/{username}/starred", {
username: options.username,
headers: {
'X-GitHub-Api-Version': '2022-11-28'
}
})
.then(repos => toRepositoryList(repos.map(repo => ({...repo, starred: true}))));
}
// Default: Get starred repos for the authenticated user (what was previously used)
return octokit
.paginate("GET /user/starred")
.then(repos => toRepositoryList(repos.map(repo => ({...repo, starred: true}))));
}
async function fetchOrganizationRepositories(octokit, includeOrgs = [], excludeOrgs = [], preserveOrgStructure = false, options = {}) {
try {
// Get all organizations the user belongs to
let allOrgs;
// If a specific username is provided, use the user-specific endpoint
if (options.username) {
allOrgs = await octokit.paginate("GET /users/{username}/orgs", {
username: options.username,
headers: {
'X-GitHub-Api-Version': '2022-11-28'
}
});
} else {
// Default: Get organizations for the authenticated user (what was previously used)
allOrgs = await octokit.paginate("GET /user/orgs");
}
// Filter organizations based on include/exclude lists
let orgsToProcess = allOrgs;
if (includeOrgs.length > 0) {
// Only include specific organizations
orgsToProcess = orgsToProcess.filter(org =>
includeOrgs.includes(org.login)
);
}
if (excludeOrgs.length > 0) {
// Exclude specific organizations
orgsToProcess = orgsToProcess.filter(org =>
!excludeOrgs.includes(org.login)
);
}
console.log(`Processing repositories from ${orgsToProcess.length} organizations`);
// Determine if we need to fetch private repositories
const privateRepoAccess = options.privateRepositories && octokit.auth;
const allOrgRepos = [];
// Process each organization
for (const org of orgsToProcess) {
const orgName = org.login;
console.log(`Fetching repositories for organization: ${orgName}`);
try {
let orgRepos = [];
// Use search API for organizations when private repositories are requested
// This is based on the GitHub community discussion recommendation
if (privateRepoAccess) {
console.log(`Using search API to fetch both public and private repositories for org: ${orgName}`);
// Query for both public and private repositories in the organization
const searchQuery = `org:${orgName}`;
const searchResults = await octokit.paginate("GET /search/repositories", {
q: searchQuery,
per_page: 100
});
// Search API returns repositories in the 'items' array
orgRepos = searchResults.flatMap(result => result.items || []);
console.log(`Found ${orgRepos.length} repositories (public and private) for org: ${orgName}`);
} else {
// Use standard API for public repositories only
orgRepos = await octokit.paginate("GET /orgs/{org}/repos", {
org: orgName
});
console.log(`Found ${orgRepos.length} public repositories for org: ${orgName}`);
}
// Add organization context to each repository if preserveOrgStructure is enabled
if (preserveOrgStructure) {
orgRepos = orgRepos.map(repo => ({
...repo,
organization: orgName
}));
}
allOrgRepos.push(...orgRepos);
} catch (orgError) {
console.error(`Error fetching repositories for org ${orgName}:`, orgError.message);
}
}
// Convert to repository list format
return toRepositoryList(allOrgRepos, preserveOrgStructure);
} catch (error) {
console.error("Error fetching organization repositories:", error.message);
return [];
}
}
function withoutForks(repositories) {
return repositories.filter((repo) => !repo.fork);
}
function toRepositoryList(repositories) {
function filterDuplicates(repositories) {
const unique = [];
const seen = new Set();
for (const repo of repositories) {
if (!seen.has(repo.url)) {
seen.add(repo.url);
unique.push(repo);
}
}
return unique;
}
function toRepositoryList(repositories, preserveOrgStructure = false) {
return repositories.map((repository) => {
return {
const repoInfo = {
name: repository.name,
url: repository.clone_url,
private: repository.private,
fork: repository.fork,
owner: repository.owner?.login,
full_name: repository.full_name,
has_issues: repository.has_issues,
};
// Add organization context if it exists and preserveOrgStructure is enabled
if (preserveOrgStructure && repository.organization) {
repoInfo.organization = repository.organization;
}
// Preserve starred status if present
if (repository.starred) {
repoInfo.starred = true;
}
return repoInfo;
});
}

563
src/index.mjs
View File

@@ -4,128 +4,7 @@ import PQueue from "p-queue";
import request from "superagent";
import { configuration } from "./configuration.mjs";
import { Logger } from "./logger.js";
async function getGithubRepositories(
username,
token,
mirrorPrivateRepositories,
mirrorForks,
include,
exclude,
) {
const octokit = new Octokit({
auth: token || null,
});
const publicRepositories = await octokit
.paginate("GET /users/:username/repos", { username: username })
.then((repositories) => toRepositoryList(repositories));
let allOwnedRepositories;
if (mirrorPrivateRepositories) {
allOwnedRepositories = await octokit
.paginate(
"GET /user/repos?visibility=public&affiliation=owner&visibility=private",
)
.then((repositories) => toRepositoryList(repositories));
}
let repositories = publicRepositories;
if (mirrorPrivateRepositories) {
repositories = filterDuplicates(
allOwnedRepositories.concat(publicRepositories),
);
}
if (!mirrorForks) {
repositories = repositories.filter((repository) => !repository.fork);
}
repositories = repositories.filter(
(repository) =>
include.some((f) => minimatch(repository.name, f)) &&
!exclude.some((f) => minimatch(repository.name, f)),
);
return repositories;
}
function toRepositoryList(repositories) {
return repositories.map((repository) => {
return {
name: repository.name,
url: repository.clone_url,
private: repository.private,
fork: repository.fork,
};
});
}
function filterDuplicates(array) {
const a = array.concat();
for (let i = 0; i < a.length; ++i) {
for (let j = i + 1; j < a.length; ++j) {
if (a[i].url === a[j].url) a.splice(j--, 1);
}
}
return a;
}
async function getGiteaUser(gitea) {
return request
.get(`${gitea.url}/api/v1/user`)
.set("Authorization", `token ${gitea.token}`)
.then((response) => {
return { id: response.body.id, name: response.body.username };
});
}
function isAlreadyMirroredOnGitea(repository, gitea, giteaUser) {
const requestUrl = `${gitea.url}/api/v1/repos/${giteaUser.name}/${repository}`;
return request
.get(requestUrl)
.set("Authorization", `token ${gitea.token}`)
.then(() => true)
.catch(() => false);
}
function mirrorOnGitea(repository, gitea, giteaUser, githubToken) {
request
.post(`${gitea.url}/api/v1/repos/migrate`)
.set("Authorization", `token ${gitea.token}`)
.send({
auth_token: githubToken || null,
clone_addr: repository.url,
mirror: true,
repo_name: repository.name,
uid: giteaUser.id,
private: repository.private,
})
.then(() => {
console.log("Did it!");
})
.catch((err) => {
console.log("Failed", err);
});
}
async function mirror(repository, gitea, giteaUser, githubToken, dryRun) {
if (await isAlreadyMirroredOnGitea(repository.name, gitea, giteaUser)) {
console.log(
"Repository is already mirrored; doing nothing: ",
repository.name,
);
return;
}
if (dryRun) {
console.log("DRY RUN: Would mirror repository to gitea: ", repository);
return;
}
console.log("Mirroring repository to gitea: ", repository.name);
await mirrorOnGitea(repository, gitea, giteaUser, githubToken);
}
import getGithubRepositories from "./get-github-repositories.mjs";
async function main() {
let config;
@@ -139,32 +18,137 @@ async function main() {
const logger = new Logger();
logger.showConfig(config);
const githubRepositories = await getGithubRepositories(
config.github.username,
config.github.token,
config.github.privateRepositories,
!config.github.skipForks,
config.include,
config.exclude,
// Create Gitea organization if specified
if (config.gitea.organization) {
await createGiteaOrganization(
{
url: config.gitea.url,
token: config.gitea.token,
},
config.gitea.organization,
config.gitea.visibility,
config.dryRun
);
}
// Create the starred repositories organization if mirror starred is enabled
if (config.github.mirrorStarred && config.gitea.starredReposOrg) {
await createGiteaOrganization(
{
url: config.gitea.url,
token: config.gitea.token,
},
config.gitea.starredReposOrg,
config.gitea.visibility,
config.dryRun
);
}
const octokit = new Octokit({
auth: config.github.token || null,
});
// Get user or organization repositories
const githubRepositories = await getGithubRepositories(octokit, {
username: config.github.username,
privateRepositories: config.github.privateRepositories,
skipForks: config.github.skipForks,
mirrorStarred: config.github.mirrorStarred,
mirrorOrganizations: config.github.mirrorOrganizations,
singleRepo: config.github.singleRepo,
includeOrgs: config.github.includeOrgs,
excludeOrgs: config.github.excludeOrgs,
preserveOrgStructure: config.github.preserveOrgStructure,
});
// Apply include/exclude filters
const filteredRepositories = githubRepositories.filter(
(repository) =>
config.include.some((f) => minimatch(repository.name, f)) &&
!config.exclude.some((f) => minimatch(repository.name, f)),
);
console.log(`Found ${githubRepositories.length} repositories on github`);
console.log(`Found ${filteredRepositories.length} repositories to mirror`);
const gitea = {
url: config.gitea.url,
token: config.gitea.token,
};
const giteaUser = await getGiteaUser(gitea);
// Get Gitea user information
const giteaUser = await getGiteaUser(gitea);
if (!giteaUser) {
console.error("Failed to get Gitea user. Exiting.");
process.exit(1);
}
// Create a map to store organization targets if preserving structure
const orgTargets = new Map();
if (config.github.preserveOrgStructure) {
// Get unique organization names from repositories
const uniqueOrgs = new Set(
filteredRepositories
.filter(repo => repo.organization)
.map(repo => repo.organization)
);
// Create or get each organization in Gitea
for (const orgName of uniqueOrgs) {
console.log(`Preparing Gitea organization for GitHub organization: ${orgName}`);
// Create the organization if it doesn't exist
await createGiteaOrganization(
gitea,
orgName,
config.gitea.visibility,
config.dryRun
);
// Get the organization details
const orgTarget = await getGiteaOrganization(gitea, orgName);
if (orgTarget) {
orgTargets.set(orgName, orgTarget);
} else {
console.error(`Failed to get or create Gitea organization: ${orgName}`);
}
}
}
// Mirror repositories
const queue = new PQueue({ concurrency: 4 });
await queue.addAll(
githubRepositories.map((repository) => {
filteredRepositories.map((repository) => {
return async () => {
// Determine the target (user or organization)
let giteaTarget;
if (config.github.preserveOrgStructure && repository.organization) {
// Use the organization as target
giteaTarget = orgTargets.get(repository.organization);
if (!giteaTarget) {
console.error(`No Gitea organization found for ${repository.organization}, using user instead`);
giteaTarget = config.gitea.organization
? await getGiteaOrganization(gitea, config.gitea.organization)
: giteaUser;
}
} else {
// Use the specified organization or user
giteaTarget = config.gitea.organization
? await getGiteaOrganization(gitea, config.gitea.organization)
: giteaUser;
}
await mirror(
repository,
gitea,
giteaUser,
{
url: config.gitea.url,
token: config.gitea.token,
skipStarredIssues: config.github.skipStarredIssues,
starredReposOrg: config.gitea.starredReposOrg
},
giteaTarget,
config.github.token,
config.github.mirrorIssues,
config.dryRun,
);
};
@@ -172,4 +156,313 @@ async function main() {
);
}
main();
// Get Gitea user information
async function getGiteaUser(gitea) {
try {
const response = await request
.get(`${gitea.url}/api/v1/user`)
.set("Authorization", `token ${gitea.token}`);
return {
id: response.body.id,
name: response.body.username,
type: "user"
};
} catch (error) {
console.error("Error fetching Gitea user:", error.message);
return null;
}
}
// Get Gitea organization information
async function getGiteaOrganization(gitea, orgName) {
try {
const response = await request
.get(`${gitea.url}/api/v1/orgs/${orgName}`)
.set("Authorization", `token ${gitea.token}`);
return {
id: response.body.id,
name: orgName,
type: "organization"
};
} catch (error) {
console.error(`Error fetching Gitea organization ${orgName}:`, error.message);
return null;
}
}
// Create a Gitea organization
async function createGiteaOrganization(gitea, orgName, visibility, dryRun) {
if (dryRun) {
console.log(`DRY RUN: Would create Gitea organization: ${orgName} (${visibility})`);
return true;
}
try {
// First check if organization already exists
try {
const existingOrg = await request
.get(`${gitea.url}/api/v1/orgs/${orgName}`)
.set("Authorization", `token ${gitea.token}`);
console.log(`Organization ${orgName} already exists`);
return true;
} catch (checkError) {
// Organization doesn't exist, continue to create it
}
const response = await request
.post(`${gitea.url}/api/v1/orgs`)
.set("Authorization", `token ${gitea.token}`)
.send({
username: orgName,
visibility: visibility || "public",
});
console.log(`Created organization: ${orgName}`);
return true;
} catch (error) {
// 422 error typically means the organization already exists
if (error.status === 422) {
console.log(`Organization ${orgName} already exists`);
return true;
}
console.error(`Error creating Gitea organization ${orgName}:`, error.message);
return false;
}
}
// Check if repository is already mirrored
async function isAlreadyMirroredOnGitea(repository, gitea, giteaTarget) {
const repoName = repository.name;
const ownerName = giteaTarget.name;
const requestUrl = `${gitea.url}/api/v1/repos/${ownerName}/${repoName}`;
try {
await request
.get(requestUrl)
.set("Authorization", `token ${gitea.token}`);
return true;
} catch (error) {
return false;
}
}
// Mirror repository to Gitea
async function mirrorOnGitea(repository, gitea, giteaTarget, githubToken) {
try {
const response = await request
.post(`${gitea.url}/api/v1/repos/migrate`)
.set("Authorization", `token ${gitea.token}`)
.send({
auth_token: githubToken || null,
clone_addr: repository.url,
mirror: true,
repo_name: repository.name,
uid: giteaTarget.id,
private: repository.private,
});
console.log(`Successfully mirrored: ${repository.name}`);
return response.body;
} catch (error) {
console.error(`Failed to mirror ${repository.name}:`, error.message);
throw error;
}
}
// Fetch issues for a repository
async function getGithubIssues(octokit, owner, repo) {
try {
const issues = await octokit.paginate("GET /repos/{owner}/{repo}/issues", {
owner,
repo,
state: "all",
per_page: 100,
});
return issues.map(issue => ({
title: issue.title,
body: issue.body || "",
state: issue.state,
labels: issue.labels.map(label => label.name),
closed: issue.state === "closed",
created_at: issue.created_at,
updated_at: issue.updated_at,
number: issue.number,
user: issue.user.login,
}));
} catch (error) {
console.error(`Error fetching issues for ${owner}/${repo}:`, error.message);
return [];
}
}
// Create an issue in a Gitea repository
async function createGiteaIssue(issue, repository, gitea, giteaTarget) {
try {
const response = await request
.post(`${gitea.url}/api/v1/repos/${giteaTarget.name}/${repository.name}/issues`)
.set("Authorization", `token ${gitea.token}`)
.send({
title: issue.title,
body: `*Originally created by @${issue.user} on ${new Date(issue.created_at).toLocaleDateString()}*\n\n${issue.body}`,
state: issue.state,
closed: issue.closed,
});
console.log(`Created issue #${response.body.number}: ${issue.title}`);
// Add labels if the issue has any
if (issue.labels && issue.labels.length > 0) {
await Promise.all(issue.labels.map(async (label) => {
try {
// First try to create the label if it doesn't exist
await request
.post(`${gitea.url}/api/v1/repos/${giteaTarget.name}/${repository.name}/labels`)
.set("Authorization", `token ${gitea.token}`)
.send({
name: label,
color: "#" + Math.floor(Math.random() * 16777215).toString(16), // Random color
})
.catch(() => {
// Label might already exist, which is fine
});
// Then add the label to the issue
await request
.post(`${gitea.url}/api/v1/repos/${giteaTarget.name}/${repository.name}/issues/${response.body.number}/labels`)
.set("Authorization", `token ${gitea.token}`)
.send({
labels: [label]
});
} catch (labelError) {
console.error(`Error adding label ${label} to issue:`, labelError.message);
}
}));
}
return response.body;
} catch (error) {
console.error(`Error creating issue "${issue.title}":`, error.message);
return null;
}
}
// Mirror issues for a repository
async function mirrorIssues(repository, gitea, giteaTarget, githubToken, dryRun) {
if (!repository.has_issues) {
console.log(`Repository ${repository.name} doesn't have issues enabled. Skipping issues mirroring.`);
return;
}
if (dryRun) {
console.log(`DRY RUN: Would mirror issues for repository: ${repository.name}`);
return;
}
try {
const octokit = new Octokit({ auth: githubToken });
const owner = repository.owner || repository.full_name.split('/')[0];
const issues = await getGithubIssues(octokit, owner, repository.name);
console.log(`Found ${issues.length} issues for ${repository.name}`);
// Create issues one by one to maintain order
for (const issue of issues) {
await createGiteaIssue(issue, repository, gitea, giteaTarget);
}
console.log(`Completed mirroring issues for ${repository.name}`);
} catch (error) {
console.error(`Error mirroring issues for ${repository.name}:`, error.message);
}
}
// Mirror a repository
async function mirror(repository, gitea, giteaTarget, githubToken, mirrorIssuesFlag, dryRun) {
// For starred repositories, use the starred repos organization if configured
if (repository.starred && gitea.starredReposOrg) {
// Get the starred repos organization
const starredOrg = await getGiteaOrganization(gitea, gitea.starredReposOrg);
if (starredOrg) {
console.log(`Using organization "${gitea.starredReposOrg}" for starred repository: ${repository.name}`);
giteaTarget = starredOrg;
} else {
console.log(`Could not find organization "${gitea.starredReposOrg}" for starred repositories, using default target`);
}
}
const isAlreadyMirrored = await isAlreadyMirroredOnGitea(repository, gitea, giteaTarget);
// Special handling for starred repositories
if (repository.starred) {
if (isAlreadyMirrored) {
console.log(`Repository ${repository.name} is already mirrored in ${giteaTarget.type} ${giteaTarget.name}; checking if it needs to be starred.`);
await starRepositoryInGitea(repository, gitea, giteaTarget, dryRun);
return;
} else if (dryRun) {
console.log(`DRY RUN: Would mirror and star repository to ${giteaTarget.type} ${giteaTarget.name}: ${repository.name} (starred)`);
return;
}
} else if (isAlreadyMirrored) {
console.log(`Repository ${repository.name} is already mirrored in ${giteaTarget.type} ${giteaTarget.name}; doing nothing.`);
return;
} else if (dryRun) {
console.log(`DRY RUN: Would mirror repository to ${giteaTarget.type} ${giteaTarget.name}: ${repository.name}`);
return;
}
console.log(`Mirroring repository to ${giteaTarget.type} ${giteaTarget.name}: ${repository.name}${repository.starred ? ' (will be starred)' : ''}`);
try {
await mirrorOnGitea(repository, gitea, giteaTarget, githubToken);
// Star the repository if it's marked as starred
if (repository.starred) {
await starRepositoryInGitea(repository, gitea, giteaTarget, dryRun);
}
// Mirror issues if requested and not in dry run mode
// Skip issues for starred repos if the skipStarredIssues option is enabled
const shouldMirrorIssues = mirrorIssuesFlag &&
!(repository.starred && gitea.skipStarredIssues);
if (shouldMirrorIssues && !dryRun) {
await mirrorIssues(repository, gitea, giteaTarget, githubToken, dryRun);
} else if (repository.starred && gitea.skipStarredIssues) {
console.log(`Skipping issues for starred repository: ${repository.name}`);
}
} catch (error) {
console.error(`Error during mirroring of ${repository.name}:`, error.message);
}
}
// Star a repository in Gitea
async function starRepositoryInGitea(repository, gitea, giteaTarget, dryRun) {
const ownerName = giteaTarget.name;
const repoName = repository.name;
if (dryRun) {
console.log(`DRY RUN: Would star repository in Gitea: ${ownerName}/${repoName}`);
return true;
}
try {
await request
.put(`${gitea.url}/api/v1/user/starred/${ownerName}/${repoName}`)
.set("Authorization", `token ${gitea.token}`);
console.log(`Successfully starred repository in Gitea: ${ownerName}/${repoName}`);
return true;
} catch (error) {
console.error(`Error starring repository ${ownerName}/${repoName}:`, error.message);
return false;
}
}
main().catch(error => {
console.error("Application error:", error);
process.exit(1);
});