From d5b80f8bd425a17832ac1634f73146e574df052c Mon Sep 17 00:00:00 2001 From: Carlos Monastyrski Date: Fri, 19 Dec 2025 15:10:14 -0300 Subject: [PATCH] Fix Github edge case when fetching many repositories, overwhelming the relay --- .../github/github-connection-fns.ts | 159 ++++++++++++------ 1 file changed, 107 insertions(+), 52 deletions(-) diff --git a/backend/src/services/app-connection/github/github-connection-fns.ts b/backend/src/services/app-connection/github/github-connection-fns.ts index 806df039f2..da17757ee0 100644 --- a/backend/src/services/app-connection/github/github-connection-fns.ts +++ b/backend/src/services/app-connection/github/github-connection-fns.ts @@ -48,11 +48,24 @@ export const getGitHubInstanceApiUrl = async (config: { return apiBase; }; +export const getGitHubGatewayConnectionDetails = async ( + gatewayId: string, + targetHost: string, + gatewayV2Service: Pick +): Promise>> => { + return gatewayV2Service.getPlatformConnectionDetailsByGatewayId({ + gatewayId, + targetHost, + targetPort: 443 + }); +}; + export const requestWithGitHubGateway = async ( appConnection: { gatewayId?: string | null }, gatewayService: Pick, gatewayV2Service: Pick, - requestConfig: AxiosRequestConfig + requestConfig: AxiosRequestConfig, + gatewayConnectionDetails?: Awaited> ): Promise> => { const { gatewayId } = appConnection; @@ -66,13 +79,10 @@ export const requestWithGitHubGateway = async ( await blockLocalAndPrivateIpAddresses(url.toString()); const [targetHost] = await verifyHostInputValidity(url.host, true); - const gatewayConnectionDetails = await gatewayV2Service.getPlatformConnectionDetailsByGatewayId({ - gatewayId, - targetHost, - targetPort: 443 - }); - if (gatewayConnectionDetails) { + const connectionDetails = gatewayConnectionDetails; + + if (connectionDetails) { return withGatewayV2Proxy( async (proxyPort) => { const httpsAgent = new https.Agent({ @@ -105,9 +115,9 @@ export const requestWithGitHubGateway = async ( }, { protocol: GatewayProxyProtocol.Tcp, - relayHost: gatewayConnectionDetails.relayHost, - gateway: gatewayConnectionDetails.gateway, - relay: gatewayConnectionDetails.relay + relayHost: connectionDetails.relayHost, + gateway: connectionDetails.gateway, + relay: connectionDetails.relay } ); } @@ -198,6 +208,10 @@ export const getGitHubAppAuthToken = async ( const apiBaseUrl = await getGitHubInstanceApiUrl(appConnection); const { installationId } = appConnection.credentials; + const gatewayConnectionDetails = appConnection.gatewayId + ? await getGitHubGatewayConnectionDetails(appConnection.gatewayId, apiBaseUrl, gatewayV2Service) + : undefined; + const response = await requestWithGitHubGateway<{ token: string; expires_at: string }>( appConnection, gatewayService, @@ -210,7 +224,8 @@ export const getGitHubAppAuthToken = async ( Authorization: `Bearer ${appJwt}`, "X-GitHub-Api-Version": "2022-11-28" } - } + }, + gatewayConnectionDetails ); return response.data.token; @@ -265,6 +280,11 @@ export const makePaginatedGitHubRequest = async ( const initialUrlObj = new URL(baseUrl); initialUrlObj.searchParams.set("per_page", "100"); + const apiBaseUrl = await getGitHubInstanceApiUrl(appConnection); + const gatewayConnectionDetails = appConnection.gatewayId + ? await getGitHubGatewayConnectionDetails(appConnection.gatewayId, apiBaseUrl, gatewayV2Service) + : undefined; + let results: T[] = []; const maxIterations = 1000; @@ -281,7 +301,8 @@ export const makePaginatedGitHubRequest = async ( Authorization: `Bearer ${token}`, "X-GitHub-Api-Version": "2022-11-28" } - } + }, + gatewayConnectionDetails ); const firstPageItems = dataMapper ? dataMapper(firstResponse.data) : (firstResponse.data as unknown as T[]); @@ -302,15 +323,21 @@ export const makePaginatedGitHubRequest = async ( pageUrlObj.searchParams.set("page", pageNum.toString()); pageRequests.push( - requestWithGitHubGateway(appConnection, gatewayService, gatewayV2Service, { - url: pageUrlObj.toString(), - method: "GET", - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "X-GitHub-Api-Version": "2022-11-28" - } - }) + requestWithGitHubGateway( + appConnection, + gatewayService, + gatewayV2Service, + { + url: pageUrlObj.toString(), + method: "GET", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }, + gatewayConnectionDetails + ) ); } const responses = await Promise.all(pageRequests); @@ -338,7 +365,8 @@ export const makePaginatedGitHubRequest = async ( Authorization: `Bearer ${token}`, "X-GitHub-Api-Version": "2022-11-28" } - } + }, + gatewayConnectionDetails ); const items = dataMapper ? dataMapper(response.data) : (response.data as unknown as T[]); @@ -469,19 +497,30 @@ export const validateGitHubConnectionCredentials = async ( ) => { const { credentials, method } = config; + const apiBaseUrl = await getGitHubInstanceApiUrl(config); + const gatewayConnectionDetails = config.gatewayId + ? await getGitHubGatewayConnectionDetails(config.gatewayId, apiBaseUrl, gatewayV2Service) + : undefined; + // PAT validation if (method === GitHubConnectionMethod.Pat) { try { const apiUrl = await getGitHubInstanceApiUrl(config); - await requestWithGitHubGateway(config, gatewayService, gatewayV2Service, { - url: `https://${apiUrl}/user`, - method: "GET", - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${credentials.personalAccessToken}`, - "X-GitHub-Api-Version": "2022-11-28" - } - }); + await requestWithGitHubGateway( + config, + gatewayService, + gatewayV2Service, + { + url: `https://${apiUrl}/user`, + method: "GET", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${credentials.personalAccessToken}`, + "X-GitHub-Api-Version": "2022-11-28" + } + }, + gatewayConnectionDetails + ); return { personalAccessToken: credentials.personalAccessToken, @@ -529,21 +568,31 @@ export const validateGitHubConnectionCredentials = async ( let tokenResp: AxiosResponse; const host = credentials.host || "github.com"; + const oauthGatewayConnectionDetails = config.gatewayId + ? await getGitHubGatewayConnectionDetails(config.gatewayId, host, gatewayV2Service) + : undefined; + try { - tokenResp = await requestWithGitHubGateway(config, gatewayService, gatewayV2Service, { - url: `https://${host}/login/oauth/access_token`, - method: "POST", - data: { - client_id: clientId, - client_secret: clientSecret, - code: credentials.code, - redirect_uri: `${SITE_URL}/organization/app-connections/github/oauth/callback` + tokenResp = await requestWithGitHubGateway( + config, + gatewayService, + gatewayV2Service, + { + url: `https://${host}/login/oauth/access_token`, + method: "POST", + data: { + client_id: clientId, + client_secret: clientSecret, + code: credentials.code, + redirect_uri: `${SITE_URL}/organization/app-connections/github/oauth/callback` + }, + headers: { + Accept: "application/json", + "Content-Type": "application/json" + } }, - headers: { - Accept: "application/json", - "Content-Type": "application/json" - } - }); + oauthGatewayConnectionDetails + ); if (isGithubErrorResponse(tokenResp?.data)) { throw new BadRequestError({ @@ -582,14 +631,20 @@ export const validateGitHubConnectionCredentials = async ( id: number; }; }[]; - }>(config, gatewayService, gatewayV2Service, { - url: `https://${await getGitHubInstanceApiUrl(config)}/user/installations`, - headers: { - Accept: "application/json", - Authorization: `Bearer ${tokenResp.data.access_token}`, - "Accept-Encoding": "application/json" - } - }); + }>( + config, + gatewayService, + gatewayV2Service, + { + url: `https://${await getGitHubInstanceApiUrl(config)}/user/installations`, + headers: { + Accept: "application/json", + Authorization: `Bearer ${tokenResp.data.access_token}`, + "Accept-Encoding": "application/json" + } + }, + gatewayConnectionDetails + ); const matchingInstallation = installationsResp.data.installations.find( (installation) => installation.id === +credentials.installationId