diff --git a/README.md b/README.md index e377ea4..57f99e6 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,26 @@ # GH Groups -Get list of GH users who contributed to the GitHub org of a given group of DAOs. +Get list of GH users who contributed to the GitHub org of a given group of DAOs. +Under the hood, it: + +1. Fetches a list of `maxOrgs` (default 100) spaces from snapshot.org based on their `minFollowers` (default 10_000). +2. For each space, it fetches the list of their repositories on GitHub. +3. For each of the org repository, it fetches the list of contributors between a `since` and `until`timestamps (default between now and now - 1 month). +4. Returns the list of unique contributors, removing potential bots (e.g dependencies, github action bots etc...) ## Usage -- script: `nps 'fetch -m -s ` -- node +You need to have a GH Personal Access Token (scopes: `public_repo`, `read:user`) defined as environment variables. - ``` - import { getSpaces} from './src/getSpaces' +### Scripts - const spaces = await getSpaces({min, size})() - ``` +- fetch spaces: `nps 'fetch.spaces -m -s ` +- fetch group of gh users: `nps fetch.ghgroup` + +### Node + +``` +import { getGhGroup } from './src' + +const group = await getGhGroup() +``` diff --git a/package-scripts.yaml b/package-scripts.yaml index 9257853..237203a 100644 --- a/package-scripts.yaml +++ b/package-scripts.yaml @@ -17,8 +17,12 @@ scripts: hiddenFromHelp: Fix linting and formatting errors fetch: - script: tsnd --cls --respawn --transpile-only scripts - description: Fetch query results (-m -s ) + spaces: + script: tsnd --cls --transpile-only scripts/get-spaces.ts + description: Fetch query results (-m -s ) + ghgroup: + script: tsnd --cls --transpile-only scripts/get-gh-group.ts + description: Fetch GH group members format: default: diff --git a/scripts/get-gh-group.ts b/scripts/get-gh-group.ts new file mode 100644 index 0000000..284153f --- /dev/null +++ b/scripts/get-gh-group.ts @@ -0,0 +1,13 @@ +import { getGhGroup } from '../src' + +const main = async () => { + const ghGroup = await getGhGroup() + console.log(ghGroup) +} + +main() + .then(() => process.exit(0)) + .catch((err) => { + console.error(err) + process.exit(1) + }) diff --git a/scripts/index.ts b/scripts/get-spaces.ts similarity index 71% rename from scripts/index.ts rename to scripts/get-spaces.ts index c5b8100..e4ab35c 100644 --- a/scripts/index.ts +++ b/scripts/get-spaces.ts @@ -1,29 +1,32 @@ import { hideBin } from 'yargs/helpers' import yargs from 'yargs/yargs' -import { getSpaces } from '../src' +import { getSpaces } from '../src/get-spaces' const options = { - min: { - alias: 'm', - describe: 'DAO min number of followers on snapshot.org', - type: 'number', - }, - size: { + maxOrgs: { alias: 's', describe: 'number of org to fetch from snapshot.org. 0 means no limit', type: 'number', }, + minFollowers: { + alias: 'm', + describe: 'DAO min number of followers on snapshot.org', + type: 'number', + }, } // @ts-expect-error const argv = yargs(hideBin(process.argv)).options(options).help().argv as { - min: number - size: number + minFollowers: number + maxOrgs: number } const main = async () => { - const spaces = await getSpaces({ min: argv.min, size: argv.size })() + const spaces = await getSpaces({ + maxOrgs: argv.maxOrgs, + minFollowers: argv.minFollowers, + })() console.log(spaces) } diff --git a/src/get-commiters-by-org.ts b/src/get-commiters-by-org.ts index d75f730..00ec4d5 100644 --- a/src/get-commiters-by-org.ts +++ b/src/get-commiters-by-org.ts @@ -2,6 +2,8 @@ import { ok } from 'assert' import { URLS } from './constants' import committersQuery from './graphql/committers-query' +const parseDate = (date: Date) => date.toISOString().split('.')[0] + 'Z' + export const getCommittersByOrg = async ({ org, since, @@ -11,14 +13,15 @@ export const getCommittersByOrg = async ({ since: Date until: Date }) => { + // console.log(parseDate(since), parseDate(until)) ok(process.env.GH_PAT, 'GH_PAT is not defined') const res = await fetch(URLS.GH_SQL, { body: JSON.stringify({ query: committersQuery, variables: { login: org, - since: since.toISOString(), - until: until.toISOString(), + since: parseDate(since), + until: parseDate(until), }, }), headers: { @@ -34,14 +37,17 @@ export const getCommittersByOrg = async ({ return [ ...new Set( (repos as any[]) - .reduce>((repos, repo) => { + .reduce((repos, repo) => { // console.log(repo) if (repo.defaultBranchRef !== null) { repos.push( - repo.defaultBranchRef.target.history.nodes.map((node: any) => ({ - date: node.committedDate, - user: node.author.user.login, - })), + (repo.defaultBranchRef.target.history.nodes as any[]).reduce< + string[] + >((users, node) => { + const login: string = node.author.user?.login + if (login !== undefined) users.push(login) + return users + }, []), ) } return repos diff --git a/src/get-gh-group.ts b/src/get-gh-group.ts new file mode 100644 index 0000000..713bcee --- /dev/null +++ b/src/get-gh-group.ts @@ -0,0 +1,30 @@ +import { getCommittersByOrg } from './get-commiters-by-org' +import { getGhOrgs } from './get-gh-orgs' + +const minusOneMonth = (date: Date) => + new Date(new Date().setMonth(date.getMonth() - 1)) + +export const getGhGroup = async ( + { + maxOrgs = 100, + minFollowers = 10_000, + since, + until = new Date(), + }: { + maxOrgs?: number + minFollowers?: number + since?: Date + until?: Date + } = { maxOrgs: 100, minFollowers: 10_000, until: new Date() }, +) => { + if (since === undefined) since = minusOneMonth(until) + const orgs = await getGhOrgs({ maxOrgs, minFollowers }) + + const users = await Promise.all( + orgs.map(async (org) => { + // @ts-expect-error - type guarded earlier + return getCommittersByOrg({ org, since, until }) + }), + ) + return [...new Set(users.flat())].filter((user) => !user.includes('[bot]')) +} diff --git a/src/get-gh-orgs.ts b/src/get-gh-orgs.ts index 008bd0c..95690a9 100644 --- a/src/get-gh-orgs.ts +++ b/src/get-gh-orgs.ts @@ -2,14 +2,18 @@ import { URLS } from './constants' import { getSpaces } from './get-spaces' import spacesGqlQuery from './graphql/spaces-gql-query' -export const getGhOrgs = async ({ - min, - size, -}: { - min: number - size: number -}): Promise => { - const spacesIds = (await getSpaces({ min, size })()).map(({ id }) => id) +export const getGhOrgs = async ( + { + maxOrgs = 100, + minFollowers = 10_000, + }: { + minFollowers: number + maxOrgs: number + } = { maxOrgs: 100, minFollowers: 10_000 }, +): Promise => { + const spacesIds = (await getSpaces({ maxOrgs, minFollowers })()).map( + ({ id }) => id, + ) const res = await fetch(URLS.SNAPSHOT_GQL, { body: JSON.stringify({ diff --git a/src/get-spaces.ts b/src/get-spaces.ts index 79bc3eb..52340f4 100644 --- a/src/get-spaces.ts +++ b/src/get-spaces.ts @@ -2,19 +2,27 @@ import { URLS } from './constants' import { Space } from './types' export const filterSpaces = - (min: number) => + (minFollowers: number) => ({ followers }: Space) => - followers !== undefined && followers >= min + followers !== undefined && followers >= minFollowers export const getSpaces = - ({ min = 10, size = 100 }: { min: number; size: number }) => + ( + { + maxOrgs = 100, + minFollowers = 10, + }: { minFollowers: number; maxOrgs: number } = { + maxOrgs: 100, + minFollowers: 10_000, + }, + ) => async () => fetch(URLS.SNAPSHOT_EXPLORE).then(async (res) => res.json().then((res) => Object.entries(res.spaces as Space[]) .reduce>( (spaces, [id, space]) => { - if (filterSpaces(min)(space)) { + if (filterSpaces(minFollowers)(space)) { // @ts-expect-error - filterSpaces already ensures that props are defined spaces.push({ followers: space.followers, id }) } @@ -24,11 +32,8 @@ export const getSpaces = [], ) .sort((a, b) => b.followers - a.followers) - .slice(0, size), + .slice(0, maxOrgs), ), ) -export const get100TopDaosWithMin10kFollowers = getSpaces({ - min: 10_000, - size: 0, -}) +export const get100TopDaosWithMin10kFollowers = getSpaces() diff --git a/src/index.ts b/src/index.ts index c60b961..d90b693 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1 +1 @@ -export * from './get-spaces' +export * from './get-gh-group' diff --git a/test/unit/filter-spaces.test.ts b/test/unit/filter-spaces.test.ts index 31e5f6b..43ed3cf 100644 --- a/test/unit/filter-spaces.test.ts +++ b/test/unit/filter-spaces.test.ts @@ -1,4 +1,4 @@ -import { filterSpaces } from '../../src' +import { filterSpaces } from '../../src/get-spaces' describe('filterSpaces', () => { it('should return true if followers is greater than min', () => { diff --git a/test/unit/get-committers.test.ts b/test/unit/get-committers.test.ts index fea9c88..4c059b0 100644 --- a/test/unit/get-committers.test.ts +++ b/test/unit/get-committers.test.ts @@ -13,12 +13,9 @@ describe('getCommitters', () => { }) expect(committers.length).toBeGreaterThan(0) - committers.forEach(({ date, user }) => { + committers.forEach((user) => { expect(typeof user).toBe('string') expect(user).toBeTruthy() - expect(typeof date).toBe('string') - expect(new Date(date).getTime()).toBeGreaterThan(SINCE.getTime()) - expect(new Date(date).getTime()).toBeLessThan(UNTIL.getTime()) }) }) }) diff --git a/test/unit/get-gh-group.test.ts b/test/unit/get-gh-group.test.ts new file mode 100644 index 0000000..099f481 --- /dev/null +++ b/test/unit/get-gh-group.test.ts @@ -0,0 +1,17 @@ +import { getGhGroup } from '../../src/get-gh-group' + +jest.setTimeout(20_000) + +describe('getGhGroup', () => { + it('should return a list of users', async () => { + const users = await getGhGroup({ maxOrgs: 5, minFollowers: 10_000 }) + expect(users.length).toBeGreaterThan(0) + users.forEach((user) => { + expect(typeof user).toBe('string') + expect(user).toBeTruthy() + expect(user.includes('[bot]')).toBeFalsy() + }) + // no duplicates + expect(Array.from(new Set(users)).sort()).toEqual(users.sort()) + }) +}) diff --git a/test/unit/get-gh-orgs.test.ts b/test/unit/get-gh-orgs.test.ts index 0281bca..e36d40b 100644 --- a/test/unit/get-gh-orgs.test.ts +++ b/test/unit/get-gh-orgs.test.ts @@ -2,13 +2,16 @@ import { getGhOrgs } from '../../src/get-gh-orgs' describe('get-gh-orgs', () => { it('should return an array of github orgs', async () => { - const MIN = 20_000 - const SIZE = 5 + const MIN_FOLLOWERS = 20_000 + const MAX_ORGS = 5 - const orgs = await getGhOrgs({ min: MIN, size: SIZE }) + const orgs = await getGhOrgs({ + maxOrgs: MAX_ORGS, + minFollowers: MIN_FOLLOWERS, + }) expect(orgs.length).toBeGreaterThan(0) - expect(orgs.length).toBeLessThanOrEqual(SIZE) + expect(orgs.length).toBeLessThanOrEqual(MAX_ORGS) orgs.forEach((org) => { expect(org).toBeDefined().toBeTruthy() }) diff --git a/test/unit/get-spaces.test.ts b/test/unit/get-spaces.test.ts index 6616e2a..f6b44a4 100644 --- a/test/unit/get-spaces.test.ts +++ b/test/unit/get-spaces.test.ts @@ -1,17 +1,23 @@ -import { get100TopDaosWithMin10kFollowers, getSpaces } from '../../src' +import { + get100TopDaosWithMin10kFollowers, + getSpaces, +} from '../../src/get-spaces' describe('getSpaces', () => { it('should return an array of spaces', async () => { - const MIN = 20_000 - const SIZE = 50 + const MIN_FOLLOWERS = 20_000 + const MAX_ORGS = 50 - const spaces = await getSpaces({ min: MIN, size: SIZE })() + const spaces = await getSpaces({ + maxOrgs: MAX_ORGS, + minFollowers: MIN_FOLLOWERS, + })() expect(spaces.length).toBeGreaterThan(0) - expect(spaces.length).toBeLessThanOrEqual(SIZE) + expect(spaces.length).toBeLessThanOrEqual(MAX_ORGS) spaces.forEach(({ followers, id }) => { expect(id).toBeDefined().toBeTruthy() - expect(followers).toBeGreaterThan(MIN) + expect(followers).toBeGreaterThan(MIN_FOLLOWERS) }) }) })