renaming for clarity & adding queued invalidation 🌠

packages/backend/package.json

@@ -31,13 +31,14 @@
     "@types/uuid": "8.3.0",
     "bluebird": "3.7.2",
     "body-parser": "1.19.0",
+    "bottleneck": "2.19.5",
     "discord.js": "12.5.3",
     "ethers": "5.4.1",
     "express": "4.17.1",
     "express-graphql": "0.12.0",
     "graphql": "15.5.0",
     "graphql-request": "3.4.0",
-    "graphql-tag": "2.12.4",
+    "graphql-tag": "2.12.5",
     "graphql-tools": "7.0.4",
     "imgix-core-js": "2.3.2",
     "node-fetch": "2.6.1",

packages/backend/src/handlers/actions/3Box/cache3BoxProfile/handler.ts

@@ -1,20 +0,0 @@
-import { Request, Response } from 'express';
-
-import { updateCachedProfile } from './updateCachedProfile';
-
-export const cache3BoxProfileHandler = async (
-  req: Request,
-  res: Response,
-): Promise<void> => {
-  const session = req.body.session_variables;
-  const role = session['x-hasura-role'];
-  const playerId = session['x-hasura-user-id'];
-
-  if (role !== 'player') {
-    throw new Error('expected role player');
-  }
-
-  const result = await updateCachedProfile(playerId);
-
-  res.json(result);
-};

packages/backend/src/handlers/actions/3Box/refresh3BoxCache/cache3BoxProfiles.ts

@@ -1,16 +0,0 @@
-import { CacheProcessOutput } from '../../../../lib/autogen/hasura-sdk';
-import { client } from '../../../../lib/hasuraClient';
-import { updateCachedProfile } from '../cache3BoxProfile/updateCachedProfile';
-
-export async function cache3BoxProfiles(): Promise<CacheProcessOutput> {
-  const data = await client.GetPlayerIds();
-  const ids = data.player.map(({ id }) => id);
-
-  try {
-    await Promise.all(ids.map((id) => updateCachedProfile(id)));
-  } catch (err) {
-    return { success: false, error: err.message };
-  }
-
-  return { success: true, error: null };
-}

packages/backend/src/handlers/actions/3Box/refresh3BoxCache/handler.ts

@@ -1,12 +0,0 @@
-import { Request, Response } from 'express';
-
-import { cache3BoxProfiles } from './cache3BoxProfiles';
-
-export const refresh3BoxCacheHandler = async (
-  _req: Request,
-  res: Response,
-): Promise<void> => {
-  const result = await cache3BoxProfiles();
-
-  res.json(result);
-};

packages/backend/src/handlers/actions/3Box/routes.ts

@@ -1,11 +0,0 @@
-import express from 'express';
-
-import { asyncHandlerWrapper } from '../../../lib/apiHelpers';
-import { cache3BoxProfileHandler } from './cache3BoxProfile/handler';
-import { refresh3BoxCacheHandler } from './refresh3BoxCache/handler';
-
-export const cacheRoutes = express.Router();
-
-cacheRoutes.post('/update', asyncHandlerWrapper(cache3BoxProfileHandler));
-
-cacheRoutes.post('/updateAll', asyncHandlerWrapper(refresh3BoxCacheHandler));

packages/backend/src/handlers/actions/idxCache/routes.ts

@@ -0,0 +1,17 @@
+import express from 'express';
+
+import { asyncHandlerWrapper } from '../../../lib/apiHelpers';
+import updateExpiredProfilesHandler from './updateExpiredProfiles/handler';
+import updateSingleProfileHandler from './updateSingleProfile/handler';
+
+export const cacheRoutes = express.Router();
+
+cacheRoutes.post(
+  '/updateSingle',
+  asyncHandlerWrapper(updateSingleProfileHandler),
+);
+
+cacheRoutes.post(
+  '/checkExpired',
+  asyncHandlerWrapper(updateExpiredProfilesHandler),
+);

packages/backend/src/handlers/actions/idxCache/updateExpiredProfiles/handler.ts

@@ -0,0 +1,34 @@
+import { Request, Response } from 'express';
+
+import { client } from '../../../../lib/hasuraClient';
+import updateCachedProfile from '../updateSingle';
+
+const INVALIDATE_AFTER_DAYS = 4; // number of days after which to recache
+
+// eslint-disable-next-line import/no-default-export
+export default async (req: Request, res: Response): Promise<void> => {
+  const expiration = new Date();
+  expiration.setDate(expiration.getDate() - INVALIDATE_AFTER_DAYS);
+  const { profile_cache: players } = await client.GetCacheEntries({
+    updatedBefore: expiration,
+  });
+  const idsToProcess: string[] = [];
+  await Promise.all(
+    players.map(async ({ playerId }) => {
+      if (!req.app.locals.queuedRecacheFor[playerId]) {
+        req.app.locals.queuedRecacheFor[playerId] = true;
+        idsToProcess.push(playerId);
+        req.app.locals.limiter.schedule(() =>
+          (async () => {
+            try {
+              await updateCachedProfile(playerId);
+            } finally {
+              req.app.locals.queuedRecacheFor[playerId] = false;
+            }
+          })(),
+        );
+      }
+    }),
+  );
+  res.json({ ids: idsToProcess });
+};

packages/backend/src/handlers/actions/idxCache/updateSingle.ts

@@ -4,16 +4,13 @@ import { getLegacy3BoxProfileAsBasicProfile, IDX } from '@ceramicstudio/idx';
 import type { BasicProfile } from '@ceramicstudio/idx-constants';
 import Box from '3box';
 
-import { CONFIG } from '../../../../config';
+import { CONFIG } from '../../../config';
 import {
   AccountType_Enum,
   UpdateBoxProfileResponse,
-} from '../../../../lib/autogen/hasura-sdk';
-import { client } from '../../../../lib/hasuraClient';
-import {
-  optimizeImage,
-  OptimizeImageParams,
-} from '../../../../lib/imageHelpers';
+} from '../../../lib/autogen/hasura-sdk';
+import { client } from '../../../lib/hasuraClient';
+import { optimizeImage, OptimizeImageParams } from '../../../lib/imageHelpers';
 
 function getImage(image: string | null | undefined, opts: OptimizeImageParams) {
   const [, imageHash] = image?.match(/^ipfs:\/\/(.+)$/) ?? [];
@@ -27,12 +24,10 @@ function getImage(image: string | null | undefined, opts: OptimizeImageParams) {
 const ceramic = new CeramicClient(CONFIG.ceramicDaemonURL);
 const idx = new IDX({ ceramic });
 
-export async function updateCachedProfile(
-  playerId: string,
-): Promise<UpdateBoxProfileResponse> {
+// eslint-disable-next-line import/no-default-export
+export default async (playerId: string): Promise<UpdateBoxProfileResponse> => {
   const updatedProfiles: string[] = [];
   const { player_by_pk: player } = await client.GetPlayer({ playerId });
-
   const ethAddress = player?.ethereum_address;
 
   if (!ethAddress) {
@@ -57,44 +52,40 @@ export async function updateCachedProfile(
 
   if (!idxProfile) {
     console.info(`No Profile For: ${ethAddress}`);
-  } else {
-    const {
-      name,
-      description,
-      emoji,
-      gender,
-      url,
-      homeLocation,
-      residenceCountry: country,
-    } = idxProfile;
-    let location = homeLocation;
-    if (country && country.length > 0) {
-      if (location && location.length > 0) {
-        location += `, ${country}`;
-      } else {
-        location = country;
-      }
-    }
-    const values = {
-      playerId,
-      name,
-      description,
-      emoji,
-      imageURL: getImage(idxProfile.image?.original?.src, {
-        ar: '1:1',
-        height: 200,
-      }),
-      backgroundImageURL: getImage(idxProfile.background?.original?.src, {
-        height: 300,
-      }),
-      gender,
-      location,
-      website: url,
-    };
-
-    await client.UpsertProfileCache({ objects: [values] });
+    idxProfile = {}; // create an empty placeholder row
   }
 
+  const {
+    name,
+    description,
+    emoji,
+    gender,
+    url,
+    homeLocation: location,
+    residenceCountry: country,
+    image,
+    background,
+  } = idxProfile;
+  const values = {
+    playerId,
+    name,
+    description,
+    emoji,
+    imageURL: getImage(image?.original?.src, {
+      ar: '1:1',
+      height: 200,
+    }),
+    backgroundImageURL: getImage(background?.original?.src, {
+      height: 300,
+    }),
+    gender,
+    location,
+    country,
+    website: url,
+  };
+
+  await client.UpsertProfileCache({ objects: [values] });
+
   // There isn't yet an interface for linking accounts on self.id
   const boxProfile = await Box.getProfile(ethAddress);
   const verifiedAccounts = await Box.getVerifiedAccounts(boxProfile);
@@ -143,4 +134,4 @@ export async function updateCachedProfile(
     success: true,
     updatedProfiles,
   };
-}
+};

packages/backend/src/handlers/actions/idxCache/updateSingleProfile/handler.ts

@@ -0,0 +1,42 @@
+import { Request, Response } from 'express';
+
+import updateCachedProfile from '../updateSingle';
+
+// eslint-disable-next-line import/no-default-export
+export default async (req: Request, res: Response): Promise<void> => {
+  const session = req.body.session_variables;
+  const role = session['x-hasura-role'];
+  const playerId = req.body.input?.playerId;
+
+  if (!['admin', 'player'].includes(role)) {
+    res.json({
+      success: false,
+      error: `Expected Role: admin or player. Got "${role}".`,
+    });
+    return;
+  }
+
+  if (!playerId) {
+    res.json({
+      success: false,
+      error: 'No playerId specified to update.',
+    });
+    return;
+  }
+
+  if (!req.app.locals.queuedRecacheFor[playerId]) {
+    req.app.locals.queuedRecacheFor[playerId] = true;
+    req.app.locals.limiter.schedule(() =>
+      (async () => {
+        try {
+          await updateCachedProfile(playerId);
+        } finally {
+          req.app.locals.queuedRecacheFor[playerId] = false;
+        }
+      })(),
+    );
+    res.json({ success: true });
+  } else {
+    res.json({ success: false, error: 'Already queued to be refreshed.' });
+  }
+};

packages/backend/src/handlers/actions/migrateSourceCredAccounts/handler.ts

@@ -91,7 +91,6 @@ export const migrateSourceCredAccounts = async (
         .reduce((t, c) => t + c, 0);
       return {
         ethereum_address: ethAddress.toLowerCase(),
-        scIdentityId: a.account.identity.id,
         totalXp: a.totalCred,
         seasonXp,
         rank,

packages/backend/src/handlers/actions/routes.ts

@@ -1,14 +1,14 @@
 import express from 'express';
 
 import { asyncHandlerWrapper } from '../../lib/apiHelpers';
-import { cacheRoutes } from './3Box/routes';
 import { guildRoutes } from './guild/routes';
+import { cacheRoutes } from './idxCache/routes';
 import { migrateSourceCredAccounts } from './migrateSourceCredAccounts/handler';
 import { questsRoutes } from './quests/routes';
 
 export const actionRoutes = express.Router();
 
-actionRoutes.use('/cache', cacheRoutes);
+actionRoutes.use('/idxCache', cacheRoutes);
 
 actionRoutes.post(
   '/migrateSourceCredAccounts',

packages/backend/src/handlers/graphql/mutations.ts

@@ -34,7 +34,17 @@ export const UpsertProfileCache = gql`
     $objects: [profile_cache_insert_input!]!
     $onConflict: profile_cache_on_conflict = {
       constraint: profile_cache_player_id_key
-      update_columns: []
+      update_columns: [
+        name
+        description
+        emoji
+        imageURL
+        backgroundImageURL
+        gender
+        location
+        country
+        website
+      ]
     }
   ) {
     insert_profile_cache(on_conflict: $onConflict, objects: $objects) {

packages/backend/src/handlers/graphql/queries.ts

@@ -119,3 +119,18 @@ export const GetDiscordGuild = gql`
   }
   ${GuildFragment}
 `;
+
+export const GetCacheEntries = gql`
+  query GetCacheEntries($updatedBefore: timestamptz!) {
+    profile_cache(
+      where: {
+        _or: [
+          { last_checked_at: { _lt: $updatedBefore } }
+          { last_checked_at: { _is_null: true } }
+        ]
+      }
+    ) {
+      playerId
+    }
+  }
+`;

packages/backend/src/handlers/triggers/cache3BoxProfile.ts

@@ -1,13 +0,0 @@
-import { Player } from '../../lib/autogen/hasura-sdk';
-import { updateCachedProfile } from '../actions/3Box/cache3BoxProfile/updateCachedProfile';
-import { TriggerPayload } from './types';
-
-export const cache3BoxProfile = async (payload: TriggerPayload<Player>) => {
-  const address = payload.event.data.new?.ethereum_address;
-
-  if (!address) return;
-
-  const playerId = payload.event.data.new?.id;
-
-  await updateCachedProfile(playerId);
-};

packages/backend/src/handlers/triggers/cacheIDXProfile.ts

@@ -0,0 +1,16 @@
+import { Player } from '../../lib/autogen/hasura-sdk';
+import updateCachedProfile from '../actions/idxCache/updateSingle';
+import { TriggerPayload } from './types';
+
+// This trigger is called when new accounts are created.
+// It skips the update queue associated with the normal
+// cache invalidation process.
+export const cacheIDXProfile = async (payload: TriggerPayload<Player>) => {
+  const address = payload.event.data.new?.ethereum_address;
+
+  if (!address) return;
+
+  const playerId = payload.event.data.new?.id;
+
+  await updateCachedProfile(playerId);
+};

packages/backend/src/handlers/triggers/handler.ts

@@ -2,12 +2,12 @@ import { Request, Response } from 'express';
 import { ParamsDictionary } from 'express-serve-static-core';
 
 import { Player } from '../../lib/autogen/hasura-sdk';
-import { cache3BoxProfile } from './cache3BoxProfile';
+import { cacheIDXProfile } from './cacheIDXProfile';
 import { TriggerPayload } from './types';
 import { updateDiscordRole } from './updateDiscordRole';
 
 const TRIGGERS = {
-  cache3BoxProfile,
+  cacheIDXProfile,
   player_rank_updated: updateDiscordRole,
 };
 

packages/backend/src/index.ts

@@ -1,4 +1,5 @@
 import bodyParser from 'body-parser';
+import Bottleneck from 'bottleneck';
 import express from 'express';
 
 import { CONFIG } from './config';
@@ -7,6 +8,12 @@ import { errorMiddleware } from './lib/apiHelpers';
 
 const app = express();
 
+app.locals.limiter = new Bottleneck({
+  maxConcurrent: 10,
+});
+// tracks the current contents of Bottleneck
+app.locals.queuedRecacheFor = {};
+
 app.use(bodyParser.json());
 
 app.use(router);