feat(ui): add migration path for client state from IndexedDB to server-backed storage

invokeai/frontend/web/package.json

@@ -63,6 +63,7 @@
     "framer-motion": "^11.10.0",
     "i18next": "^25.3.2",
     "i18next-http-backend": "^3.0.2",
+    "idb-keyval": "6.2.1",
     "jsondiffpatch": "^0.7.3",
     "konva": "^9.3.22",
     "linkify-react": "^4.3.1",

invokeai/frontend/web/pnpm-lock.yaml

@@ -80,6 +80,9 @@ importers:
       i18next-http-backend:
         specifier: ^3.0.2
         version: 3.0.2
+      idb-keyval:
+        specifier: 6.2.1
+        version: 6.2.1
       jsondiffpatch:
         specifier: ^0.7.3
         version: 0.7.3
@@ -2775,6 +2778,9 @@ packages:
       typescript:
         optional: true
 
+  idb-keyval@6.2.1:
+    resolution: {integrity: sha512-8Sb3veuYCyrZL+VBt9LJfZjLUPWVvqn8tG28VqYNFCo43KHcKuq+b4EiXGeuaLAQWL2YmyDgMp2aSpH9JHsEQg==}
+
   ieee754@1.2.1:
     resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
 
@@ -7269,6 +7275,8 @@ snapshots:
     optionalDependencies:
       typescript: 5.8.3
 
+  idb-keyval@6.2.1: {}
+
   ieee754@1.2.1: {}
 
   ignore@5.3.2: {}

invokeai/frontend/web/src/app/store/enhancers/reduxRemember/driver.ts

@@ -3,8 +3,12 @@ import { StorageError } from 'app/store/enhancers/reduxRemember/errors';
 import { $authToken } from 'app/store/nanostores/authToken';
 import { $projectId } from 'app/store/nanostores/projectId';
 import { $queueId } from 'app/store/nanostores/queueId';
+import type { UseStore } from 'idb-keyval';
+import { createStore as idbCreateStore, del as idbDel, get as idbGet } from 'idb-keyval';
 import type { Driver } from 'redux-remember';
+import { serializeError } from 'serialize-error';
 import { buildV1Url, getBaseUrl } from 'services/api';
+import type { JsonObject } from 'type-fest';
 
 const log = logger('system');
 
@@ -52,68 +56,124 @@ let persistRefCount = 0;
 // This logic is unknown to `redux-remember`. When an omitted field changes, it will still attempt to persist the
 // whole slice, even if the final, _serialized_ slice value is unchanged.
 //
-// To avoid unnecessary network requests, we keep track of the last persisted state for each key. If the value to
-// be persisted is the same as the last persisted value, we can skip the network request.
+// To avoid unnecessary network requests, we keep track of the last persisted state for each key in this map.
+// If the value to be persisted is the same as the last persisted value, we will skip the network request.
 const lastPersistedState = new Map<string, string | undefined>();
 
-export const reduxRememberDriver: Driver = {
-  getItem: async (key: string) => {
-    try {
-      const url = getUrl('get_by_key', key);
-      const headers = getHeaders();
-      const res = await fetch(url, { method: 'GET', headers });
-      if (!res.ok) {
-        throw new Error(`Response status: ${res.status}`);
-      }
-      const value = await res.json();
-      lastPersistedState.set(key, value);
-      log.trace({ key, last: lastPersistedState.get(key), next: value }, `Getting state for ${key}`);
-      return value;
-    } catch (originalError) {
-      throw new StorageError({
-        key,
-        projectId: $projectId.get(),
-        originalError,
-      });
-    }
-  },
-  setItem: async (key: string, value: string) => {
-    try {
-      persistRefCount++;
-      if (lastPersistedState.get(key) === value) {
-        log.trace(
-          { key, last: lastPersistedState.get(key), next: value },
-          `Skipping persist for ${key} as value is unchanged`
-        );
-        return value;
-      }
-      log.trace({ key, last: lastPersistedState.get(key), next: value }, `Persisting state for ${key}`);
-      const url = getUrl('set_by_key', key);
-      const headers = getHeaders();
-      const res = await fetch(url, { method: 'POST', headers, body: value });
-      if (!res.ok) {
-        throw new Error(`Response status: ${res.status}`);
-      }
-      const resultValue = await res.json();
-      lastPersistedState.set(key, resultValue);
-      return resultValue;
-    } catch (originalError) {
-      throw new StorageError({
-        key,
-        value,
-        projectId: $projectId.get(),
-        originalError,
-      });
-    } finally {
-      persistRefCount--;
-      if (persistRefCount < 0) {
-        log.trace('Persist ref count is negative, resetting to 0');
-        persistRefCount = 0;
-      }
-    }
-  },
+// As of v6.3.0, we use server-backed storage for client state. This replaces the previous IndexedDB-based storage,
+// which was implemented using `idb-keyval`.
+//
+// To facilitate a smooth transition, we implement a migration strategy that attempts to retrieve values from IndexedDB
+// and persist them to the new server-backed storage. This is done on a best-effort basis.
+
+// These constants were used in the previous IndexedDB-based storage implementation.
+const IDB_DB_NAME = 'invoke';
+const IDB_STORE_NAME = 'invoke-store';
+const IDB_STORAGE_PREFIX = '@@invokeai-';
+
+// Lazy store creation
+let _idbKeyValStore: UseStore | null = null;
+const getIdbKeyValStore = () => {
+  if (_idbKeyValStore === null) {
+    _idbKeyValStore = idbCreateStore(IDB_DB_NAME, IDB_STORE_NAME);
+  }
+  return _idbKeyValStore;
 };
 
+const getIdbKey = (key: string) => {
+  return `${IDB_STORAGE_PREFIX}${key}`;
+};
+
+const getItem = async (key: string) => {
+  try {
+    const url = getUrl('get_by_key', key);
+    const headers = getHeaders();
+    const res = await fetch(url, { method: 'GET', headers });
+    if (!res.ok) {
+      throw new Error(`Response status: ${res.status}`);
+    }
+    const value = await res.json();
+
+    // Best-effort migration from IndexedDB to the new storage system
+    log.trace({ key, value }, 'Server-backed storage value retrieved');
+
+    if (!value) {
+      const idbKey = getIdbKey(key);
+      try {
+        // It's a bit tricky to query IndexedDB directly to check if value exists, so we use `idb-keyval` to do it.
+        // Thing is, `idb-keyval` requires you to create a store to query it. End result - we are creating a store
+        // even if we don't use it for anything besides checking if the key is present.
+        const idbKeyValStore = getIdbKeyValStore();
+        const idbValue = await idbGet(idbKey, idbKeyValStore);
+        if (idbValue) {
+          log.debug(
+            { key, idbKey, idbValue },
+            'No value in server-backed storage, but found value in IndexedDB - attempting migration'
+          );
+          await idbDel(idbKey, idbKeyValStore);
+          await setItem(key, idbValue);
+          log.debug({ key, idbKey, idbValue }, 'Migration successful');
+          return idbValue;
+        }
+      } catch (error) {
+        // Just log if IndexedDB retrieval fails - this is a best-effort migration.
+        log.debug(
+          { key, idbKey, error: serializeError(error) } as JsonObject,
+          'Error checking for or migrating from IndexedDB'
+        );
+      }
+    }
+
+    lastPersistedState.set(key, value);
+    log.trace({ key, last: lastPersistedState.get(key), next: value }, `Getting state for ${key}`);
+    return value;
+  } catch (originalError) {
+    throw new StorageError({
+      key,
+      projectId: $projectId.get(),
+      originalError,
+    });
+  }
+};
+
+const setItem = async (key: string, value: string) => {
+  try {
+    persistRefCount++;
+    if (lastPersistedState.get(key) === value) {
+      log.trace(
+        { key, last: lastPersistedState.get(key), next: value },
+        `Skipping persist for ${key} as value is unchanged`
+      );
+      return value;
+    }
+    log.trace({ key, last: lastPersistedState.get(key), next: value }, `Persisting state for ${key}`);
+    const url = getUrl('set_by_key', key);
+    const headers = getHeaders();
+    const res = await fetch(url, { method: 'POST', headers, body: value });
+    if (!res.ok) {
+      throw new Error(`Response status: ${res.status}`);
+    }
+    const resultValue = await res.json();
+    lastPersistedState.set(key, resultValue);
+    return resultValue;
+  } catch (originalError) {
+    throw new StorageError({
+      key,
+      value,
+      projectId: $projectId.get(),
+      originalError,
+    });
+  } finally {
+    persistRefCount--;
+    if (persistRefCount < 0) {
+      log.trace('Persist ref count is negative, resetting to 0');
+      persistRefCount = 0;
+    }
+  }
+};
+
+export const reduxRememberDriver: Driver = { getItem, setItem };
+
 export const clearStorage = async () => {
   try {
     persistRefCount++;