From 20b4737dbff1352f7d204e4fd9f4722d2b66b24e Mon Sep 17 00:00:00 2001 From: Evi Nova Date: Sat, 10 Jan 2026 14:15:21 +1000 Subject: [PATCH] refactor: use google cloud bucket upload instead of SSH in --- common/package.json | 1 + common/scripts/ofac/Dockerfile | 2 +- .../scripts/ofac/OFAC_AUTO_UPDATER_README.md | 105 +++++--- common/scripts/ofac/gcsUpload.ts | 227 ++++++++++++++++ common/scripts/ofac/index.ts | 2 - common/scripts/ofac/runOfacAutoUpdate.ts | 246 ++++++----------- common/scripts/ofac/test-ofac-update.sh | 227 ++++++++++++++++ test-dry-run.sh | 46 ++++ yarn.lock | 254 +++++++++++++++++- 9 files changed, 895 insertions(+), 215 deletions(-) create mode 100644 common/scripts/ofac/gcsUpload.ts create mode 100755 common/scripts/ofac/test-ofac-update.sh create mode 100755 test-dry-run.sh diff --git a/common/package.json b/common/package.json index b18520853..37fd488f9 100644 --- a/common/package.json +++ b/common/package.json @@ -661,6 +661,7 @@ }, "dependencies": { "@anon-aadhaar/core": "npm:@selfxyz/anon-aadhaar-core@^0.0.1", + "@google-cloud/storage": "^7.18.0", "@noble/hashes": "^1.5.0", "@openpassport/zk-kit-imt": "^0.0.5", "@openpassport/zk-kit-lean-imt": "^0.0.6", diff --git a/common/scripts/ofac/Dockerfile b/common/scripts/ofac/Dockerfile index b06dc77b9..27be592cc 100644 --- a/common/scripts/ofac/Dockerfile +++ b/common/scripts/ofac/Dockerfile @@ -3,7 +3,7 @@ FROM node:22-slim WORKDIR /app RUN apt-get update \ - && apt-get install -y --no-install-recommends git openssh-client ca-certificates \ + && apt-get install -y --no-install-recommends git ca-certificates \ && rm -rf /var/lib/apt/lists/* RUN corepack enable && corepack prepare yarn@stable --activate diff --git a/common/scripts/ofac/OFAC_AUTO_UPDATER_README.md b/common/scripts/ofac/OFAC_AUTO_UPDATER_README.md index afafbeeaf..2c3eb6df4 100644 --- a/common/scripts/ofac/OFAC_AUTO_UPDATER_README.md +++ b/common/scripts/ofac/OFAC_AUTO_UPDATER_README.md @@ -1,35 +1,23 @@ # OFAC Sanctions List Automation -Automated pipeline for updating OFAC sanctions list with ~6-7 second mismatch window. +Automated pipeline for updating OFAC sanctions list with ~200-500ms mismatch window using Google Cloud Storage. ## Prerequisites -### SSH Access +### Google Cloud Storage Access -Add to `~/.ssh/config`: - -``` -Host self-infra-prod - HostName - User ec2-user - IdentityFile ~/.ssh/infra.pem - -Host self-infra-staging - HostName 54.71.62.30 - User ec2-user - IdentityFile ~/.ssh/infra.pem -``` - -### VPN - -Connect to NordLayer VPN before running any commands. +1. Create a GCS bucket for OFAC data (e.g., `self-ofac-prod`, `self-ofac-staging`) +2. Enable versioning on the bucket for rollback capability +3. Create a service account with `roles/storage.objectAdmin` permission +4. Download the service account key JSON file +5. Set `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the key file path --- ## Single-Shot Auto Update (Docker/TEE) This is the unified flow that runs the pipeline, updates on-chain roots directly, -and performs the prestaged upload + atomic move in one run. +and uploads files to Google Cloud Storage with atomic pointer updates. ### Docker @@ -39,29 +27,30 @@ Build the image: docker build -f common/scripts/ofac/Dockerfile -t ofac-auto-updater . ``` -Run with a single mount (all inputs/outputs live under `/data/ofac`): +Run with GCS credentials: ```bash docker run --rm \\ -e PRIVATE_KEY=0x... \\ -e NETWORK=celo \\ - -e SSH_HOST=self-infra-prod \\ - -e UPLOAD_PATH=/home/ec2-user/self-infra/merkle-tree-reader/common/constants/ofac \\ + -e GCS_BUCKET_NAME=self-ofac-prod \\ + -e GOOGLE_APPLICATION_CREDENTIALS=/secrets/gcs-key.json \\ -v /local/ofac:/data \\ - -v ~/.ssh:/root/.ssh:ro \\ + -v /local/gcs-key.json:/secrets/gcs-key.json:ro \\ ofac-auto-updater ``` Environment variables: - `PRIVATE_KEY` (required): signer key used for on-chain updates - Signer must have `TEE_ROLE` on the registry contracts -- `NETWORK`: `celo`, `celo-sepolia`, or `sepolia` -- `RPC_URL` or network-specific RPC envs (`CELO_RPC_URL`, `CELO_SEPOLIA_RPC_URL`, `SEPOLIA_RPC_URL`) +- `NETWORK`: `celo` or `celo-sepolia` +- `RPC_URL` or network-specific RPC envs (`CELO_RPC_URL`, `CELO_SEPOLIA_RPC_URL`) - `OFAC_DATA_DIR` (default: `/data/ofac`) -- `SSH_HOST` (default: `self-infra-staging`) -- `UPLOAD_PATH` (default: production path for the chosen network) +- `GCS_BUCKET_NAME` (default: `self-ofac-prod` for celo, `self-ofac-staging` for celo-sepolia) +- `GCS_BASE_PATH` (default: `ofac`) +- `GOOGLE_APPLICATION_CREDENTIALS` (required): path to GCS service account key JSON - `DRY_RUN=true` to skip on-chain update and upload -- `SKIP_PRESTAGE=true` to skip pre-staging (not recommended) +- `SKIP_UPLOAD=true` to skip GCS upload (not recommended) If deploying this change to existing registries, call `initializeTeeRole(TEE_ADDRESS)` after upgrade to set role admin and grant `TEE_ROLE`. @@ -70,17 +59,59 @@ If deploying this change to existing registries, call `initializeTeeRole(TEE_ADD ```bash PRIVATE_KEY=0x... \\ NETWORK=celo \\ -SSH_HOST=self-infra-prod \\ +GCS_BUCKET_NAME=self-ofac-prod \\ +GOOGLE_APPLICATION_CREDENTIALS=/path/to/gcs-key.json \\ yarn tsx common/scripts/ofac/runOfacAutoUpdate.ts ``` --- -## How SSH Is Used +## How GCS Is Used -The updater uses SSH only for the file staging + atomic move: -1. Pre-stage generated tree files to a temp directory on the server. -2. After on-chain updates complete, atomically move the files into production. -3. Optionally verify the production directory contents. +The updater uses Google Cloud Storage for atomic file deployment: -If SSH isn’t available (e.g., missing VPN/host config), the on-chain updates can still run, -but the tree deployment step will fail unless `DRY_RUN=true`. +1. **Upload Phase**: Upload all tree files to a versioned path (e.g., `ofac/2026-01-09-1736437890/`) +2. **On-Chain Update**: Submit transactions to update OFAC roots on smart contracts +3. **Atomic Switch**: Update `current.json` pointer file to reference the new version path + +### Mismatch Window + +- **Old (SSH)**: 6-7 seconds during file move operation +- **New (GCS)**: 200-500ms during `current.json` upload +- **Consistency**: Readers always see a complete snapshot (all files from the same version) + +### File Structure + +``` +gs://self-ofac-prod/ + ofac/ + current.json # Pointer to active version + 2026-01-09-1736437890/ # Versioned directory + passportNoAndNationalitySMT.json + nameAndDobSMT.json + nameAndYobSMT.json + nameAndDobSMT_ID.json + nameAndYobSMT_ID.json + nameAndDobSMT_AADHAAR.json + nameAndYobSMT_AADHAAR.json + roots.json + latest-roots.json +``` + +### current.json Format + +```json +{ + "timestamp": "2026-01-09T12:34:56.789Z", + "path": "ofac/2026-01-09-1736437890", + "roots": { + "passport_no_and_nationality": "12345...", + "name_and_dob": "67890...", + "name_and_yob": "11111..." + } +} +``` + +Readers should: +1. Fetch `gs://bucket/ofac/current.json` +2. Parse the `path` field +3. Fetch tree files from `gs://bucket/{path}/` diff --git a/common/scripts/ofac/gcsUpload.ts b/common/scripts/ofac/gcsUpload.ts new file mode 100644 index 000000000..3c4d4fd8f --- /dev/null +++ b/common/scripts/ofac/gcsUpload.ts @@ -0,0 +1,227 @@ +/** + * Google Cloud Storage Upload Module + * + * Handles uploading OFAC tree files to GCS with versioned paths + * and atomic pointer file updates for minimal mismatch window. + */ + +import { Storage } from '@google-cloud/storage'; +import * as fs from 'fs'; +import * as path from 'path'; + +export interface GcsUploadOptions { + bucketName: string; + basePath: string; + treesDir: string; + roots: Record; + timestamp: number; + dryRun?: boolean; +} + +export interface GcsUploadResult { + success: boolean; + versionPath?: string; + filesUploaded?: number; + error?: string; +} + +export interface PointerUpdateResult { + success: boolean; + durationMs: number; + error?: string; +} + +const TREE_FILES = [ + 'passportNoAndNationalitySMT.json', + 'nameAndDobSMT.json', + 'nameAndYobSMT.json', + 'nameAndDobSMT_ID.json', + 'nameAndYobSMT_ID.json', + 'nameAndDobSMT_AADHAAR.json', + 'nameAndYobSMT_AADHAAR.json', + 'roots.json', + 'latest-roots.json', +]; + +function log(msg: string) { + const timestamp = new Date().toISOString().slice(11, 23); + console.log(`[${timestamp}] ${msg}`); +} + +/** + * Initialize GCS client with authentication + */ +function getStorageClient(): Storage { + const credentialsPath = process.env.GOOGLE_APPLICATION_CREDENTIALS; + + if (credentialsPath && !fs.existsSync(credentialsPath)) { + throw new Error(`GCS credentials file not found: ${credentialsPath}`); + } + + return new Storage(); +} + +/** + * Upload all tree files to a versioned path in GCS + */ +export async function uploadToGcs(options: GcsUploadOptions): Promise { + const { bucketName, basePath, treesDir, timestamp, dryRun = false } = options; + + const versionPath = `${basePath}/${new Date(timestamp).toISOString().split('T')[0]}-${timestamp}`; + + log(`Uploading to gs://${bucketName}/${versionPath}`); + + const filesToUpload = TREE_FILES + .map((f) => path.join(treesDir, f)) + .filter((f) => fs.existsSync(f)); + + if (filesToUpload.length === 0) { + return { + success: false, + error: 'No tree files found to upload', + }; + } + + log(` Found ${filesToUpload.length} files to upload`); + + if (dryRun) { + log(' [DRY RUN] Would upload:'); + filesToUpload.forEach((f) => log(` - ${path.basename(f)}`)); + return { + success: true, + versionPath, + filesUploaded: filesToUpload.length, + }; + } + + try { + const storage = getStorageClient(); + const bucket = storage.bucket(bucketName); + + for (const filePath of filesToUpload) { + const fileName = path.basename(filePath); + const destination = `${versionPath}/${fileName}`; + + process.stdout.write(` Uploading ${fileName}...`); + + await bucket.upload(filePath, { + destination, + metadata: { + cacheControl: 'public, max-age=300', + metadata: { + uploadedAt: new Date().toISOString(), + }, + }, + }); + + console.log(' ok'); + } + + log(`Successfully uploaded ${filesToUpload.length} files to ${versionPath}`); + + return { + success: true, + versionPath, + filesUploaded: filesToUpload.length, + }; + } catch (error) { + return { + success: false, + error: `GCS upload failed: ${(error as Error).message}`, + }; + } +} + +/** + * Update the current.json pointer file to point to the new version + * This is the atomic switch that minimizes mismatch window + */ +export async function updatePointerFile( + bucketName: string, + basePath: string, + versionPath: string, + roots: Record, + dryRun: boolean = false +): Promise { + log(`Updating pointer file: gs://${bucketName}/${basePath}/current.json`); + + if (dryRun) { + log(' [DRY RUN] Would update pointer to: ' + versionPath); + return { success: true, durationMs: 0 }; + } + + const startTime = Date.now(); + + try { + const storage = getStorageClient(); + const bucket = storage.bucket(bucketName); + + const pointerData = { + timestamp: new Date().toISOString(), + path: versionPath, + roots, + }; + + const pointerFile = bucket.file(`${basePath}/current.json`); + + await pointerFile.save(JSON.stringify(pointerData, null, 2), { + metadata: { + contentType: 'application/json', + cacheControl: 'no-cache, no-store, must-revalidate', + }, + }); + + const durationMs = Date.now() - startTime; + log(`Pointer updated in ${durationMs}ms`); + + return { success: true, durationMs }; + } catch (error) { + const durationMs = Date.now() - startTime; + return { + success: false, + durationMs, + error: `Pointer update failed: ${(error as Error).message}`, + }; + } +} + +/** + * Verify that all files were uploaded successfully + */ +export async function verifyGcsFiles( + bucketName: string, + versionPath: string, + dryRun: boolean = false +): Promise { + log('Verifying uploaded files...'); + + if (dryRun) { + log(' [DRY RUN] Skipping verification'); + return; + } + + try { + const storage = getStorageClient(); + const bucket = storage.bucket(bucketName); + + const [files] = await bucket.getFiles({ + prefix: versionPath, + }); + + if (files.length === 0) { + log(' WARNING: No files found at version path'); + return; + } + + log(` Found ${files.length} files:`); + files.slice(0, 10).forEach((file) => { + log(` - ${file.name}`); + }); + + if (files.length > 10) { + log(` ... and ${files.length - 10} more`); + } + } catch (error) { + log(` Could not verify files: ${(error as Error).message}`); + } +} diff --git a/common/scripts/ofac/index.ts b/common/scripts/ofac/index.ts index 26c1ee4b8..402de3441 100644 --- a/common/scripts/ofac/index.ts +++ b/common/scripts/ofac/index.ts @@ -178,8 +178,6 @@ async function main() { } // Run if executed directly -import { fileURLToPath } from 'url'; - const isMainModule = process.argv[1] === fileURLToPath(import.meta.url); if (isMainModule) { main().catch((error) => { diff --git a/common/scripts/ofac/runOfacAutoUpdate.ts b/common/scripts/ofac/runOfacAutoUpdate.ts index 52656101b..e1c9e17ac 100644 --- a/common/scripts/ofac/runOfacAutoUpdate.ts +++ b/common/scripts/ofac/runOfacAutoUpdate.ts @@ -1,21 +1,21 @@ /** * OFAC Auto Updater (Single-Shot) * - * Pipeline + on-chain update + prestaged upload in one run: + * Pipeline + on-chain update + GCS upload in one run: * 1. Download + parse OFAC SDN list * 2. Build all OFAC Merkle trees - * 3. Pre-stage trees to server + * 3. Upload trees to versioned GCS path * 4. Update on-chain OFAC roots (direct signer, no multisig) - * 5. Atomically move trees into production + * 5. Atomically update GCS pointer file */ import { ethers } from 'ethers'; import * as fs from 'fs'; import * as path from 'path'; -import { execSync } from 'child_process'; import { fileURLToPath } from 'url'; import { runOfacPipeline } from './index.js'; +import { uploadToGcs, updatePointerFile, verifyGcsFiles } from './gcsUpload.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -24,13 +24,11 @@ const __dirname = path.dirname(__filename); const DEFAULT_RPC_URLS: Record = { celo: 'https://forno.celo.org', 'celo-sepolia': 'https://celo-sepolia.drpc.org', - sepolia: 'https://rpc.sepolia.org', }; -const DEFAULT_UPLOAD_PATHS: Record = { - celo: '/home/ec2-user/self-infra/merkle-tree-reader/common/constants/ofac', - 'celo-sepolia': '/home/ec2-user/self-infra-staging/merkle-tree-reader/common/constants/ofac', - sepolia: '/home/ec2-user/ofac-e2e-test', +const DEFAULT_GCS_BUCKETS: Record = { + celo: 'self-ofac-prod', + 'celo-sepolia': 'self-ofac-staging', }; // Hardcoded registry addresses (Celo Mainnet) @@ -44,9 +42,7 @@ const CELO_REGISTRY_ADDRESSES: Record = { interface RegistryConfig { name: string; registryKey: string; - hasPassportNo: boolean; - hasNameAndDob: boolean; - hasNameAndYob: boolean; + rootTypes: Array<'passportNo' | 'nameAndDob' | 'nameAndYob'>; rootTreePrefix: string; } @@ -54,25 +50,19 @@ const REGISTRY_CONFIGS: RegistryConfig[] = [ { name: 'Passport Registry', registryKey: 'IdentityRegistry', - hasPassportNo: true, - hasNameAndDob: true, - hasNameAndYob: true, + rootTypes: ['passportNo', 'nameAndDob', 'nameAndYob'], rootTreePrefix: '', }, { name: 'ID Card Registry', registryKey: 'IdentityRegistryIdCard', - hasPassportNo: false, - hasNameAndDob: true, - hasNameAndYob: true, + rootTypes: ['nameAndDob', 'nameAndYob'], rootTreePrefix: '_id_card', }, { name: 'Aadhaar Registry', registryKey: 'IdentityRegistryAadhaar', - hasPassportNo: false, - hasNameAndDob: true, - hasNameAndYob: true, + rootTypes: ['nameAndDob', 'nameAndYob'], rootTreePrefix: '_aadhaar', }, ]; @@ -87,18 +77,6 @@ const REGISTRY_ABI = [ 'function updateNameAndYobOfacRoot(uint256 root)', ]; -// Tree files to upload -const TREE_FILES = [ - 'passportNoAndNationalitySMT.json', - 'nameAndDobSMT.json', - 'nameAndYobSMT.json', - 'nameAndDobSMT_ID.json', - 'nameAndYobSMT_ID.json', - 'nameAndDobSMT_AADHAAR.json', - 'nameAndYobSMT_AADHAAR.json', - 'roots.json', - 'latest-roots.json', -]; function log(msg: string) { const timestamp = new Date().toISOString().slice(11, 23); @@ -111,8 +89,6 @@ function getRpcUrl(network: string): string | undefined { return process.env.CELO_RPC_URL || DEFAULT_RPC_URLS.celo; case 'celo-sepolia': return process.env.CELO_SEPOLIA_RPC_URL || DEFAULT_RPC_URLS['celo-sepolia']; - case 'sepolia': - return process.env.SEPOLIA_RPC_URL || DEFAULT_RPC_URLS.sepolia; default: return process.env.RPC_URL; } @@ -187,7 +163,7 @@ async function updateRegistryRoots( const contract = new ethers.Contract(registryAddress, REGISTRY_ABI, signer); let updates = 0; - async function maybeUpdate( + async function updateRoot( rootType: 'passportNo' | 'nameAndDob' | 'nameAndYob', updateFn: keyof ethers.Contract ) { @@ -196,7 +172,7 @@ async function updateRegistryRoots( const oldRoot = await getCurrentRoot(contract, rootType); if (oldRoot === newRoot) { - log(`No change for ${config.name} ${rootType}`); + log(`Skipping ${config.name} ${rootType}: on-chain root already matches`); return; } @@ -217,106 +193,19 @@ async function updateRegistryRoots( updates += 1; } - if (config.hasPassportNo) { - await maybeUpdate('passportNo', 'updatePassportNoOfacRoot'); - } - if (config.hasNameAndDob) { - await maybeUpdate('nameAndDob', 'updateNameAndDobOfacRoot'); - } - if (config.hasNameAndYob) { - await maybeUpdate('nameAndYob', 'updateNameAndYobOfacRoot'); + for (const rootType of config.rootTypes) { + if (rootType === 'passportNo') { + await updateRoot('passportNo', 'updatePassportNoOfacRoot'); + } else if (rootType === 'nameAndDob') { + await updateRoot('nameAndDob', 'updateNameAndDobOfacRoot'); + } else { + await updateRoot('nameAndYob', 'updateNameAndYobOfacRoot'); + } } return updates; } -function prestageFiles( - treesDir: string, - sshHost: string, - stagingPath: string, - dryRun: boolean -): boolean { - log(`PRE-STAGING: Uploading trees to ${sshHost}:${stagingPath}`); - - const filesToUpload = TREE_FILES - .map((f) => path.join(treesDir, f)) - .filter((f) => fs.existsSync(f)); - - if (filesToUpload.length === 0) { - log('ERROR: No tree files found to upload!'); - return false; - } - - log(` Found ${filesToUpload.length} files`); - - if (dryRun) { - log(' [DRY RUN] Would upload:'); - filesToUpload.forEach((f) => log(` - ${path.basename(f)}`)); - return true; - } - - try { - execSync(`ssh ${sshHost} "mkdir -p ${stagingPath}"`, { stdio: 'pipe' }); - - for (const file of filesToUpload) { - const basename = path.basename(file); - process.stdout.write(` Uploading ${basename}...`); - execSync(`scp "${file}" "${sshHost}:${stagingPath}/"`, { stdio: 'pipe' }); - console.log(' ok'); - } - - log(`Pre-staged ${filesToUpload.length} files`); - return true; - } catch (error) { - log(`ERROR: Pre-staging failed: ${error}`); - return false; - } -} - -function atomicMove( - sshHost: string, - stagingPath: string, - productionPath: string, - dryRun: boolean -): { success: boolean; durationMs: number } { - log(`ATOMIC MOVE: ${stagingPath} -> ${productionPath}`); - - if (dryRun) { - log(' [DRY RUN] Would move files'); - return { success: true, durationMs: 0 }; - } - - const startTime = Date.now(); - - try { - execSync(`ssh ${sshHost} "mkdir -p ${productionPath}"`, { stdio: 'pipe' }); - - const moveCmd = `ssh ${sshHost} "mv ${stagingPath}/*.json ${productionPath}/ && rm -rf ${stagingPath}"`; - execSync(moveCmd, { stdio: 'pipe' }); - - const durationMs = Date.now() - startTime; - log(`Atomic move completed in ${durationMs}ms`); - - return { success: true, durationMs }; - } catch (error) { - const durationMs = Date.now() - startTime; - log(`ERROR: Atomic move failed after ${durationMs}ms: ${error}`); - return { success: false, durationMs }; - } -} - -function verifyProduction(sshHost: string, productionPath: string): void { - log('Verifying production files...'); - try { - const result = execSync( - `ssh ${sshHost} "ls -la ${productionPath}/*.json 2>/dev/null | tail -10"`, - { encoding: 'utf-8' } - ); - console.log(result); - } catch { - log('Could not verify (may still be successful)'); - } -} async function main() { console.log(''); @@ -347,21 +236,19 @@ async function main() { const rootsPath = process.env.ROOTS_PATH || path.join(outputDir, 'latest-roots.json'); const treesDir = process.env.TREES_DIR || outputDir; - const sshHost = process.env.SSH_HOST || 'self-infra-staging'; - const productionPath = - process.env.UPLOAD_PATH || DEFAULT_UPLOAD_PATHS[network] || DEFAULT_UPLOAD_PATHS.celo; - const skipPrestage = process.env.SKIP_PRESTAGE === 'true'; + const bucketName = + process.env.GCS_BUCKET_NAME || DEFAULT_GCS_BUCKETS[network] || DEFAULT_GCS_BUCKETS.celo; + const basePath = process.env.GCS_BASE_PATH || 'ofac'; + const skipUpload = process.env.SKIP_UPLOAD === 'true'; const timestamp = Date.now(); - const stagingPath = process.env.STAGING_PATH || `/tmp/ofac-prestage-${timestamp}`; log(`Network: ${network}`); log(`RPC: ${rpcUrl}`); log(`Data dir: ${dataDir}`); log(`Trees dir: ${treesDir}`); - log(`SSH host: ${sshHost}`); - log(`Staging: ${stagingPath}`); - log(`Production: ${productionPath}`); + log(`GCS bucket: ${bucketName}`); + log(`GCS base path: ${basePath}`); log(`Dry Run: ${dryRun}`); console.log(''); @@ -378,24 +265,7 @@ async function main() { process.exit(1); } - // Step 4: Pre-stage files - console.log(''); - console.log('-'.repeat(70)); - console.log(' PHASE: PRE-STAGE FILES'); - console.log('-'.repeat(70)); - console.log(''); - - if (!skipPrestage) { - const prestageSuccess = prestageFiles(treesDir, sshHost, stagingPath, dryRun); - if (!prestageSuccess && !dryRun) { - console.error('ERROR: Pre-staging failed. Aborting before on-chain update.'); - process.exit(1); - } - } else { - log('Skipping pre-stage (SKIP_PRESTAGE=true)'); - } - - // Step 5: On-chain updates + // Step 4: On-chain updates console.log(''); console.log('-'.repeat(70)); console.log(' PHASE: ON-CHAIN UPDATES'); @@ -423,21 +293,65 @@ async function main() { } log(`Total updates submitted: ${totalUpdates}`); + if (totalUpdates === 0) { + log('No on-chain updates needed; skipping tree deployment.'); + return; + } - // Step 6: Atomic move to production + // Step 5: Upload to GCS (versioned path) console.log(''); console.log('-'.repeat(70)); - console.log(' PHASE: ATOMIC MOVE'); + console.log(' PHASE: UPLOAD TO GCS'); console.log('-'.repeat(70)); console.log(''); - const moveResult = atomicMove(sshHost, stagingPath, productionPath, dryRun); - if (moveResult.success) { - verifyProduction(sshHost, productionPath); - log(`Mismatch window: ${moveResult.durationMs}ms (~${(moveResult.durationMs / 1000).toFixed(1)}s)`); + if (skipUpload) { + log('Skipping upload (SKIP_UPLOAD=true)'); + return; + } + + const uploadResult = await uploadToGcs({ + bucketName, + basePath, + treesDir, + roots, + timestamp, + dryRun, + }); + + if (!uploadResult.success) { + console.error('ERROR: GCS upload failed:', uploadResult.error); + console.error('On-chain updates succeeded but file upload failed.'); + process.exit(1); + } + + log(`Uploaded ${uploadResult.filesUploaded} files to ${uploadResult.versionPath}`); + + // Step 6: Update pointer file (atomic switch) + console.log(''); + console.log('-'.repeat(70)); + console.log(' PHASE: ATOMIC POINTER UPDATE'); + console.log('-'.repeat(70)); + console.log(''); + + const pointerResult = await updatePointerFile( + bucketName, + basePath, + uploadResult.versionPath!, + roots, + dryRun + ); + + if (pointerResult.success) { + await verifyGcsFiles(bucketName, uploadResult.versionPath!, dryRun); + log( + `Mismatch window: ${pointerResult.durationMs}ms (~${(pointerResult.durationMs / 1000).toFixed(1)}s)` + ); } else { - console.error('WARNING: On-chain updates succeeded but move failed. Manual move required.'); - console.error(` ssh ${sshHost} "mv ${stagingPath}/*.json ${productionPath}/"`); + console.error('WARNING: On-chain updates succeeded but pointer update failed.'); + console.error(`Error: ${pointerResult.error}`); + console.error(`Files are at: gs://${bucketName}/${uploadResult.versionPath}`); + console.error('Manual pointer update may be required.'); process.exit(1); } diff --git a/common/scripts/ofac/test-ofac-update.sh b/common/scripts/ofac/test-ofac-update.sh new file mode 100755 index 000000000..c6813a917 --- /dev/null +++ b/common/scripts/ofac/test-ofac-update.sh @@ -0,0 +1,227 @@ +#!/bin/bash +# OFAC Auto Updater Test Script +# Tests the complete OFAC update pipeline in dry-run mode + +set -e + +echo "╔══════════════════════════════════════════════════════════════╗" +echo "║ OFAC AUTO UPDATER - TEST SUITE ║" +echo "╚══════════════════════════════════════════════════════════════╝" +echo "" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Test configuration +TEST_DIR="./test-ofac-$(date +%s)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +echo -e "${BLUE}📍 Repository root: $REPO_ROOT${NC}" +echo -e "${BLUE}📂 Test directory: $TEST_DIR${NC}" +echo "" + +# Create test directory +mkdir -p "$TEST_DIR" +cd "$REPO_ROOT" + +# ═══════════════════════════════════════════════════════════════ +# TEST 1: Pipeline Test (Download + Parse + Build) +# ═══════════════════════════════════════════════════════════════ +echo "" +echo "═══════════════════════════════════════════════════════════════" +echo " TEST 1: OFAC Pipeline (Download → Parse → Build)" +echo "═══════════════════════════════════════════════════════════════" +echo "" + +echo -e "${YELLOW}→ Running pipeline...${NC}" +yarn tsx common/scripts/ofac/index.ts --output-dir "$TEST_DIR" + +if [ $? -eq 0 ]; then + echo "" + echo -e "${GREEN}✓ Pipeline completed successfully${NC}" +else + echo -e "${RED}✗ Pipeline failed${NC}" + exit 1 +fi + +# Verify outputs +echo "" +echo -e "${YELLOW}→ Verifying outputs...${NC}" + +EXPECTED_FILES=( + "raw/sdn-latest.xml" + "inputs/names.json" + "inputs/passports.json" + "outputs/passportNoAndNationalitySMT.json" + "outputs/nameAndDobSMT.json" + "outputs/nameAndYobSMT.json" + "outputs/nameAndDobSMT_ID.json" + "outputs/nameAndYobSMT_ID.json" + "outputs/nameAndDobSMT_AADHAAR.json" + "outputs/nameAndYobSMT_AADHAAR.json" + "outputs/roots.json" + "outputs/latest-roots.json" +) + +MISSING_FILES=0 +for file in "${EXPECTED_FILES[@]}"; do + if [ -f "$TEST_DIR/$file" ]; then + SIZE=$(du -h "$TEST_DIR/$file" | cut -f1) + echo -e "${GREEN} ✓ $file ($SIZE)${NC}" + else + echo -e "${RED} ✗ Missing: $file${NC}" + MISSING_FILES=$((MISSING_FILES + 1)) + fi +done + +if [ $MISSING_FILES -eq 0 ]; then + echo "" + echo -e "${GREEN}✓ All expected files created${NC}" +else + echo "" + echo -e "${RED}✗ $MISSING_FILES files missing${NC}" + exit 1 +fi + +# ═══════════════════════════════════════════════════════════════ +# TEST 2: Roots Validation +# ═══════════════════════════════════════════════════════════════ +echo "" +echo "═══════════════════════════════════════════════════════════════" +echo " TEST 2: Roots Validation" +echo "═══════════════════════════════════════════════════════════════" +echo "" + +echo -e "${YELLOW}→ Checking roots.json structure...${NC}" + +# Check if roots.json is valid JSON and has expected keys +ROOTS_FILE="$TEST_DIR/outputs/roots.json" +if ! jq empty "$ROOTS_FILE" 2>/dev/null; then + echo -e "${RED}✗ roots.json is not valid JSON${NC}" + exit 1 +fi + +EXPECTED_ROOTS=( + "passport_no_and_nationality" + "name_and_dob" + "name_and_yob" + "name_and_dob_id_card" + "name_and_yob_id_card" + "aadhaar_name_and_dob" + "aadhaar_name_and_yob" +) + +MISSING_ROOTS=0 +for root in "${EXPECTED_ROOTS[@]}"; do + if jq -e ".$root" "$ROOTS_FILE" > /dev/null 2>&1; then + ROOT_VALUE=$(jq -r ".$root" "$ROOTS_FILE") + echo -e "${GREEN} ✓ $root: ${ROOT_VALUE:0:20}...${NC}" + else + echo -e "${RED} ✗ Missing root: $root${NC}" + MISSING_ROOTS=$((MISSING_ROOTS + 1)) + fi +done + +if [ $MISSING_ROOTS -eq 0 ]; then + echo "" + echo -e "${GREEN}✓ All roots present and valid${NC}" +else + echo "" + echo -e "${RED}✗ $MISSING_ROOTS roots missing${NC}" + exit 1 +fi + +# ═══════════════════════════════════════════════════════════════ +# TEST 3: Tree Statistics +# ═══════════════════════════════════════════════════════════════ +echo "" +echo "═══════════════════════════════════════════════════════════════" +echo " TEST 3: Tree Statistics" +echo "═══════════════════════════════════════════════════════════════" +echo "" + +NAMES_COUNT=$(jq '. | length' "$TEST_DIR/inputs/names.json") +PASSPORTS_COUNT=$(jq '. | length' "$TEST_DIR/inputs/passports.json") + +echo -e "${BLUE} Names entries: $NAMES_COUNT${NC}" +echo -e "${BLUE} Passport entries: $PASSPORTS_COUNT${NC}" + +# Check if numbers are reasonable (should be thousands) +if [ "$NAMES_COUNT" -lt 1000 ]; then + echo -e "${RED}✗ Too few names entries (expected > 1000)${NC}" + exit 1 +fi + +if [ "$PASSPORTS_COUNT" -lt 100 ]; then + echo -e "${RED}✗ Too few passport entries (expected > 100)${NC}" + exit 1 +fi + +echo "" +echo -e "${GREEN}✓ Entry counts look reasonable${NC}" + +# ═══════════════════════════════════════════════════════════════ +# TEST 4: Dry-Run Full Update (if credentials available) +# ═══════════════════════════════════════════════════════════════ +echo "" +echo "═══════════════════════════════════════════════════════════════" +echo " TEST 4: Dry-Run Full Update" +echo "═══════════════════════════════════════════════════════════════" +echo "" + +if [ -z "$PRIVATE_KEY" ]; then + echo -e "${YELLOW}⚠ PRIVATE_KEY not set - skipping dry-run test${NC}" + echo -e "${YELLOW} To run: export PRIVATE_KEY=0x...${NC}" +else + echo -e "${YELLOW}→ Running dry-run update...${NC}" + + export NETWORK="${NETWORK:-celo-sepolia}" + export DRY_RUN=true + export OFAC_DATA_DIR="$TEST_DIR" + export GCS_BUCKET_NAME="${GCS_BUCKET_NAME:-self-ofac-test}" + + # Run with existing data (skip download) + yarn tsx common/scripts/ofac/runOfacAutoUpdate.ts + + if [ $? -eq 0 ]; then + echo "" + echo -e "${GREEN}✓ Dry-run completed successfully${NC}" + else + echo -e "${RED}✗ Dry-run failed${NC}" + exit 1 + fi +fi + +# ═══════════════════════════════════════════════════════════════ +# SUMMARY +# ═══════════════════════════════════════════════════════════════ +echo "" +echo "╔══════════════════════════════════════════════════════════════╗" +echo "║ TEST SUMMARY ║" +echo "╚══════════════════════════════════════════════════════════════╝" +echo "" +echo -e "${GREEN}✓ All tests passed!${NC}" +echo "" +echo "Test artifacts saved to: $TEST_DIR" +echo "" +echo "Next steps:" +echo " 1. Review generated files in $TEST_DIR" +echo " 2. Test with real GCS credentials (set GOOGLE_APPLICATION_CREDENTIALS)" +echo " 3. Test on staging network (celo-sepolia)" +echo " 4. Deploy to production when ready" +echo "" + +# Optional: Clean up test directory +read -p "Delete test directory? (y/N) " -n 1 -r +echo +if [[ $REPLY =~ ^[Yy]$ ]]; then + rm -rf "$TEST_DIR" + echo -e "${GREEN}✓ Test directory deleted${NC}" +else + echo -e "${BLUE}ℹ Test directory preserved: $TEST_DIR${NC}" +fi diff --git a/test-dry-run.sh b/test-dry-run.sh new file mode 100755 index 000000000..8934ada0d --- /dev/null +++ b/test-dry-run.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Complete Dry-Run Test for OFAC Auto Updater +# This simulates the full flow without making real updates + +set -e + +echo "╔══════════════════════════════════════════════════════════════╗" +echo "║ OFAC AUTO UPDATER - DRY RUN TEST ║" +echo "╚══════════════════════════════════════════════════════════════╝" +echo "" + +# Set environment variables for dry-run +export PRIVATE_KEY=0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef +export NETWORK=celo-sepolia +export DRY_RUN=true +export OFAC_DATA_DIR=/Users/evinova/Documents/self/test-ofac-data +export GCS_BUCKET_NAME=self-ofac-test +export GCS_BASE_PATH=ofac +export SKIP_UPLOAD=false # Test GCS upload logic (but won't actually upload in dry-run) + +echo "🔧 Configuration:" +echo " Network: $NETWORK" +echo " Data Dir: $OFAC_DATA_DIR" +echo " GCS Bucket: $GCS_BUCKET_NAME" +echo " Dry Run: $DRY_RUN" +echo "" +echo "This will simulate:" +echo " ✓ Pipeline execution (using existing data)" +echo " ✓ On-chain root updates (simulated)" +echo " ✓ GCS upload (simulated)" +echo " ✓ Pointer file update (simulated)" +echo "" + +read -p "Press Enter to start dry-run test..." +echo "" + +cd /Users/evinova/Documents/self + +# Run the complete auto-updater +yarn tsx common/scripts/ofac/runOfacAutoUpdate.ts + +echo "" +echo "╔══════════════════════════════════════════════════════════════╗" +echo "║ DRY RUN TEST COMPLETED ║" +echo "╚══════════════════════════════════════════════════════════════╝" + diff --git a/yarn.lock b/yarn.lock index d9e270a75..67f61f8df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4482,6 +4482,53 @@ __metadata: languageName: node linkType: hard +"@google-cloud/paginator@npm:^5.0.0": + version: 5.0.2 + resolution: "@google-cloud/paginator@npm:5.0.2" + dependencies: + arrify: "npm:^2.0.0" + extend: "npm:^3.0.2" + checksum: 10c0/aac4ed986c2b274ac9fdca3f68d5ba6ee95f4c35370b11db25c288bf485352e2ec5df16bf9c3cff554a2e73a07e62f10044d273788df61897b81fe47bb18106d + languageName: node + linkType: hard + +"@google-cloud/projectify@npm:^4.0.0": + version: 4.0.0 + resolution: "@google-cloud/projectify@npm:4.0.0" + checksum: 10c0/0d0a6ceca76a138973fcb3ad577f209acdbd9d9aed1c645b09f98d5e5a258053dbbe6c1f13e6f85310cc0d9308f5f3a84f8fa4f1a132549a68d86174fb21067f + languageName: node + linkType: hard + +"@google-cloud/promisify@npm:<4.1.0": + version: 4.0.0 + resolution: "@google-cloud/promisify@npm:4.0.0" + checksum: 10c0/4332cbd923d7c6943ecdf46f187f1417c84bb9c801525cd74d719c766bfaad650f7964fb74576345f6537b6d6273a4f2992c8d79ebec6c8b8401b23d626b8dd3 + languageName: node + linkType: hard + +"@google-cloud/storage@npm:^7.18.0": + version: 7.18.0 + resolution: "@google-cloud/storage@npm:7.18.0" + dependencies: + "@google-cloud/paginator": "npm:^5.0.0" + "@google-cloud/projectify": "npm:^4.0.0" + "@google-cloud/promisify": "npm:<4.1.0" + abort-controller: "npm:^3.0.0" + async-retry: "npm:^1.3.3" + duplexify: "npm:^4.1.3" + fast-xml-parser: "npm:^4.4.1" + gaxios: "npm:^6.0.2" + google-auth-library: "npm:^9.6.3" + html-entities: "npm:^2.5.2" + mime: "npm:^3.0.0" + p-limit: "npm:^3.0.1" + retry-request: "npm:^7.0.0" + teeny-request: "npm:^9.0.0" + uuid: "npm:^8.0.0" + checksum: 10c0/1879a7c60a0a23890067d0b17359da701d0504e46b8e4c0b3cdfd29dcd54fcaaddada68206d1d14fafadea86eb0a885bd8cc725c453def845f9bd9aae2cc3a85 + languageName: node + linkType: hard + "@hapi/hoek@npm:^9.0.0, @hapi/hoek@npm:^9.3.0": version: 9.3.0 resolution: "@hapi/hoek@npm:9.3.0" @@ -8385,6 +8432,7 @@ __metadata: resolution: "@selfxyz/common@workspace:common" dependencies: "@anon-aadhaar/core": "npm:@selfxyz/anon-aadhaar-core@^0.0.1" + "@google-cloud/storage": "npm:^7.18.0" "@noble/hashes": "npm:^1.5.0" "@openpassport/zk-kit-imt": "npm:^0.0.5" "@openpassport/zk-kit-lean-imt": "npm:^0.0.6" @@ -13156,6 +13204,13 @@ __metadata: languageName: node linkType: hard +"@tootallnate/once@npm:2": + version: 2.0.0 + resolution: "@tootallnate/once@npm:2.0.0" + checksum: 10c0/073bfa548026b1ebaf1659eb8961e526be22fa77139b10d60e712f46d2f0f05f4e6c8bec62a087d41088ee9e29faa7f54838568e475ab2f776171003c3920858 + languageName: node + linkType: hard + "@tootallnate/quickjs-emscripten@npm:^0.23.0": version: 0.23.0 resolution: "@tootallnate/quickjs-emscripten@npm:0.23.0" @@ -13512,6 +13567,13 @@ __metadata: languageName: node linkType: hard +"@types/caseless@npm:*": + version: 0.12.5 + resolution: "@types/caseless@npm:0.12.5" + checksum: 10c0/b1f8b8a38ce747b643115d37a40ea824c658bd7050e4b69427a10e9d12d1606ed17a0f6018241c08291cd59f70aeb3c1f3754ad61e45f8dbba708ec72dde7ec8 + languageName: node + linkType: hard + "@types/chai-as-promised@npm:^7.1.3, @types/chai-as-promised@npm:^7.1.6, @types/chai-as-promised@npm:^7.1.8": version: 7.1.8 resolution: "@types/chai-as-promised@npm:7.1.8" @@ -14007,6 +14069,18 @@ __metadata: languageName: node linkType: hard +"@types/request@npm:^2.48.8": + version: 2.48.13 + resolution: "@types/request@npm:2.48.13" + dependencies: + "@types/caseless": "npm:*" + "@types/node": "npm:*" + "@types/tough-cookie": "npm:*" + form-data: "npm:^2.5.5" + checksum: 10c0/1c6798d926a6577f213dbc04aa09945590f260ea367537c20824ff337b0a49d56e5199a6a6029e625568d97c3bbb98908bdb8d9158eb421f70a0d03ae230ff72 + languageName: node + linkType: hard + "@types/responselike@npm:^1.0.0": version: 1.0.3 resolution: "@types/responselike@npm:1.0.3" @@ -14118,6 +14192,13 @@ __metadata: languageName: node linkType: hard +"@types/tough-cookie@npm:*": + version: 4.0.5 + resolution: "@types/tough-cookie@npm:4.0.5" + checksum: 10c0/68c6921721a3dcb40451543db2174a145ef915bc8bcbe7ad4e59194a0238e776e782b896c7a59f4b93ac6acefca9161fccb31d1ce3b3445cb6faa467297fb473 + languageName: node + linkType: hard + "@types/treeify@npm:^1.0.0": version: 1.0.3 resolution: "@types/treeify@npm:1.0.3" @@ -16264,6 +16345,13 @@ __metadata: languageName: node linkType: hard +"arrify@npm:^2.0.0": + version: 2.0.1 + resolution: "arrify@npm:2.0.1" + checksum: 10c0/3fb30b5e7c37abea1907a60b28a554d2f0fc088757ca9bf5b684786e583fdf14360721eb12575c1ce6f995282eab936712d3c4389122682eafab0e0b57f78dbb + languageName: node + linkType: hard + "asap@npm:~2.0.3, asap@npm:~2.0.6": version: 2.0.6 resolution: "asap@npm:2.0.6" @@ -16835,7 +16923,7 @@ __metadata: languageName: node linkType: hard -"base64-js@npm:^1.0.2, base64-js@npm:^1.3.1, base64-js@npm:^1.5.1": +"base64-js@npm:^1.0.2, base64-js@npm:^1.3.0, base64-js@npm:^1.3.1, base64-js@npm:^1.5.1": version: 1.5.1 resolution: "base64-js@npm:1.5.1" checksum: 10c0/f23823513b63173a001030fae4f2dabe283b99a9d324ade3ad3d148e218134676f1ee8568c877cd79ec1c53158dcf2d2ba527a97c606618928ba99dd930102bf @@ -19477,6 +19565,18 @@ __metadata: languageName: node linkType: hard +"duplexify@npm:^4.1.3": + version: 4.1.3 + resolution: "duplexify@npm:4.1.3" + dependencies: + end-of-stream: "npm:^1.4.1" + inherits: "npm:^2.0.3" + readable-stream: "npm:^3.1.1" + stream-shift: "npm:^1.0.2" + checksum: 10c0/8a7621ae95c89f3937f982fe36d72ea997836a708471a75bb2a0eecde3330311b1e128a6dad510e0fd64ace0c56bff3484ed2e82af0e465600c82117eadfbda5 + languageName: node + linkType: hard + "eastasianwidth@npm:^0.2.0": version: 0.2.0 resolution: "eastasianwidth@npm:0.2.0" @@ -19484,7 +19584,7 @@ __metadata: languageName: node linkType: hard -"ecdsa-sig-formatter@npm:1.0.11": +"ecdsa-sig-formatter@npm:1.0.11, ecdsa-sig-formatter@npm:^1.0.11": version: 1.0.11 resolution: "ecdsa-sig-formatter@npm:1.0.11" dependencies: @@ -19611,7 +19711,7 @@ __metadata: languageName: node linkType: hard -"end-of-stream@npm:^1.1.0": +"end-of-stream@npm:^1.1.0, end-of-stream@npm:^1.4.1": version: 1.4.5 resolution: "end-of-stream@npm:1.4.5" dependencies: @@ -21426,6 +21526,13 @@ __metadata: languageName: node linkType: hard +"extend@npm:^3.0.2": + version: 3.0.2 + resolution: "extend@npm:3.0.2" + checksum: 10c0/73bf6e27406e80aa3e85b0d1c4fd987261e628064e170ca781125c0b635a3dabad5e05adbf07595ea0cf1e6c5396cacb214af933da7cbaf24fe75ff14818e8f9 + languageName: node + linkType: hard + "extract-zip@npm:^2.0.1": version: 2.0.1 resolution: "extract-zip@npm:2.0.1" @@ -22008,7 +22115,7 @@ __metadata: languageName: node linkType: hard -"form-data@npm:^2.2.0": +"form-data@npm:^2.2.0, form-data@npm:^2.5.5": version: 2.5.5 resolution: "form-data@npm:2.5.5" dependencies: @@ -22284,6 +22391,30 @@ __metadata: languageName: node linkType: hard +"gaxios@npm:^6.0.0, gaxios@npm:^6.0.2, gaxios@npm:^6.1.1": + version: 6.7.1 + resolution: "gaxios@npm:6.7.1" + dependencies: + extend: "npm:^3.0.2" + https-proxy-agent: "npm:^7.0.1" + is-stream: "npm:^2.0.0" + node-fetch: "npm:^2.6.9" + uuid: "npm:^9.0.1" + checksum: 10c0/53e92088470661c5bc493a1de29d05aff58b1f0009ec5e7903f730f892c3642a93e264e61904383741ccbab1ce6e519f12a985bba91e13527678b32ee6d7d3fd + languageName: node + linkType: hard + +"gcp-metadata@npm:^6.1.0": + version: 6.1.1 + resolution: "gcp-metadata@npm:6.1.1" + dependencies: + gaxios: "npm:^6.1.1" + google-logging-utils: "npm:^0.0.2" + json-bigint: "npm:^1.0.0" + checksum: 10c0/71f6ad4800aa622c246ceec3955014c0c78cdcfe025971f9558b9379f4019f5e65772763428ee8c3244fa81b8631977316eaa71a823493f82e5c44d7259ffac8 + languageName: node + linkType: hard + "generator-function@npm:^2.0.0": version: 2.0.1 resolution: "generator-function@npm:2.0.1" @@ -22655,6 +22786,27 @@ __metadata: languageName: node linkType: hard +"google-auth-library@npm:^9.6.3": + version: 9.15.1 + resolution: "google-auth-library@npm:9.15.1" + dependencies: + base64-js: "npm:^1.3.0" + ecdsa-sig-formatter: "npm:^1.0.11" + gaxios: "npm:^6.1.1" + gcp-metadata: "npm:^6.1.0" + gtoken: "npm:^7.0.0" + jws: "npm:^4.0.0" + checksum: 10c0/6eef36d9a9cb7decd11e920ee892579261c6390104b3b24d3e0f3889096673189fe2ed0ee43fd563710e2560de98e63ad5aa4967b91e7f4e69074a422d5f7b65 + languageName: node + linkType: hard + +"google-logging-utils@npm:^0.0.2": + version: 0.0.2 + resolution: "google-logging-utils@npm:0.0.2" + checksum: 10c0/9a4bbd470dd101c77405e450fffca8592d1d7114f245a121288d04a957aca08c9dea2dd1a871effe71e41540d1bb0494731a0b0f6fea4358e77f06645e4268c1 + languageName: node + linkType: hard + "gopd@npm:^1.0.1, gopd@npm:^1.2.0": version: 1.2.0 resolution: "gopd@npm:1.2.0" @@ -22702,6 +22854,16 @@ __metadata: languageName: node linkType: hard +"gtoken@npm:^7.0.0": + version: 7.1.0 + resolution: "gtoken@npm:7.1.0" + dependencies: + gaxios: "npm:^6.0.0" + jws: "npm:^4.0.0" + checksum: 10c0/0a3dcacb1a3c4578abe1ee01c7d0bf20bffe8ded3ee73fc58885d53c00f6eb43b4e1372ff179f0da3ed5cfebd5b7c6ab8ae2776f1787e90d943691b4fe57c716 + languageName: node + linkType: hard + "h3@npm:^1.15.4": version: 1.15.4 resolution: "h3@npm:1.15.4" @@ -23089,6 +23251,13 @@ __metadata: languageName: node linkType: hard +"html-entities@npm:^2.5.2": + version: 2.6.0 + resolution: "html-entities@npm:2.6.0" + checksum: 10c0/7c8b15d9ea0cd00dc9279f61bab002ba6ca8a7a0f3c36ed2db3530a67a9621c017830d1d2c1c65beb9b8e3436ea663e9cf8b230472e0e413359399413b27c8b7 + languageName: node + linkType: hard + "html-escaper@npm:^2.0.0": version: 2.0.2 resolution: "html-escaper@npm:2.0.2" @@ -23176,6 +23345,17 @@ __metadata: languageName: node linkType: hard +"http-proxy-agent@npm:^5.0.0": + version: 5.0.0 + resolution: "http-proxy-agent@npm:5.0.0" + dependencies: + "@tootallnate/once": "npm:2" + agent-base: "npm:6" + debug: "npm:4" + checksum: 10c0/32a05e413430b2c1e542e5c74b38a9f14865301dd69dff2e53ddb684989440e3d2ce0c4b64d25eb63cf6283e6265ff979a61cf93e3ca3d23047ddfdc8df34a32 + languageName: node + linkType: hard + "http-proxy-agent@npm:^7.0.0, http-proxy-agent@npm:^7.0.1, http-proxy-agent@npm:^7.0.2": version: 7.0.2 resolution: "http-proxy-agent@npm:7.0.2" @@ -25337,7 +25517,7 @@ __metadata: languageName: node linkType: hard -"jws@npm:^4.0.1": +"jws@npm:^4.0.0, jws@npm:^4.0.1": version: 4.0.1 resolution: "jws@npm:4.0.1" dependencies: @@ -26688,6 +26868,15 @@ __metadata: languageName: node linkType: hard +"mime@npm:^3.0.0": + version: 3.0.0 + resolution: "mime@npm:3.0.0" + bin: + mime: cli.js + checksum: 10c0/402e792a8df1b2cc41cb77f0dcc46472b7944b7ec29cb5bbcd398624b6b97096728f1239766d3fdeb20551dd8d94738344c195a6ea10c4f906eb0356323b0531 + languageName: node + linkType: hard + "mimic-fn@npm:^2.1.0": version: 2.1.0 resolution: "mimic-fn@npm:2.1.0" @@ -27510,7 +27699,7 @@ __metadata: languageName: node linkType: hard -"node-fetch@npm:^2.1.1, node-fetch@npm:^2.2.0, node-fetch@npm:^2.6.1, node-fetch@npm:^2.6.7, node-fetch@npm:^2.7.0": +"node-fetch@npm:^2.1.1, node-fetch@npm:^2.2.0, node-fetch@npm:^2.6.1, node-fetch@npm:^2.6.7, node-fetch@npm:^2.6.9, node-fetch@npm:^2.7.0": version: 2.7.0 resolution: "node-fetch@npm:2.7.0" dependencies: @@ -28412,7 +28601,7 @@ __metadata: languageName: node linkType: hard -"p-limit@npm:^3.0.2, p-limit@npm:^3.1.0": +"p-limit@npm:^3.0.1, p-limit@npm:^3.0.2, p-limit@npm:^3.1.0": version: 3.1.0 resolution: "p-limit@npm:3.1.0" dependencies: @@ -30359,7 +30548,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:3, readable-stream@npm:^3.0.0, readable-stream@npm:^3.0.6, readable-stream@npm:^3.4.0, readable-stream@npm:^3.5.0, readable-stream@npm:^3.6.0": +"readable-stream@npm:3, readable-stream@npm:^3.0.0, readable-stream@npm:^3.0.6, readable-stream@npm:^3.1.1, readable-stream@npm:^3.4.0, readable-stream@npm:^3.5.0, readable-stream@npm:^3.6.0": version: 3.6.2 resolution: "readable-stream@npm:3.6.2" dependencies: @@ -30817,6 +31006,17 @@ __metadata: languageName: node linkType: hard +"retry-request@npm:^7.0.0": + version: 7.0.2 + resolution: "retry-request@npm:7.0.2" + dependencies: + "@types/request": "npm:^2.48.8" + extend: "npm:^3.0.2" + teeny-request: "npm:^9.0.0" + checksum: 10c0/c79936695a43db1bc82a7bad348a1e0be1c363799be2e1fa87b8c3aeb5dabf0ccb023b811aa5000c000ee73e196b88febff7d3e22cbb63a77175228514256155 + languageName: node + linkType: hard + "retry@npm:0.13.1, retry@npm:^0.13.1": version: 0.13.1 resolution: "retry@npm:0.13.1" @@ -32405,6 +32605,22 @@ __metadata: languageName: node linkType: hard +"stream-events@npm:^1.0.5": + version: 1.0.5 + resolution: "stream-events@npm:1.0.5" + dependencies: + stubs: "npm:^3.0.0" + checksum: 10c0/5d235a5799a483e94ea8829526fe9d95d76460032d5e78555fe4f801949ac6a27ea2212e4e0827c55f78726b3242701768adf2d33789465f51b31ed8ebd6b086 + languageName: node + linkType: hard + +"stream-shift@npm:^1.0.2": + version: 1.0.3 + resolution: "stream-shift@npm:1.0.3" + checksum: 10c0/939cd1051ca750d240a0625b106a2b988c45fb5a3be0cebe9a9858cb01bc1955e8c7b9fac17a9462976bea4a7b704e317c5c2200c70f0ca715a3363b9aa4fd3b + languageName: node + linkType: hard + "streamx@npm:^2.15.0, streamx@npm:^2.21.0": version: 2.23.0 resolution: "streamx@npm:2.23.0" @@ -32744,6 +32960,13 @@ __metadata: languageName: node linkType: hard +"stubs@npm:^3.0.0": + version: 3.0.0 + resolution: "stubs@npm:3.0.0" + checksum: 10c0/841a4ab8c76795d34aefe129185763b55fbf2e4693208215627caea4dd62e1299423dcd96f708d3128e3dfa0e669bae2cb912e6e906d7d81eaf6493196570923 + languageName: node + linkType: hard + "style-value-types@npm:5.0.0": version: 5.0.0 resolution: "style-value-types@npm:5.0.0" @@ -33092,6 +33315,19 @@ __metadata: languageName: node linkType: hard +"teeny-request@npm:^9.0.0": + version: 9.0.0 + resolution: "teeny-request@npm:9.0.0" + dependencies: + http-proxy-agent: "npm:^5.0.0" + https-proxy-agent: "npm:^5.0.0" + node-fetch: "npm:^2.6.9" + stream-events: "npm:^1.0.5" + uuid: "npm:^9.0.0" + checksum: 10c0/1c51a284075b57b7b7f970fc8d855d611912f0e485aa1d1dfda3c0be3f2df392e4ce83b1b39877134041abb7c255f3777f175b27323ef5bf008839e42a1958bc + languageName: node + linkType: hard + "temp@npm:^0.8.4": version: 0.8.4 resolution: "temp@npm:0.8.4" @@ -34498,7 +34734,7 @@ __metadata: languageName: node linkType: hard -"uuid@npm:^8.3.2": +"uuid@npm:^8.0.0, uuid@npm:^8.3.2": version: 8.3.2 resolution: "uuid@npm:8.3.2" bin: