mirror of
https://github.com/simstudioai/sim.git
synced 2026-01-09 15:07:55 -05:00
fix(redaction): consolidate redaction utils, apply them to inputs and outputs before persisting logs (#2478)
* fix(redaction): consolidate redaction utils, apply them to inputs and outputs before persisting logs * added testing utils
This commit is contained in:
@@ -2,6 +2,7 @@ import { type NextRequest, NextResponse } from 'next/server'
|
|||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { auth } from '@/lib/auth'
|
import { auth } from '@/lib/auth'
|
||||||
import { env } from '@/lib/core/config/env'
|
import { env } from '@/lib/core/config/env'
|
||||||
|
import { REDACTED_MARKER } from '@/lib/core/security/redaction'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
|
|
||||||
const logger = createLogger('SSO-Register')
|
const logger = createLogger('SSO-Register')
|
||||||
@@ -236,13 +237,13 @@ export async function POST(request: NextRequest) {
|
|||||||
oidcConfig: providerConfig.oidcConfig
|
oidcConfig: providerConfig.oidcConfig
|
||||||
? {
|
? {
|
||||||
...providerConfig.oidcConfig,
|
...providerConfig.oidcConfig,
|
||||||
clientSecret: '[REDACTED]',
|
clientSecret: REDACTED_MARKER,
|
||||||
}
|
}
|
||||||
: undefined,
|
: undefined,
|
||||||
samlConfig: providerConfig.samlConfig
|
samlConfig: providerConfig.samlConfig
|
||||||
? {
|
? {
|
||||||
...providerConfig.samlConfig,
|
...providerConfig.samlConfig,
|
||||||
cert: '[REDACTED]',
|
cert: REDACTED_MARKER,
|
||||||
}
|
}
|
||||||
: undefined,
|
: undefined,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { type NextRequest, NextResponse } from 'next/server'
|
import { type NextRequest, NextResponse } from 'next/server'
|
||||||
import { z } from 'zod'
|
import { z } from 'zod'
|
||||||
import { env } from '@/lib/core/config/env'
|
import { env } from '@/lib/core/config/env'
|
||||||
|
import { isSensitiveKey, REDACTED_MARKER } from '@/lib/core/security/redaction'
|
||||||
import { createLogger } from '@/lib/logs/console/logger'
|
import { createLogger } from '@/lib/logs/console/logger'
|
||||||
import { ensureZodObject, normalizeUrl } from '@/app/api/tools/stagehand/utils'
|
import { ensureZodObject, normalizeUrl } from '@/app/api/tools/stagehand/utils'
|
||||||
|
|
||||||
@@ -188,7 +189,7 @@ export async function POST(request: NextRequest) {
|
|||||||
|
|
||||||
if (variablesObject && Object.keys(variablesObject).length > 0) {
|
if (variablesObject && Object.keys(variablesObject).length > 0) {
|
||||||
const safeVarKeys = Object.keys(variablesObject).map((key) => {
|
const safeVarKeys = Object.keys(variablesObject).map((key) => {
|
||||||
return key.toLowerCase().includes('password') ? `${key}: [REDACTED]` : key
|
return isSensitiveKey(key) ? `${key}: ${REDACTED_MARKER}` : key
|
||||||
})
|
})
|
||||||
logger.info('Variables available for task', { variables: safeVarKeys })
|
logger.info('Variables available for task', { variables: safeVarKeys })
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { env } from './lib/core/config/env'
|
import { env } from './lib/core/config/env'
|
||||||
|
import { sanitizeEventData } from './lib/core/security/redaction'
|
||||||
|
|
||||||
if (typeof window !== 'undefined') {
|
if (typeof window !== 'undefined') {
|
||||||
const TELEMETRY_STATUS_KEY = 'simstudio-telemetry-status'
|
const TELEMETRY_STATUS_KEY = 'simstudio-telemetry-status'
|
||||||
@@ -41,37 +42,6 @@ if (typeof window !== 'undefined') {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Sanitize event data to remove sensitive information
|
|
||||||
*/
|
|
||||||
function sanitizeEvent(event: any): any {
|
|
||||||
const patterns = ['password', 'token', 'secret', 'key', 'auth', 'credential', 'private']
|
|
||||||
const sensitiveRe = new RegExp(patterns.join('|'), 'i')
|
|
||||||
|
|
||||||
const scrubString = (s: string) => (s && sensitiveRe.test(s) ? '[redacted]' : s)
|
|
||||||
|
|
||||||
if (event == null) return event
|
|
||||||
if (typeof event === 'string') return scrubString(event)
|
|
||||||
if (typeof event !== 'object') return event
|
|
||||||
|
|
||||||
if (Array.isArray(event)) {
|
|
||||||
return event.map((item) => sanitizeEvent(item))
|
|
||||||
}
|
|
||||||
|
|
||||||
const sanitized: Record<string, unknown> = {}
|
|
||||||
for (const [key, value] of Object.entries(event)) {
|
|
||||||
const lowerKey = key.toLowerCase()
|
|
||||||
if (patterns.some((p) => lowerKey.includes(p))) continue
|
|
||||||
|
|
||||||
if (typeof value === 'string') sanitized[key] = scrubString(value)
|
|
||||||
else if (Array.isArray(value)) sanitized[key] = value.map((v) => sanitizeEvent(v))
|
|
||||||
else if (value && typeof value === 'object') sanitized[key] = sanitizeEvent(value)
|
|
||||||
else sanitized[key] = value
|
|
||||||
}
|
|
||||||
|
|
||||||
return sanitized
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Flush batch of events to server
|
* Flush batch of events to server
|
||||||
*/
|
*/
|
||||||
@@ -84,7 +54,7 @@ if (typeof window !== 'undefined') {
|
|||||||
batchTimer = null
|
batchTimer = null
|
||||||
}
|
}
|
||||||
|
|
||||||
const sanitizedBatch = batch.map(sanitizeEvent)
|
const sanitizedBatch = batch.map(sanitizeEventData)
|
||||||
|
|
||||||
const payload = JSON.stringify({
|
const payload = JSON.stringify({
|
||||||
category: 'batch',
|
category: 'batch',
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
import { describe, expect, it } from 'vitest'
|
import { describe, expect, it } from 'vitest'
|
||||||
import {
|
import {
|
||||||
createPinnedUrl,
|
createPinnedUrl,
|
||||||
sanitizeForLogging,
|
|
||||||
validateAlphanumericId,
|
validateAlphanumericId,
|
||||||
validateEnum,
|
validateEnum,
|
||||||
validateFileExtension,
|
validateFileExtension,
|
||||||
@@ -11,6 +10,7 @@ import {
|
|||||||
validateUrlWithDNS,
|
validateUrlWithDNS,
|
||||||
validateUUID,
|
validateUUID,
|
||||||
} from '@/lib/core/security/input-validation'
|
} from '@/lib/core/security/input-validation'
|
||||||
|
import { sanitizeForLogging } from '@/lib/core/security/redaction'
|
||||||
|
|
||||||
describe('validatePathSegment', () => {
|
describe('validatePathSegment', () => {
|
||||||
describe('valid inputs', () => {
|
describe('valid inputs', () => {
|
||||||
|
|||||||
@@ -556,29 +556,6 @@ export function validateFileExtension(
|
|||||||
return { isValid: true, sanitized: normalizedExt }
|
return { isValid: true, sanitized: normalizedExt }
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Sanitizes a string for safe logging (removes potential sensitive data patterns)
|
|
||||||
*
|
|
||||||
* @param value - The value to sanitize
|
|
||||||
* @param maxLength - Maximum length to return (default: 100)
|
|
||||||
* @returns Sanitized string safe for logging
|
|
||||||
*/
|
|
||||||
export function sanitizeForLogging(value: string, maxLength = 100): string {
|
|
||||||
if (!value) return ''
|
|
||||||
|
|
||||||
// Truncate long values
|
|
||||||
let sanitized = value.substring(0, maxLength)
|
|
||||||
|
|
||||||
// Mask common sensitive patterns
|
|
||||||
sanitized = sanitized
|
|
||||||
.replace(/Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi, 'Bearer [REDACTED]')
|
|
||||||
.replace(/password['":\s]*['"]\w+['"]/gi, 'password: "[REDACTED]"')
|
|
||||||
.replace(/token['":\s]*['"]\w+['"]/gi, 'token: "[REDACTED]"')
|
|
||||||
.replace(/api[_-]?key['":\s]*['"]\w+['"]/gi, 'api_key: "[REDACTED]"')
|
|
||||||
|
|
||||||
return sanitized
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Validates Microsoft Graph API resource IDs
|
* Validates Microsoft Graph API resource IDs
|
||||||
*
|
*
|
||||||
|
|||||||
391
apps/sim/lib/core/security/redaction.test.ts
Normal file
391
apps/sim/lib/core/security/redaction.test.ts
Normal file
@@ -0,0 +1,391 @@
|
|||||||
|
import { describe, expect, it } from 'vitest'
|
||||||
|
import {
|
||||||
|
isSensitiveKey,
|
||||||
|
REDACTED_MARKER,
|
||||||
|
redactApiKeys,
|
||||||
|
redactSensitiveValues,
|
||||||
|
sanitizeEventData,
|
||||||
|
sanitizeForLogging,
|
||||||
|
} from './redaction'
|
||||||
|
|
||||||
|
describe('REDACTED_MARKER', () => {
|
||||||
|
it.concurrent('should be the standard marker', () => {
|
||||||
|
expect(REDACTED_MARKER).toBe('[REDACTED]')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('isSensitiveKey', () => {
|
||||||
|
describe('exact matches', () => {
|
||||||
|
it.concurrent('should match apiKey variations', () => {
|
||||||
|
expect(isSensitiveKey('apiKey')).toBe(true)
|
||||||
|
expect(isSensitiveKey('api_key')).toBe(true)
|
||||||
|
expect(isSensitiveKey('api-key')).toBe(true)
|
||||||
|
expect(isSensitiveKey('APIKEY')).toBe(true)
|
||||||
|
expect(isSensitiveKey('API_KEY')).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should match token variations', () => {
|
||||||
|
expect(isSensitiveKey('access_token')).toBe(true)
|
||||||
|
expect(isSensitiveKey('refresh_token')).toBe(true)
|
||||||
|
expect(isSensitiveKey('auth_token')).toBe(true)
|
||||||
|
expect(isSensitiveKey('accessToken')).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should match secret variations', () => {
|
||||||
|
expect(isSensitiveKey('client_secret')).toBe(true)
|
||||||
|
expect(isSensitiveKey('clientSecret')).toBe(true)
|
||||||
|
expect(isSensitiveKey('secret')).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should match other sensitive keys', () => {
|
||||||
|
expect(isSensitiveKey('private_key')).toBe(true)
|
||||||
|
expect(isSensitiveKey('authorization')).toBe(true)
|
||||||
|
expect(isSensitiveKey('bearer')).toBe(true)
|
||||||
|
expect(isSensitiveKey('private')).toBe(true)
|
||||||
|
expect(isSensitiveKey('auth')).toBe(true)
|
||||||
|
expect(isSensitiveKey('password')).toBe(true)
|
||||||
|
expect(isSensitiveKey('credential')).toBe(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('suffix matches', () => {
|
||||||
|
it.concurrent('should match keys ending in secret', () => {
|
||||||
|
expect(isSensitiveKey('clientSecret')).toBe(true)
|
||||||
|
expect(isSensitiveKey('appSecret')).toBe(true)
|
||||||
|
expect(isSensitiveKey('mySecret')).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should match keys ending in password', () => {
|
||||||
|
expect(isSensitiveKey('userPassword')).toBe(true)
|
||||||
|
expect(isSensitiveKey('dbPassword')).toBe(true)
|
||||||
|
expect(isSensitiveKey('adminPassword')).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should match keys ending in token', () => {
|
||||||
|
expect(isSensitiveKey('accessToken')).toBe(true)
|
||||||
|
expect(isSensitiveKey('refreshToken')).toBe(true)
|
||||||
|
expect(isSensitiveKey('bearerToken')).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should match keys ending in credential', () => {
|
||||||
|
expect(isSensitiveKey('userCredential')).toBe(true)
|
||||||
|
expect(isSensitiveKey('dbCredential')).toBe(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('non-sensitive keys (no false positives)', () => {
|
||||||
|
it.concurrent('should not match keys with sensitive words as prefix only', () => {
|
||||||
|
expect(isSensitiveKey('tokenCount')).toBe(false)
|
||||||
|
expect(isSensitiveKey('tokenizer')).toBe(false)
|
||||||
|
expect(isSensitiveKey('secretKey')).toBe(false)
|
||||||
|
expect(isSensitiveKey('passwordStrength')).toBe(false)
|
||||||
|
expect(isSensitiveKey('authMethod')).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should match keys ending with sensitive words (intentional)', () => {
|
||||||
|
expect(isSensitiveKey('hasSecret')).toBe(true)
|
||||||
|
expect(isSensitiveKey('userPassword')).toBe(true)
|
||||||
|
expect(isSensitiveKey('sessionToken')).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should not match normal field names', () => {
|
||||||
|
expect(isSensitiveKey('name')).toBe(false)
|
||||||
|
expect(isSensitiveKey('email')).toBe(false)
|
||||||
|
expect(isSensitiveKey('id')).toBe(false)
|
||||||
|
expect(isSensitiveKey('value')).toBe(false)
|
||||||
|
expect(isSensitiveKey('data')).toBe(false)
|
||||||
|
expect(isSensitiveKey('count')).toBe(false)
|
||||||
|
expect(isSensitiveKey('status')).toBe(false)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('redactSensitiveValues', () => {
|
||||||
|
it.concurrent('should redact Bearer tokens', () => {
|
||||||
|
const input = 'Authorization: Bearer abc123xyz456'
|
||||||
|
const result = redactSensitiveValues(input)
|
||||||
|
expect(result).toBe('Authorization: Bearer [REDACTED]')
|
||||||
|
expect(result).not.toContain('abc123xyz456')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should redact Basic auth', () => {
|
||||||
|
const input = 'Authorization: Basic dXNlcjpwYXNz'
|
||||||
|
const result = redactSensitiveValues(input)
|
||||||
|
expect(result).toBe('Authorization: Basic [REDACTED]')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should redact API key prefixes', () => {
|
||||||
|
const input = 'Using key sk-1234567890abcdefghijklmnop'
|
||||||
|
const result = redactSensitiveValues(input)
|
||||||
|
expect(result).toContain('[REDACTED]')
|
||||||
|
expect(result).not.toContain('sk-1234567890abcdefghijklmnop')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should redact JSON-style password fields', () => {
|
||||||
|
const input = 'password: "mysecretpass123"'
|
||||||
|
const result = redactSensitiveValues(input)
|
||||||
|
expect(result).toContain('[REDACTED]')
|
||||||
|
expect(result).not.toContain('mysecretpass123')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should redact JSON-style token fields', () => {
|
||||||
|
const input = 'token: "tokenvalue123"'
|
||||||
|
const result = redactSensitiveValues(input)
|
||||||
|
expect(result).toContain('[REDACTED]')
|
||||||
|
expect(result).not.toContain('tokenvalue123')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should redact JSON-style api_key fields', () => {
|
||||||
|
const input = 'api_key: "key123456"'
|
||||||
|
const result = redactSensitiveValues(input)
|
||||||
|
expect(result).toContain('[REDACTED]')
|
||||||
|
expect(result).not.toContain('key123456')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should not modify safe strings', () => {
|
||||||
|
const input = 'This is a normal string with no secrets'
|
||||||
|
const result = redactSensitiveValues(input)
|
||||||
|
expect(result).toBe(input)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should handle empty strings', () => {
|
||||||
|
expect(redactSensitiveValues('')).toBe('')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should handle null/undefined gracefully', () => {
|
||||||
|
expect(redactSensitiveValues(null as any)).toBe(null)
|
||||||
|
expect(redactSensitiveValues(undefined as any)).toBe(undefined)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('redactApiKeys', () => {
|
||||||
|
describe('object redaction', () => {
|
||||||
|
it.concurrent('should redact sensitive keys in flat objects', () => {
|
||||||
|
const obj = {
|
||||||
|
apiKey: 'secret-key',
|
||||||
|
api_key: 'another-secret',
|
||||||
|
access_token: 'token-value',
|
||||||
|
secret: 'secret-value',
|
||||||
|
password: 'password-value',
|
||||||
|
normalField: 'normal-value',
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = redactApiKeys(obj)
|
||||||
|
|
||||||
|
expect(result.apiKey).toBe('[REDACTED]')
|
||||||
|
expect(result.api_key).toBe('[REDACTED]')
|
||||||
|
expect(result.access_token).toBe('[REDACTED]')
|
||||||
|
expect(result.secret).toBe('[REDACTED]')
|
||||||
|
expect(result.password).toBe('[REDACTED]')
|
||||||
|
expect(result.normalField).toBe('normal-value')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should redact sensitive keys in nested objects', () => {
|
||||||
|
const obj = {
|
||||||
|
config: {
|
||||||
|
apiKey: 'secret-key',
|
||||||
|
normalField: 'normal-value',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = redactApiKeys(obj)
|
||||||
|
|
||||||
|
expect(result.config.apiKey).toBe('[REDACTED]')
|
||||||
|
expect(result.config.normalField).toBe('normal-value')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should redact sensitive keys in arrays', () => {
|
||||||
|
const arr = [{ apiKey: 'secret-key-1' }, { apiKey: 'secret-key-2' }]
|
||||||
|
|
||||||
|
const result = redactApiKeys(arr)
|
||||||
|
|
||||||
|
expect(result[0].apiKey).toBe('[REDACTED]')
|
||||||
|
expect(result[1].apiKey).toBe('[REDACTED]')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should handle deeply nested structures', () => {
|
||||||
|
const obj = {
|
||||||
|
users: [
|
||||||
|
{
|
||||||
|
name: 'John',
|
||||||
|
credentials: {
|
||||||
|
apiKey: 'secret-key',
|
||||||
|
username: 'john_doe',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
config: {
|
||||||
|
database: {
|
||||||
|
password: 'db-password',
|
||||||
|
host: 'localhost',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = redactApiKeys(obj)
|
||||||
|
|
||||||
|
expect(result.users[0].name).toBe('John')
|
||||||
|
expect(result.users[0].credentials.apiKey).toBe('[REDACTED]')
|
||||||
|
expect(result.users[0].credentials.username).toBe('john_doe')
|
||||||
|
expect(result.config.database.password).toBe('[REDACTED]')
|
||||||
|
expect(result.config.database.host).toBe('localhost')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('primitive handling', () => {
|
||||||
|
it.concurrent('should return primitives unchanged', () => {
|
||||||
|
expect(redactApiKeys('string')).toBe('string')
|
||||||
|
expect(redactApiKeys(123)).toBe(123)
|
||||||
|
expect(redactApiKeys(true)).toBe(true)
|
||||||
|
expect(redactApiKeys(null)).toBe(null)
|
||||||
|
expect(redactApiKeys(undefined)).toBe(undefined)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('no false positives', () => {
|
||||||
|
it.concurrent('should not redact keys with sensitive words as prefix only', () => {
|
||||||
|
const obj = {
|
||||||
|
tokenCount: 100,
|
||||||
|
secretKey: 'not-actually-secret',
|
||||||
|
passwordStrength: 'strong',
|
||||||
|
authMethod: 'oauth',
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = redactApiKeys(obj)
|
||||||
|
|
||||||
|
expect(result.tokenCount).toBe(100)
|
||||||
|
expect(result.secretKey).toBe('not-actually-secret')
|
||||||
|
expect(result.passwordStrength).toBe('strong')
|
||||||
|
expect(result.authMethod).toBe('oauth')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('sanitizeForLogging', () => {
|
||||||
|
it.concurrent('should truncate long strings', () => {
|
||||||
|
const longString = 'a'.repeat(200)
|
||||||
|
const result = sanitizeForLogging(longString, 50)
|
||||||
|
expect(result.length).toBe(50)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should use default max length of 100', () => {
|
||||||
|
const longString = 'a'.repeat(200)
|
||||||
|
const result = sanitizeForLogging(longString)
|
||||||
|
expect(result.length).toBe(100)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should redact sensitive patterns', () => {
|
||||||
|
const input = 'Bearer abc123xyz456'
|
||||||
|
const result = sanitizeForLogging(input)
|
||||||
|
expect(result).toContain('[REDACTED]')
|
||||||
|
expect(result).not.toContain('abc123xyz456')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should handle empty strings', () => {
|
||||||
|
expect(sanitizeForLogging('')).toBe('')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should not modify safe short strings', () => {
|
||||||
|
const input = 'Safe string'
|
||||||
|
const result = sanitizeForLogging(input)
|
||||||
|
expect(result).toBe(input)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('sanitizeEventData', () => {
|
||||||
|
describe('object sanitization', () => {
|
||||||
|
it.concurrent('should remove sensitive keys entirely', () => {
|
||||||
|
const event = {
|
||||||
|
action: 'login',
|
||||||
|
apiKey: 'secret-key',
|
||||||
|
password: 'secret-pass',
|
||||||
|
userId: '123',
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = sanitizeEventData(event)
|
||||||
|
|
||||||
|
expect(result.action).toBe('login')
|
||||||
|
expect(result.userId).toBe('123')
|
||||||
|
expect(result).not.toHaveProperty('apiKey')
|
||||||
|
expect(result).not.toHaveProperty('password')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should redact sensitive patterns in string values', () => {
|
||||||
|
const event = {
|
||||||
|
message: 'Auth: Bearer abc123token',
|
||||||
|
normal: 'normal value',
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = sanitizeEventData(event)
|
||||||
|
|
||||||
|
expect(result.message).toContain('[REDACTED]')
|
||||||
|
expect(result.message).not.toContain('abc123token')
|
||||||
|
expect(result.normal).toBe('normal value')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should handle nested objects', () => {
|
||||||
|
const event = {
|
||||||
|
user: {
|
||||||
|
id: '123',
|
||||||
|
accessToken: 'secret-token',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = sanitizeEventData(event)
|
||||||
|
|
||||||
|
expect(result.user.id).toBe('123')
|
||||||
|
expect(result.user).not.toHaveProperty('accessToken')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should handle arrays', () => {
|
||||||
|
const event = {
|
||||||
|
items: [
|
||||||
|
{ id: 1, apiKey: 'key1' },
|
||||||
|
{ id: 2, apiKey: 'key2' },
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = sanitizeEventData(event)
|
||||||
|
|
||||||
|
expect(result.items[0].id).toBe(1)
|
||||||
|
expect(result.items[0]).not.toHaveProperty('apiKey')
|
||||||
|
expect(result.items[1].id).toBe(2)
|
||||||
|
expect(result.items[1]).not.toHaveProperty('apiKey')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('primitive handling', () => {
|
||||||
|
it.concurrent('should return primitives appropriately', () => {
|
||||||
|
expect(sanitizeEventData(null)).toBe(null)
|
||||||
|
expect(sanitizeEventData(undefined)).toBe(undefined)
|
||||||
|
expect(sanitizeEventData(123)).toBe(123)
|
||||||
|
expect(sanitizeEventData(true)).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should redact sensitive patterns in top-level strings', () => {
|
||||||
|
const result = sanitizeEventData('Bearer secrettoken123')
|
||||||
|
expect(result).toContain('[REDACTED]')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.concurrent('should not redact normal strings', () => {
|
||||||
|
const result = sanitizeEventData('normal string')
|
||||||
|
expect(result).toBe('normal string')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('no false positives', () => {
|
||||||
|
it.concurrent('should not remove keys with sensitive words in middle', () => {
|
||||||
|
const event = {
|
||||||
|
tokenCount: 500,
|
||||||
|
isAuthenticated: true,
|
||||||
|
hasSecretFeature: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = sanitizeEventData(event)
|
||||||
|
|
||||||
|
expect(result.tokenCount).toBe(500)
|
||||||
|
expect(result.isAuthenticated).toBe(true)
|
||||||
|
expect(result.hasSecretFeature).toBe(false)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -1,28 +1,122 @@
|
|||||||
/**
|
/**
|
||||||
* Recursively redacts API keys in an object
|
* Centralized redaction utilities for sensitive data
|
||||||
* @param obj The object to redact API keys from
|
|
||||||
* @returns A new object with API keys redacted
|
|
||||||
*/
|
*/
|
||||||
export const redactApiKeys = (obj: any): any => {
|
|
||||||
if (!obj || typeof obj !== 'object') {
|
/** Standard marker used for all redacted values */
|
||||||
|
export const REDACTED_MARKER = '[REDACTED]'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Patterns for sensitive key names (case-insensitive matching)
|
||||||
|
* These patterns match common naming conventions for sensitive data
|
||||||
|
*/
|
||||||
|
const SENSITIVE_KEY_PATTERNS: RegExp[] = [
|
||||||
|
/^api[_-]?key$/i,
|
||||||
|
/^access[_-]?token$/i,
|
||||||
|
/^refresh[_-]?token$/i,
|
||||||
|
/^client[_-]?secret$/i,
|
||||||
|
/^private[_-]?key$/i,
|
||||||
|
/^auth[_-]?token$/i,
|
||||||
|
/^.*secret$/i,
|
||||||
|
/^.*password$/i,
|
||||||
|
/^.*token$/i,
|
||||||
|
/^.*credential$/i,
|
||||||
|
/^authorization$/i,
|
||||||
|
/^bearer$/i,
|
||||||
|
/^private$/i,
|
||||||
|
/^auth$/i,
|
||||||
|
]
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Patterns for sensitive values in strings (for redacting values, not keys)
|
||||||
|
* Each pattern has a replacement function
|
||||||
|
*/
|
||||||
|
const SENSITIVE_VALUE_PATTERNS: Array<{
|
||||||
|
pattern: RegExp
|
||||||
|
replacement: string
|
||||||
|
}> = [
|
||||||
|
// Bearer tokens
|
||||||
|
{
|
||||||
|
pattern: /Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi,
|
||||||
|
replacement: `Bearer ${REDACTED_MARKER}`,
|
||||||
|
},
|
||||||
|
// Basic auth
|
||||||
|
{
|
||||||
|
pattern: /Basic\s+[A-Za-z0-9+/]+=*/gi,
|
||||||
|
replacement: `Basic ${REDACTED_MARKER}`,
|
||||||
|
},
|
||||||
|
// API keys that look like sk-..., pk-..., etc.
|
||||||
|
{
|
||||||
|
pattern: /\b(sk|pk|api|key)[_-][A-Za-z0-9\-._]{20,}\b/gi,
|
||||||
|
replacement: REDACTED_MARKER,
|
||||||
|
},
|
||||||
|
// JSON-style password fields: password: "value" or password: 'value'
|
||||||
|
{
|
||||||
|
pattern: /password['":\s]*['"][^'"]+['"]/gi,
|
||||||
|
replacement: `password: "${REDACTED_MARKER}"`,
|
||||||
|
},
|
||||||
|
// JSON-style token fields: token: "value" or token: 'value'
|
||||||
|
{
|
||||||
|
pattern: /token['":\s]*['"][^'"]+['"]/gi,
|
||||||
|
replacement: `token: "${REDACTED_MARKER}"`,
|
||||||
|
},
|
||||||
|
// JSON-style api_key fields: api_key: "value" or api-key: "value"
|
||||||
|
{
|
||||||
|
pattern: /api[_-]?key['":\s]*['"][^'"]+['"]/gi,
|
||||||
|
replacement: `api_key: "${REDACTED_MARKER}"`,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks if a key name matches any sensitive pattern
|
||||||
|
* @param key - The key name to check
|
||||||
|
* @returns True if the key is considered sensitive
|
||||||
|
*/
|
||||||
|
export function isSensitiveKey(key: string): boolean {
|
||||||
|
const lowerKey = key.toLowerCase()
|
||||||
|
return SENSITIVE_KEY_PATTERNS.some((pattern) => pattern.test(lowerKey))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Redacts sensitive patterns from a string value
|
||||||
|
* @param value - The string to redact
|
||||||
|
* @returns The string with sensitive patterns redacted
|
||||||
|
*/
|
||||||
|
export function redactSensitiveValues(value: string): string {
|
||||||
|
if (!value || typeof value !== 'string') {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = value
|
||||||
|
for (const { pattern, replacement } of SENSITIVE_VALUE_PATTERNS) {
|
||||||
|
result = result.replace(pattern, replacement)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recursively redacts sensitive data (API keys, passwords, tokens, etc.) from an object
|
||||||
|
*
|
||||||
|
* @param obj - The object to redact sensitive data from
|
||||||
|
* @returns A new object with sensitive data redacted
|
||||||
|
*/
|
||||||
|
export function redactApiKeys(obj: any): any {
|
||||||
|
if (obj === null || obj === undefined) {
|
||||||
|
return obj
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof obj !== 'object') {
|
||||||
return obj
|
return obj
|
||||||
}
|
}
|
||||||
|
|
||||||
if (Array.isArray(obj)) {
|
if (Array.isArray(obj)) {
|
||||||
return obj.map(redactApiKeys)
|
return obj.map((item) => redactApiKeys(item))
|
||||||
}
|
}
|
||||||
|
|
||||||
const result: Record<string, any> = {}
|
const result: Record<string, any> = {}
|
||||||
|
|
||||||
for (const [key, value] of Object.entries(obj)) {
|
for (const [key, value] of Object.entries(obj)) {
|
||||||
if (
|
if (isSensitiveKey(key)) {
|
||||||
key.toLowerCase() === 'apikey' ||
|
result[key] = REDACTED_MARKER
|
||||||
key.toLowerCase() === 'api_key' ||
|
|
||||||
key.toLowerCase() === 'access_token' ||
|
|
||||||
/\bsecret\b/i.test(key.toLowerCase()) ||
|
|
||||||
/\bpassword\b/i.test(key.toLowerCase())
|
|
||||||
) {
|
|
||||||
result[key] = '***REDACTED***'
|
|
||||||
} else if (typeof value === 'object' && value !== null) {
|
} else if (typeof value === 'object' && value !== null) {
|
||||||
result[key] = redactApiKeys(value)
|
result[key] = redactApiKeys(value)
|
||||||
} else {
|
} else {
|
||||||
@@ -32,3 +126,64 @@ export const redactApiKeys = (obj: any): any => {
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sanitizes a string for safe logging by truncating and redacting sensitive patterns
|
||||||
|
*
|
||||||
|
* @param value - The string to sanitize
|
||||||
|
* @param maxLength - Maximum length of the output (default: 100)
|
||||||
|
* @returns The sanitized string
|
||||||
|
*/
|
||||||
|
export function sanitizeForLogging(value: string, maxLength = 100): string {
|
||||||
|
if (!value) return ''
|
||||||
|
|
||||||
|
let sanitized = value.substring(0, maxLength)
|
||||||
|
|
||||||
|
sanitized = redactSensitiveValues(sanitized)
|
||||||
|
|
||||||
|
return sanitized
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sanitizes event data for error reporting/analytics
|
||||||
|
*
|
||||||
|
* @param event - The event data to sanitize
|
||||||
|
* @returns Sanitized event data safe for external reporting
|
||||||
|
*/
|
||||||
|
export function sanitizeEventData(event: any): any {
|
||||||
|
if (event === null || event === undefined) {
|
||||||
|
return event
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof event === 'string') {
|
||||||
|
return redactSensitiveValues(event)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof event !== 'object') {
|
||||||
|
return event
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Array.isArray(event)) {
|
||||||
|
return event.map((item) => sanitizeEventData(item))
|
||||||
|
}
|
||||||
|
|
||||||
|
const sanitized: Record<string, unknown> = {}
|
||||||
|
|
||||||
|
for (const [key, value] of Object.entries(event)) {
|
||||||
|
if (isSensitiveKey(key)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof value === 'string') {
|
||||||
|
sanitized[key] = redactSensitiveValues(value)
|
||||||
|
} else if (Array.isArray(value)) {
|
||||||
|
sanitized[key] = value.map((v) => sanitizeEventData(v))
|
||||||
|
} else if (value && typeof value === 'object') {
|
||||||
|
sanitized[key] = sanitizeEventData(value)
|
||||||
|
} else {
|
||||||
|
sanitized[key] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return sanitized
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||||
import { getRotatingApiKey } from '@/lib/core/config/api-keys'
|
import { getRotatingApiKey } from '@/lib/core/config/api-keys'
|
||||||
import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption'
|
import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption'
|
||||||
import { redactApiKeys } from '@/lib/core/security/redaction'
|
|
||||||
import { cn } from '@/lib/core/utils/cn'
|
import { cn } from '@/lib/core/utils/cn'
|
||||||
import {
|
import {
|
||||||
formatDate,
|
formatDate,
|
||||||
@@ -229,86 +228,6 @@ describe('getTimezoneAbbreviation', () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('redactApiKeys', () => {
|
|
||||||
it.concurrent('should redact API keys in objects', () => {
|
|
||||||
const obj = {
|
|
||||||
apiKey: 'secret-key',
|
|
||||||
api_key: 'another-secret',
|
|
||||||
access_token: 'token-value',
|
|
||||||
secret: 'secret-value',
|
|
||||||
password: 'password-value',
|
|
||||||
normalField: 'normal-value',
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = redactApiKeys(obj)
|
|
||||||
|
|
||||||
expect(result.apiKey).toBe('***REDACTED***')
|
|
||||||
expect(result.api_key).toBe('***REDACTED***')
|
|
||||||
expect(result.access_token).toBe('***REDACTED***')
|
|
||||||
expect(result.secret).toBe('***REDACTED***')
|
|
||||||
expect(result.password).toBe('***REDACTED***')
|
|
||||||
expect(result.normalField).toBe('normal-value')
|
|
||||||
})
|
|
||||||
|
|
||||||
it.concurrent('should redact API keys in nested objects', () => {
|
|
||||||
const obj = {
|
|
||||||
config: {
|
|
||||||
apiKey: 'secret-key',
|
|
||||||
normalField: 'normal-value',
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = redactApiKeys(obj)
|
|
||||||
|
|
||||||
expect(result.config.apiKey).toBe('***REDACTED***')
|
|
||||||
expect(result.config.normalField).toBe('normal-value')
|
|
||||||
})
|
|
||||||
|
|
||||||
it.concurrent('should redact API keys in arrays', () => {
|
|
||||||
const arr = [{ apiKey: 'secret-key-1' }, { apiKey: 'secret-key-2' }]
|
|
||||||
|
|
||||||
const result = redactApiKeys(arr)
|
|
||||||
|
|
||||||
expect(result[0].apiKey).toBe('***REDACTED***')
|
|
||||||
expect(result[1].apiKey).toBe('***REDACTED***')
|
|
||||||
})
|
|
||||||
|
|
||||||
it.concurrent('should handle primitive values', () => {
|
|
||||||
expect(redactApiKeys('string')).toBe('string')
|
|
||||||
expect(redactApiKeys(123)).toBe(123)
|
|
||||||
expect(redactApiKeys(null)).toBe(null)
|
|
||||||
expect(redactApiKeys(undefined)).toBe(undefined)
|
|
||||||
})
|
|
||||||
|
|
||||||
it.concurrent('should handle complex nested structures', () => {
|
|
||||||
const obj = {
|
|
||||||
users: [
|
|
||||||
{
|
|
||||||
name: 'John',
|
|
||||||
credentials: {
|
|
||||||
apiKey: 'secret-key',
|
|
||||||
username: 'john_doe',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
config: {
|
|
||||||
database: {
|
|
||||||
password: 'db-password',
|
|
||||||
host: 'localhost',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = redactApiKeys(obj)
|
|
||||||
|
|
||||||
expect(result.users[0].name).toBe('John')
|
|
||||||
expect(result.users[0].credentials.apiKey).toBe('***REDACTED***')
|
|
||||||
expect(result.users[0].credentials.username).toBe('john_doe')
|
|
||||||
expect(result.config.database.password).toBe('***REDACTED***')
|
|
||||||
expect(result.config.database.host).toBe('localhost')
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
describe('validateName', () => {
|
describe('validateName', () => {
|
||||||
it.concurrent('should remove invalid characters', () => {
|
it.concurrent('should remove invalid characters', () => {
|
||||||
const result = validateName('test@#$%name')
|
const result = validateName('test@#$%name')
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
|
|||||||
return { entries: state.entries }
|
return { entries: state.entries }
|
||||||
}
|
}
|
||||||
|
|
||||||
// Redact API keys from output
|
// Redact API keys from output and input
|
||||||
const redactedEntry = { ...entry }
|
const redactedEntry = { ...entry }
|
||||||
if (
|
if (
|
||||||
!isStreamingOutput(entry.output) &&
|
!isStreamingOutput(entry.output) &&
|
||||||
@@ -89,6 +89,9 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
|
|||||||
) {
|
) {
|
||||||
redactedEntry.output = redactApiKeys(redactedEntry.output)
|
redactedEntry.output = redactApiKeys(redactedEntry.output)
|
||||||
}
|
}
|
||||||
|
if (redactedEntry.input && typeof redactedEntry.input === 'object') {
|
||||||
|
redactedEntry.input = redactApiKeys(redactedEntry.input)
|
||||||
|
}
|
||||||
|
|
||||||
// Create new entry with ID and timestamp
|
// Create new entry with ID and timestamp
|
||||||
const newEntry: ConsoleEntry = {
|
const newEntry: ConsoleEntry = {
|
||||||
@@ -275,12 +278,17 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (update.replaceOutput !== undefined) {
|
if (update.replaceOutput !== undefined) {
|
||||||
updatedEntry.output = update.replaceOutput
|
updatedEntry.output =
|
||||||
|
typeof update.replaceOutput === 'object' && update.replaceOutput !== null
|
||||||
|
? redactApiKeys(update.replaceOutput)
|
||||||
|
: update.replaceOutput
|
||||||
} else if (update.output !== undefined) {
|
} else if (update.output !== undefined) {
|
||||||
updatedEntry.output = {
|
const mergedOutput = {
|
||||||
...(entry.output || {}),
|
...(entry.output || {}),
|
||||||
...update.output,
|
...update.output,
|
||||||
}
|
}
|
||||||
|
updatedEntry.output =
|
||||||
|
typeof mergedOutput === 'object' ? redactApiKeys(mergedOutput) : mergedOutput
|
||||||
}
|
}
|
||||||
|
|
||||||
if (update.error !== undefined) {
|
if (update.error !== undefined) {
|
||||||
@@ -304,7 +312,10 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (update.input !== undefined) {
|
if (update.input !== undefined) {
|
||||||
updatedEntry.input = update.input
|
updatedEntry.input =
|
||||||
|
typeof update.input === 'object' && update.input !== null
|
||||||
|
? redactApiKeys(update.input)
|
||||||
|
: update.input
|
||||||
}
|
}
|
||||||
|
|
||||||
return updatedEntry
|
return updatedEntry
|
||||||
|
|||||||
Reference in New Issue
Block a user