mirror of
https://github.com/simstudioai/sim.git
synced 2026-01-08 22:48:14 -05:00
fix(redaction): consolidate redaction utils, apply them to inputs and outputs before persisting logs (#2478)
* fix(redaction): consolidate redaction utils, apply them to inputs and outputs before persisting logs * added testing utils
This commit is contained in:
@@ -2,6 +2,7 @@ import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { z } from 'zod'
|
||||
import { auth } from '@/lib/auth'
|
||||
import { env } from '@/lib/core/config/env'
|
||||
import { REDACTED_MARKER } from '@/lib/core/security/redaction'
|
||||
import { createLogger } from '@/lib/logs/console/logger'
|
||||
|
||||
const logger = createLogger('SSO-Register')
|
||||
@@ -236,13 +237,13 @@ export async function POST(request: NextRequest) {
|
||||
oidcConfig: providerConfig.oidcConfig
|
||||
? {
|
||||
...providerConfig.oidcConfig,
|
||||
clientSecret: '[REDACTED]',
|
||||
clientSecret: REDACTED_MARKER,
|
||||
}
|
||||
: undefined,
|
||||
samlConfig: providerConfig.samlConfig
|
||||
? {
|
||||
...providerConfig.samlConfig,
|
||||
cert: '[REDACTED]',
|
||||
cert: REDACTED_MARKER,
|
||||
}
|
||||
: undefined,
|
||||
},
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { z } from 'zod'
|
||||
import { env } from '@/lib/core/config/env'
|
||||
import { isSensitiveKey, REDACTED_MARKER } from '@/lib/core/security/redaction'
|
||||
import { createLogger } from '@/lib/logs/console/logger'
|
||||
import { ensureZodObject, normalizeUrl } from '@/app/api/tools/stagehand/utils'
|
||||
|
||||
@@ -188,7 +189,7 @@ export async function POST(request: NextRequest) {
|
||||
|
||||
if (variablesObject && Object.keys(variablesObject).length > 0) {
|
||||
const safeVarKeys = Object.keys(variablesObject).map((key) => {
|
||||
return key.toLowerCase().includes('password') ? `${key}: [REDACTED]` : key
|
||||
return isSensitiveKey(key) ? `${key}: ${REDACTED_MARKER}` : key
|
||||
})
|
||||
logger.info('Variables available for task', { variables: safeVarKeys })
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
*/
|
||||
|
||||
import { env } from './lib/core/config/env'
|
||||
import { sanitizeEventData } from './lib/core/security/redaction'
|
||||
|
||||
if (typeof window !== 'undefined') {
|
||||
const TELEMETRY_STATUS_KEY = 'simstudio-telemetry-status'
|
||||
@@ -41,37 +42,6 @@ if (typeof window !== 'undefined') {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize event data to remove sensitive information
|
||||
*/
|
||||
function sanitizeEvent(event: any): any {
|
||||
const patterns = ['password', 'token', 'secret', 'key', 'auth', 'credential', 'private']
|
||||
const sensitiveRe = new RegExp(patterns.join('|'), 'i')
|
||||
|
||||
const scrubString = (s: string) => (s && sensitiveRe.test(s) ? '[redacted]' : s)
|
||||
|
||||
if (event == null) return event
|
||||
if (typeof event === 'string') return scrubString(event)
|
||||
if (typeof event !== 'object') return event
|
||||
|
||||
if (Array.isArray(event)) {
|
||||
return event.map((item) => sanitizeEvent(item))
|
||||
}
|
||||
|
||||
const sanitized: Record<string, unknown> = {}
|
||||
for (const [key, value] of Object.entries(event)) {
|
||||
const lowerKey = key.toLowerCase()
|
||||
if (patterns.some((p) => lowerKey.includes(p))) continue
|
||||
|
||||
if (typeof value === 'string') sanitized[key] = scrubString(value)
|
||||
else if (Array.isArray(value)) sanitized[key] = value.map((v) => sanitizeEvent(v))
|
||||
else if (value && typeof value === 'object') sanitized[key] = sanitizeEvent(value)
|
||||
else sanitized[key] = value
|
||||
}
|
||||
|
||||
return sanitized
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush batch of events to server
|
||||
*/
|
||||
@@ -84,7 +54,7 @@ if (typeof window !== 'undefined') {
|
||||
batchTimer = null
|
||||
}
|
||||
|
||||
const sanitizedBatch = batch.map(sanitizeEvent)
|
||||
const sanitizedBatch = batch.map(sanitizeEventData)
|
||||
|
||||
const payload = JSON.stringify({
|
||||
category: 'batch',
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
createPinnedUrl,
|
||||
sanitizeForLogging,
|
||||
validateAlphanumericId,
|
||||
validateEnum,
|
||||
validateFileExtension,
|
||||
@@ -11,6 +10,7 @@ import {
|
||||
validateUrlWithDNS,
|
||||
validateUUID,
|
||||
} from '@/lib/core/security/input-validation'
|
||||
import { sanitizeForLogging } from '@/lib/core/security/redaction'
|
||||
|
||||
describe('validatePathSegment', () => {
|
||||
describe('valid inputs', () => {
|
||||
|
||||
@@ -556,29 +556,6 @@ export function validateFileExtension(
|
||||
return { isValid: true, sanitized: normalizedExt }
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitizes a string for safe logging (removes potential sensitive data patterns)
|
||||
*
|
||||
* @param value - The value to sanitize
|
||||
* @param maxLength - Maximum length to return (default: 100)
|
||||
* @returns Sanitized string safe for logging
|
||||
*/
|
||||
export function sanitizeForLogging(value: string, maxLength = 100): string {
|
||||
if (!value) return ''
|
||||
|
||||
// Truncate long values
|
||||
let sanitized = value.substring(0, maxLength)
|
||||
|
||||
// Mask common sensitive patterns
|
||||
sanitized = sanitized
|
||||
.replace(/Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi, 'Bearer [REDACTED]')
|
||||
.replace(/password['":\s]*['"]\w+['"]/gi, 'password: "[REDACTED]"')
|
||||
.replace(/token['":\s]*['"]\w+['"]/gi, 'token: "[REDACTED]"')
|
||||
.replace(/api[_-]?key['":\s]*['"]\w+['"]/gi, 'api_key: "[REDACTED]"')
|
||||
|
||||
return sanitized
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates Microsoft Graph API resource IDs
|
||||
*
|
||||
|
||||
391
apps/sim/lib/core/security/redaction.test.ts
Normal file
391
apps/sim/lib/core/security/redaction.test.ts
Normal file
@@ -0,0 +1,391 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
isSensitiveKey,
|
||||
REDACTED_MARKER,
|
||||
redactApiKeys,
|
||||
redactSensitiveValues,
|
||||
sanitizeEventData,
|
||||
sanitizeForLogging,
|
||||
} from './redaction'
|
||||
|
||||
describe('REDACTED_MARKER', () => {
|
||||
it.concurrent('should be the standard marker', () => {
|
||||
expect(REDACTED_MARKER).toBe('[REDACTED]')
|
||||
})
|
||||
})
|
||||
|
||||
describe('isSensitiveKey', () => {
|
||||
describe('exact matches', () => {
|
||||
it.concurrent('should match apiKey variations', () => {
|
||||
expect(isSensitiveKey('apiKey')).toBe(true)
|
||||
expect(isSensitiveKey('api_key')).toBe(true)
|
||||
expect(isSensitiveKey('api-key')).toBe(true)
|
||||
expect(isSensitiveKey('APIKEY')).toBe(true)
|
||||
expect(isSensitiveKey('API_KEY')).toBe(true)
|
||||
})
|
||||
|
||||
it.concurrent('should match token variations', () => {
|
||||
expect(isSensitiveKey('access_token')).toBe(true)
|
||||
expect(isSensitiveKey('refresh_token')).toBe(true)
|
||||
expect(isSensitiveKey('auth_token')).toBe(true)
|
||||
expect(isSensitiveKey('accessToken')).toBe(true)
|
||||
})
|
||||
|
||||
it.concurrent('should match secret variations', () => {
|
||||
expect(isSensitiveKey('client_secret')).toBe(true)
|
||||
expect(isSensitiveKey('clientSecret')).toBe(true)
|
||||
expect(isSensitiveKey('secret')).toBe(true)
|
||||
})
|
||||
|
||||
it.concurrent('should match other sensitive keys', () => {
|
||||
expect(isSensitiveKey('private_key')).toBe(true)
|
||||
expect(isSensitiveKey('authorization')).toBe(true)
|
||||
expect(isSensitiveKey('bearer')).toBe(true)
|
||||
expect(isSensitiveKey('private')).toBe(true)
|
||||
expect(isSensitiveKey('auth')).toBe(true)
|
||||
expect(isSensitiveKey('password')).toBe(true)
|
||||
expect(isSensitiveKey('credential')).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('suffix matches', () => {
|
||||
it.concurrent('should match keys ending in secret', () => {
|
||||
expect(isSensitiveKey('clientSecret')).toBe(true)
|
||||
expect(isSensitiveKey('appSecret')).toBe(true)
|
||||
expect(isSensitiveKey('mySecret')).toBe(true)
|
||||
})
|
||||
|
||||
it.concurrent('should match keys ending in password', () => {
|
||||
expect(isSensitiveKey('userPassword')).toBe(true)
|
||||
expect(isSensitiveKey('dbPassword')).toBe(true)
|
||||
expect(isSensitiveKey('adminPassword')).toBe(true)
|
||||
})
|
||||
|
||||
it.concurrent('should match keys ending in token', () => {
|
||||
expect(isSensitiveKey('accessToken')).toBe(true)
|
||||
expect(isSensitiveKey('refreshToken')).toBe(true)
|
||||
expect(isSensitiveKey('bearerToken')).toBe(true)
|
||||
})
|
||||
|
||||
it.concurrent('should match keys ending in credential', () => {
|
||||
expect(isSensitiveKey('userCredential')).toBe(true)
|
||||
expect(isSensitiveKey('dbCredential')).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('non-sensitive keys (no false positives)', () => {
|
||||
it.concurrent('should not match keys with sensitive words as prefix only', () => {
|
||||
expect(isSensitiveKey('tokenCount')).toBe(false)
|
||||
expect(isSensitiveKey('tokenizer')).toBe(false)
|
||||
expect(isSensitiveKey('secretKey')).toBe(false)
|
||||
expect(isSensitiveKey('passwordStrength')).toBe(false)
|
||||
expect(isSensitiveKey('authMethod')).toBe(false)
|
||||
})
|
||||
|
||||
it.concurrent('should match keys ending with sensitive words (intentional)', () => {
|
||||
expect(isSensitiveKey('hasSecret')).toBe(true)
|
||||
expect(isSensitiveKey('userPassword')).toBe(true)
|
||||
expect(isSensitiveKey('sessionToken')).toBe(true)
|
||||
})
|
||||
|
||||
it.concurrent('should not match normal field names', () => {
|
||||
expect(isSensitiveKey('name')).toBe(false)
|
||||
expect(isSensitiveKey('email')).toBe(false)
|
||||
expect(isSensitiveKey('id')).toBe(false)
|
||||
expect(isSensitiveKey('value')).toBe(false)
|
||||
expect(isSensitiveKey('data')).toBe(false)
|
||||
expect(isSensitiveKey('count')).toBe(false)
|
||||
expect(isSensitiveKey('status')).toBe(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('redactSensitiveValues', () => {
|
||||
it.concurrent('should redact Bearer tokens', () => {
|
||||
const input = 'Authorization: Bearer abc123xyz456'
|
||||
const result = redactSensitiveValues(input)
|
||||
expect(result).toBe('Authorization: Bearer [REDACTED]')
|
||||
expect(result).not.toContain('abc123xyz456')
|
||||
})
|
||||
|
||||
it.concurrent('should redact Basic auth', () => {
|
||||
const input = 'Authorization: Basic dXNlcjpwYXNz'
|
||||
const result = redactSensitiveValues(input)
|
||||
expect(result).toBe('Authorization: Basic [REDACTED]')
|
||||
})
|
||||
|
||||
it.concurrent('should redact API key prefixes', () => {
|
||||
const input = 'Using key sk-1234567890abcdefghijklmnop'
|
||||
const result = redactSensitiveValues(input)
|
||||
expect(result).toContain('[REDACTED]')
|
||||
expect(result).not.toContain('sk-1234567890abcdefghijklmnop')
|
||||
})
|
||||
|
||||
it.concurrent('should redact JSON-style password fields', () => {
|
||||
const input = 'password: "mysecretpass123"'
|
||||
const result = redactSensitiveValues(input)
|
||||
expect(result).toContain('[REDACTED]')
|
||||
expect(result).not.toContain('mysecretpass123')
|
||||
})
|
||||
|
||||
it.concurrent('should redact JSON-style token fields', () => {
|
||||
const input = 'token: "tokenvalue123"'
|
||||
const result = redactSensitiveValues(input)
|
||||
expect(result).toContain('[REDACTED]')
|
||||
expect(result).not.toContain('tokenvalue123')
|
||||
})
|
||||
|
||||
it.concurrent('should redact JSON-style api_key fields', () => {
|
||||
const input = 'api_key: "key123456"'
|
||||
const result = redactSensitiveValues(input)
|
||||
expect(result).toContain('[REDACTED]')
|
||||
expect(result).not.toContain('key123456')
|
||||
})
|
||||
|
||||
it.concurrent('should not modify safe strings', () => {
|
||||
const input = 'This is a normal string with no secrets'
|
||||
const result = redactSensitiveValues(input)
|
||||
expect(result).toBe(input)
|
||||
})
|
||||
|
||||
it.concurrent('should handle empty strings', () => {
|
||||
expect(redactSensitiveValues('')).toBe('')
|
||||
})
|
||||
|
||||
it.concurrent('should handle null/undefined gracefully', () => {
|
||||
expect(redactSensitiveValues(null as any)).toBe(null)
|
||||
expect(redactSensitiveValues(undefined as any)).toBe(undefined)
|
||||
})
|
||||
})
|
||||
|
||||
describe('redactApiKeys', () => {
|
||||
describe('object redaction', () => {
|
||||
it.concurrent('should redact sensitive keys in flat objects', () => {
|
||||
const obj = {
|
||||
apiKey: 'secret-key',
|
||||
api_key: 'another-secret',
|
||||
access_token: 'token-value',
|
||||
secret: 'secret-value',
|
||||
password: 'password-value',
|
||||
normalField: 'normal-value',
|
||||
}
|
||||
|
||||
const result = redactApiKeys(obj)
|
||||
|
||||
expect(result.apiKey).toBe('[REDACTED]')
|
||||
expect(result.api_key).toBe('[REDACTED]')
|
||||
expect(result.access_token).toBe('[REDACTED]')
|
||||
expect(result.secret).toBe('[REDACTED]')
|
||||
expect(result.password).toBe('[REDACTED]')
|
||||
expect(result.normalField).toBe('normal-value')
|
||||
})
|
||||
|
||||
it.concurrent('should redact sensitive keys in nested objects', () => {
|
||||
const obj = {
|
||||
config: {
|
||||
apiKey: 'secret-key',
|
||||
normalField: 'normal-value',
|
||||
},
|
||||
}
|
||||
|
||||
const result = redactApiKeys(obj)
|
||||
|
||||
expect(result.config.apiKey).toBe('[REDACTED]')
|
||||
expect(result.config.normalField).toBe('normal-value')
|
||||
})
|
||||
|
||||
it.concurrent('should redact sensitive keys in arrays', () => {
|
||||
const arr = [{ apiKey: 'secret-key-1' }, { apiKey: 'secret-key-2' }]
|
||||
|
||||
const result = redactApiKeys(arr)
|
||||
|
||||
expect(result[0].apiKey).toBe('[REDACTED]')
|
||||
expect(result[1].apiKey).toBe('[REDACTED]')
|
||||
})
|
||||
|
||||
it.concurrent('should handle deeply nested structures', () => {
|
||||
const obj = {
|
||||
users: [
|
||||
{
|
||||
name: 'John',
|
||||
credentials: {
|
||||
apiKey: 'secret-key',
|
||||
username: 'john_doe',
|
||||
},
|
||||
},
|
||||
],
|
||||
config: {
|
||||
database: {
|
||||
password: 'db-password',
|
||||
host: 'localhost',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const result = redactApiKeys(obj)
|
||||
|
||||
expect(result.users[0].name).toBe('John')
|
||||
expect(result.users[0].credentials.apiKey).toBe('[REDACTED]')
|
||||
expect(result.users[0].credentials.username).toBe('john_doe')
|
||||
expect(result.config.database.password).toBe('[REDACTED]')
|
||||
expect(result.config.database.host).toBe('localhost')
|
||||
})
|
||||
})
|
||||
|
||||
describe('primitive handling', () => {
|
||||
it.concurrent('should return primitives unchanged', () => {
|
||||
expect(redactApiKeys('string')).toBe('string')
|
||||
expect(redactApiKeys(123)).toBe(123)
|
||||
expect(redactApiKeys(true)).toBe(true)
|
||||
expect(redactApiKeys(null)).toBe(null)
|
||||
expect(redactApiKeys(undefined)).toBe(undefined)
|
||||
})
|
||||
})
|
||||
|
||||
describe('no false positives', () => {
|
||||
it.concurrent('should not redact keys with sensitive words as prefix only', () => {
|
||||
const obj = {
|
||||
tokenCount: 100,
|
||||
secretKey: 'not-actually-secret',
|
||||
passwordStrength: 'strong',
|
||||
authMethod: 'oauth',
|
||||
}
|
||||
|
||||
const result = redactApiKeys(obj)
|
||||
|
||||
expect(result.tokenCount).toBe(100)
|
||||
expect(result.secretKey).toBe('not-actually-secret')
|
||||
expect(result.passwordStrength).toBe('strong')
|
||||
expect(result.authMethod).toBe('oauth')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('sanitizeForLogging', () => {
|
||||
it.concurrent('should truncate long strings', () => {
|
||||
const longString = 'a'.repeat(200)
|
||||
const result = sanitizeForLogging(longString, 50)
|
||||
expect(result.length).toBe(50)
|
||||
})
|
||||
|
||||
it.concurrent('should use default max length of 100', () => {
|
||||
const longString = 'a'.repeat(200)
|
||||
const result = sanitizeForLogging(longString)
|
||||
expect(result.length).toBe(100)
|
||||
})
|
||||
|
||||
it.concurrent('should redact sensitive patterns', () => {
|
||||
const input = 'Bearer abc123xyz456'
|
||||
const result = sanitizeForLogging(input)
|
||||
expect(result).toContain('[REDACTED]')
|
||||
expect(result).not.toContain('abc123xyz456')
|
||||
})
|
||||
|
||||
it.concurrent('should handle empty strings', () => {
|
||||
expect(sanitizeForLogging('')).toBe('')
|
||||
})
|
||||
|
||||
it.concurrent('should not modify safe short strings', () => {
|
||||
const input = 'Safe string'
|
||||
const result = sanitizeForLogging(input)
|
||||
expect(result).toBe(input)
|
||||
})
|
||||
})
|
||||
|
||||
describe('sanitizeEventData', () => {
|
||||
describe('object sanitization', () => {
|
||||
it.concurrent('should remove sensitive keys entirely', () => {
|
||||
const event = {
|
||||
action: 'login',
|
||||
apiKey: 'secret-key',
|
||||
password: 'secret-pass',
|
||||
userId: '123',
|
||||
}
|
||||
|
||||
const result = sanitizeEventData(event)
|
||||
|
||||
expect(result.action).toBe('login')
|
||||
expect(result.userId).toBe('123')
|
||||
expect(result).not.toHaveProperty('apiKey')
|
||||
expect(result).not.toHaveProperty('password')
|
||||
})
|
||||
|
||||
it.concurrent('should redact sensitive patterns in string values', () => {
|
||||
const event = {
|
||||
message: 'Auth: Bearer abc123token',
|
||||
normal: 'normal value',
|
||||
}
|
||||
|
||||
const result = sanitizeEventData(event)
|
||||
|
||||
expect(result.message).toContain('[REDACTED]')
|
||||
expect(result.message).not.toContain('abc123token')
|
||||
expect(result.normal).toBe('normal value')
|
||||
})
|
||||
|
||||
it.concurrent('should handle nested objects', () => {
|
||||
const event = {
|
||||
user: {
|
||||
id: '123',
|
||||
accessToken: 'secret-token',
|
||||
},
|
||||
}
|
||||
|
||||
const result = sanitizeEventData(event)
|
||||
|
||||
expect(result.user.id).toBe('123')
|
||||
expect(result.user).not.toHaveProperty('accessToken')
|
||||
})
|
||||
|
||||
it.concurrent('should handle arrays', () => {
|
||||
const event = {
|
||||
items: [
|
||||
{ id: 1, apiKey: 'key1' },
|
||||
{ id: 2, apiKey: 'key2' },
|
||||
],
|
||||
}
|
||||
|
||||
const result = sanitizeEventData(event)
|
||||
|
||||
expect(result.items[0].id).toBe(1)
|
||||
expect(result.items[0]).not.toHaveProperty('apiKey')
|
||||
expect(result.items[1].id).toBe(2)
|
||||
expect(result.items[1]).not.toHaveProperty('apiKey')
|
||||
})
|
||||
})
|
||||
|
||||
describe('primitive handling', () => {
|
||||
it.concurrent('should return primitives appropriately', () => {
|
||||
expect(sanitizeEventData(null)).toBe(null)
|
||||
expect(sanitizeEventData(undefined)).toBe(undefined)
|
||||
expect(sanitizeEventData(123)).toBe(123)
|
||||
expect(sanitizeEventData(true)).toBe(true)
|
||||
})
|
||||
|
||||
it.concurrent('should redact sensitive patterns in top-level strings', () => {
|
||||
const result = sanitizeEventData('Bearer secrettoken123')
|
||||
expect(result).toContain('[REDACTED]')
|
||||
})
|
||||
|
||||
it.concurrent('should not redact normal strings', () => {
|
||||
const result = sanitizeEventData('normal string')
|
||||
expect(result).toBe('normal string')
|
||||
})
|
||||
})
|
||||
|
||||
describe('no false positives', () => {
|
||||
it.concurrent('should not remove keys with sensitive words in middle', () => {
|
||||
const event = {
|
||||
tokenCount: 500,
|
||||
isAuthenticated: true,
|
||||
hasSecretFeature: false,
|
||||
}
|
||||
|
||||
const result = sanitizeEventData(event)
|
||||
|
||||
expect(result.tokenCount).toBe(500)
|
||||
expect(result.isAuthenticated).toBe(true)
|
||||
expect(result.hasSecretFeature).toBe(false)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,28 +1,122 @@
|
||||
/**
|
||||
* Recursively redacts API keys in an object
|
||||
* @param obj The object to redact API keys from
|
||||
* @returns A new object with API keys redacted
|
||||
* Centralized redaction utilities for sensitive data
|
||||
*/
|
||||
export const redactApiKeys = (obj: any): any => {
|
||||
if (!obj || typeof obj !== 'object') {
|
||||
|
||||
/** Standard marker used for all redacted values */
|
||||
export const REDACTED_MARKER = '[REDACTED]'
|
||||
|
||||
/**
|
||||
* Patterns for sensitive key names (case-insensitive matching)
|
||||
* These patterns match common naming conventions for sensitive data
|
||||
*/
|
||||
const SENSITIVE_KEY_PATTERNS: RegExp[] = [
|
||||
/^api[_-]?key$/i,
|
||||
/^access[_-]?token$/i,
|
||||
/^refresh[_-]?token$/i,
|
||||
/^client[_-]?secret$/i,
|
||||
/^private[_-]?key$/i,
|
||||
/^auth[_-]?token$/i,
|
||||
/^.*secret$/i,
|
||||
/^.*password$/i,
|
||||
/^.*token$/i,
|
||||
/^.*credential$/i,
|
||||
/^authorization$/i,
|
||||
/^bearer$/i,
|
||||
/^private$/i,
|
||||
/^auth$/i,
|
||||
]
|
||||
|
||||
/**
|
||||
* Patterns for sensitive values in strings (for redacting values, not keys)
|
||||
* Each pattern has a replacement function
|
||||
*/
|
||||
const SENSITIVE_VALUE_PATTERNS: Array<{
|
||||
pattern: RegExp
|
||||
replacement: string
|
||||
}> = [
|
||||
// Bearer tokens
|
||||
{
|
||||
pattern: /Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi,
|
||||
replacement: `Bearer ${REDACTED_MARKER}`,
|
||||
},
|
||||
// Basic auth
|
||||
{
|
||||
pattern: /Basic\s+[A-Za-z0-9+/]+=*/gi,
|
||||
replacement: `Basic ${REDACTED_MARKER}`,
|
||||
},
|
||||
// API keys that look like sk-..., pk-..., etc.
|
||||
{
|
||||
pattern: /\b(sk|pk|api|key)[_-][A-Za-z0-9\-._]{20,}\b/gi,
|
||||
replacement: REDACTED_MARKER,
|
||||
},
|
||||
// JSON-style password fields: password: "value" or password: 'value'
|
||||
{
|
||||
pattern: /password['":\s]*['"][^'"]+['"]/gi,
|
||||
replacement: `password: "${REDACTED_MARKER}"`,
|
||||
},
|
||||
// JSON-style token fields: token: "value" or token: 'value'
|
||||
{
|
||||
pattern: /token['":\s]*['"][^'"]+['"]/gi,
|
||||
replacement: `token: "${REDACTED_MARKER}"`,
|
||||
},
|
||||
// JSON-style api_key fields: api_key: "value" or api-key: "value"
|
||||
{
|
||||
pattern: /api[_-]?key['":\s]*['"][^'"]+['"]/gi,
|
||||
replacement: `api_key: "${REDACTED_MARKER}"`,
|
||||
},
|
||||
]
|
||||
|
||||
/**
|
||||
* Checks if a key name matches any sensitive pattern
|
||||
* @param key - The key name to check
|
||||
* @returns True if the key is considered sensitive
|
||||
*/
|
||||
export function isSensitiveKey(key: string): boolean {
|
||||
const lowerKey = key.toLowerCase()
|
||||
return SENSITIVE_KEY_PATTERNS.some((pattern) => pattern.test(lowerKey))
|
||||
}
|
||||
|
||||
/**
|
||||
* Redacts sensitive patterns from a string value
|
||||
* @param value - The string to redact
|
||||
* @returns The string with sensitive patterns redacted
|
||||
*/
|
||||
export function redactSensitiveValues(value: string): string {
|
||||
if (!value || typeof value !== 'string') {
|
||||
return value
|
||||
}
|
||||
|
||||
let result = value
|
||||
for (const { pattern, replacement } of SENSITIVE_VALUE_PATTERNS) {
|
||||
result = result.replace(pattern, replacement)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively redacts sensitive data (API keys, passwords, tokens, etc.) from an object
|
||||
*
|
||||
* @param obj - The object to redact sensitive data from
|
||||
* @returns A new object with sensitive data redacted
|
||||
*/
|
||||
export function redactApiKeys(obj: any): any {
|
||||
if (obj === null || obj === undefined) {
|
||||
return obj
|
||||
}
|
||||
|
||||
if (typeof obj !== 'object') {
|
||||
return obj
|
||||
}
|
||||
|
||||
if (Array.isArray(obj)) {
|
||||
return obj.map(redactApiKeys)
|
||||
return obj.map((item) => redactApiKeys(item))
|
||||
}
|
||||
|
||||
const result: Record<string, any> = {}
|
||||
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
if (
|
||||
key.toLowerCase() === 'apikey' ||
|
||||
key.toLowerCase() === 'api_key' ||
|
||||
key.toLowerCase() === 'access_token' ||
|
||||
/\bsecret\b/i.test(key.toLowerCase()) ||
|
||||
/\bpassword\b/i.test(key.toLowerCase())
|
||||
) {
|
||||
result[key] = '***REDACTED***'
|
||||
if (isSensitiveKey(key)) {
|
||||
result[key] = REDACTED_MARKER
|
||||
} else if (typeof value === 'object' && value !== null) {
|
||||
result[key] = redactApiKeys(value)
|
||||
} else {
|
||||
@@ -32,3 +126,64 @@ export const redactApiKeys = (obj: any): any => {
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitizes a string for safe logging by truncating and redacting sensitive patterns
|
||||
*
|
||||
* @param value - The string to sanitize
|
||||
* @param maxLength - Maximum length of the output (default: 100)
|
||||
* @returns The sanitized string
|
||||
*/
|
||||
export function sanitizeForLogging(value: string, maxLength = 100): string {
|
||||
if (!value) return ''
|
||||
|
||||
let sanitized = value.substring(0, maxLength)
|
||||
|
||||
sanitized = redactSensitiveValues(sanitized)
|
||||
|
||||
return sanitized
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitizes event data for error reporting/analytics
|
||||
*
|
||||
* @param event - The event data to sanitize
|
||||
* @returns Sanitized event data safe for external reporting
|
||||
*/
|
||||
export function sanitizeEventData(event: any): any {
|
||||
if (event === null || event === undefined) {
|
||||
return event
|
||||
}
|
||||
|
||||
if (typeof event === 'string') {
|
||||
return redactSensitiveValues(event)
|
||||
}
|
||||
|
||||
if (typeof event !== 'object') {
|
||||
return event
|
||||
}
|
||||
|
||||
if (Array.isArray(event)) {
|
||||
return event.map((item) => sanitizeEventData(item))
|
||||
}
|
||||
|
||||
const sanitized: Record<string, unknown> = {}
|
||||
|
||||
for (const [key, value] of Object.entries(event)) {
|
||||
if (isSensitiveKey(key)) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (typeof value === 'string') {
|
||||
sanitized[key] = redactSensitiveValues(value)
|
||||
} else if (Array.isArray(value)) {
|
||||
sanitized[key] = value.map((v) => sanitizeEventData(v))
|
||||
} else if (value && typeof value === 'object') {
|
||||
sanitized[key] = sanitizeEventData(value)
|
||||
} else {
|
||||
sanitized[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
return sanitized
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { getRotatingApiKey } from '@/lib/core/config/api-keys'
|
||||
import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption'
|
||||
import { redactApiKeys } from '@/lib/core/security/redaction'
|
||||
import { cn } from '@/lib/core/utils/cn'
|
||||
import {
|
||||
formatDate,
|
||||
@@ -229,86 +228,6 @@ describe('getTimezoneAbbreviation', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('redactApiKeys', () => {
|
||||
it.concurrent('should redact API keys in objects', () => {
|
||||
const obj = {
|
||||
apiKey: 'secret-key',
|
||||
api_key: 'another-secret',
|
||||
access_token: 'token-value',
|
||||
secret: 'secret-value',
|
||||
password: 'password-value',
|
||||
normalField: 'normal-value',
|
||||
}
|
||||
|
||||
const result = redactApiKeys(obj)
|
||||
|
||||
expect(result.apiKey).toBe('***REDACTED***')
|
||||
expect(result.api_key).toBe('***REDACTED***')
|
||||
expect(result.access_token).toBe('***REDACTED***')
|
||||
expect(result.secret).toBe('***REDACTED***')
|
||||
expect(result.password).toBe('***REDACTED***')
|
||||
expect(result.normalField).toBe('normal-value')
|
||||
})
|
||||
|
||||
it.concurrent('should redact API keys in nested objects', () => {
|
||||
const obj = {
|
||||
config: {
|
||||
apiKey: 'secret-key',
|
||||
normalField: 'normal-value',
|
||||
},
|
||||
}
|
||||
|
||||
const result = redactApiKeys(obj)
|
||||
|
||||
expect(result.config.apiKey).toBe('***REDACTED***')
|
||||
expect(result.config.normalField).toBe('normal-value')
|
||||
})
|
||||
|
||||
it.concurrent('should redact API keys in arrays', () => {
|
||||
const arr = [{ apiKey: 'secret-key-1' }, { apiKey: 'secret-key-2' }]
|
||||
|
||||
const result = redactApiKeys(arr)
|
||||
|
||||
expect(result[0].apiKey).toBe('***REDACTED***')
|
||||
expect(result[1].apiKey).toBe('***REDACTED***')
|
||||
})
|
||||
|
||||
it.concurrent('should handle primitive values', () => {
|
||||
expect(redactApiKeys('string')).toBe('string')
|
||||
expect(redactApiKeys(123)).toBe(123)
|
||||
expect(redactApiKeys(null)).toBe(null)
|
||||
expect(redactApiKeys(undefined)).toBe(undefined)
|
||||
})
|
||||
|
||||
it.concurrent('should handle complex nested structures', () => {
|
||||
const obj = {
|
||||
users: [
|
||||
{
|
||||
name: 'John',
|
||||
credentials: {
|
||||
apiKey: 'secret-key',
|
||||
username: 'john_doe',
|
||||
},
|
||||
},
|
||||
],
|
||||
config: {
|
||||
database: {
|
||||
password: 'db-password',
|
||||
host: 'localhost',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const result = redactApiKeys(obj)
|
||||
|
||||
expect(result.users[0].name).toBe('John')
|
||||
expect(result.users[0].credentials.apiKey).toBe('***REDACTED***')
|
||||
expect(result.users[0].credentials.username).toBe('john_doe')
|
||||
expect(result.config.database.password).toBe('***REDACTED***')
|
||||
expect(result.config.database.host).toBe('localhost')
|
||||
})
|
||||
})
|
||||
|
||||
describe('validateName', () => {
|
||||
it.concurrent('should remove invalid characters', () => {
|
||||
const result = validateName('test@#$%name')
|
||||
|
||||
@@ -80,7 +80,7 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
|
||||
return { entries: state.entries }
|
||||
}
|
||||
|
||||
// Redact API keys from output
|
||||
// Redact API keys from output and input
|
||||
const redactedEntry = { ...entry }
|
||||
if (
|
||||
!isStreamingOutput(entry.output) &&
|
||||
@@ -89,6 +89,9 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
|
||||
) {
|
||||
redactedEntry.output = redactApiKeys(redactedEntry.output)
|
||||
}
|
||||
if (redactedEntry.input && typeof redactedEntry.input === 'object') {
|
||||
redactedEntry.input = redactApiKeys(redactedEntry.input)
|
||||
}
|
||||
|
||||
// Create new entry with ID and timestamp
|
||||
const newEntry: ConsoleEntry = {
|
||||
@@ -275,12 +278,17 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
|
||||
}
|
||||
|
||||
if (update.replaceOutput !== undefined) {
|
||||
updatedEntry.output = update.replaceOutput
|
||||
updatedEntry.output =
|
||||
typeof update.replaceOutput === 'object' && update.replaceOutput !== null
|
||||
? redactApiKeys(update.replaceOutput)
|
||||
: update.replaceOutput
|
||||
} else if (update.output !== undefined) {
|
||||
updatedEntry.output = {
|
||||
const mergedOutput = {
|
||||
...(entry.output || {}),
|
||||
...update.output,
|
||||
}
|
||||
updatedEntry.output =
|
||||
typeof mergedOutput === 'object' ? redactApiKeys(mergedOutput) : mergedOutput
|
||||
}
|
||||
|
||||
if (update.error !== undefined) {
|
||||
@@ -304,7 +312,10 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
|
||||
}
|
||||
|
||||
if (update.input !== undefined) {
|
||||
updatedEntry.input = update.input
|
||||
updatedEntry.input =
|
||||
typeof update.input === 'object' && update.input !== null
|
||||
? redactApiKeys(update.input)
|
||||
: update.input
|
||||
}
|
||||
|
||||
return updatedEntry
|
||||
|
||||
Reference in New Issue
Block a user