Redact additional keys in logs of Flows (#18534)

* Redact additional keys in logs of Flows

* Create moody-poems-pump.md

* Move REDACTED_TEXT to constants package & update tests

* Revert "Move REDACTED_TEXT to constants package & update tests"

This reverts commit 0f5b227253.

* Update redacted value in blackbox test

* Use own redact implementation

* Move REDACTED_TEXT to constants package

* Replace outdated comment

* Fix misleading return type

Since values might change (redacted), output isn't necessarily the same type anymore

api/src/constants.ts

@@ -81,5 +81,3 @@ export const SUPPORTED_IMAGE_METADATA_FORMATS = [
 	'image/tiff',
 	'image/avif',
 ];
-
-export const REDACT_TEXT = '--redact--';

api/src/flows.ts

@@ -1,3 +1,4 @@
+import { Action, REDACTED_TEXT } from '@directus/constants';
 import * as sharedExceptions from '@directus/exceptions';
 import type {
 	Accountability,
@@ -8,9 +9,7 @@ import type {
 	OperationHandler,
 	SchemaOverview,
 } from '@directus/types';
-import { Action } from '@directus/constants';
 import { applyOptionsData, isValidJSON, parseJSON, toArray } from '@directus/utils';
-import fastRedact from 'fast-redact';
 import type { Knex } from 'knex';
 import { omit, pick } from 'lodash-es';
 import { get } from 'micromustache';
@@ -22,24 +21,19 @@ import * as exceptions from './exceptions/index.js';
 import logger from './logger.js';
 import { getMessenger } from './messenger.js';
 import { ActivityService } from './services/activity.js';
-import * as services from './services/index.js';
 import { FlowsService } from './services/flows.js';
+import * as services from './services/index.js';
 import { RevisionsService } from './services/revisions.js';
 import type { EventHandler } from './types/index.js';
 import { constructFlowTree } from './utils/construct-flow-tree.js';
 import { getSchema } from './utils/get-schema.js';
 import { JobQueue } from './utils/job-queue.js';
 import { mapValuesDeep } from './utils/map-values-deep.js';
+import { redact } from './utils/redact.js';
 import { sanitizeError } from './utils/sanitize-error.js';
 
 let flowManager: FlowManager | undefined;
 
-const redactLogs = fastRedact({
-	censor: '--redacted--',
-	paths: ['*.headers.authorization', '*.access_token', '*.headers.cookie'],
-	serialize: false,
-});
-
 export function getFlowManager(): FlowManager {
 	if (flowManager) {
 		return flowManager;
@@ -369,7 +363,16 @@ class FlowManager {
 					item: flow.id,
 					data: {
 						steps: steps,
-						data: redactLogs(omit(keyedData, '$accountability.permissions')), // Permissions is a ton of data, and is just a copy of what's in the directus_permissions table
+						data: redact(
+							omit(keyedData, '$accountability.permissions'), // Permissions is a ton of data, and is just a copy of what's in the directus_permissions table
+							[
+								['**', 'headers', 'authorization'],
+								['**', 'headers', 'cookie'],
+								['**', 'query', 'access_token'],
+								['**', 'payload', 'password'],
+							],
+							REDACTED_TEXT
+						),
 					},
 				});
 			}

api/src/logger.test.ts

@@ -1,7 +1,7 @@
+import { REDACTED_TEXT } from '@directus/constants';
 import { Writable } from 'node:stream';
 import { pino } from 'pino';
 import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
-import { REDACT_TEXT } from './constants.js';
 
 const REFRESH_TOKEN_COOKIE_NAME = 'directus_refresh_token';
 
@@ -54,7 +54,7 @@ describe('req.headers.authorization', () => {
 		expect(logOutput.mock.calls[0][0]).toMatchObject({
 			req: {
 				headers: {
-					authorization: REDACT_TEXT,
+					authorization: REDACTED_TEXT,
 				},
 			},
 		});
@@ -76,7 +76,7 @@ describe('req.headers.cookie', () => {
 		expect(logOutput.mock.calls[0][0]).toMatchObject({
 			req: {
 				headers: {
-					cookie: REDACT_TEXT,
+					cookie: REDACTED_TEXT,
 				},
 			},
 		});
@@ -96,7 +96,7 @@ describe('req.headers.cookie', () => {
 		expect(logOutput.mock.calls[0][0]).toMatchObject({
 			req: {
 				headers: {
-					cookie: REDACT_TEXT,
+					cookie: REDACTED_TEXT,
 				},
 			},
 		});
@@ -118,7 +118,7 @@ describe('res.headers', () => {
 		expect(logOutput.mock.calls[0][0]).toMatchObject({
 			res: {
 				headers: {
-					'set-cookie': REDACT_TEXT,
+					'set-cookie': REDACTED_TEXT,
 				},
 			},
 		});
@@ -143,7 +143,7 @@ describe('res.headers', () => {
 		expect(logOutput.mock.calls[0][0]).toMatchObject({
 			res: {
 				headers: {
-					'set-cookie': REDACT_TEXT,
+					'set-cookie': REDACTED_TEXT,
 				},
 			},
 		});

api/src/logger.ts

@@ -1,19 +1,19 @@
+import { REDACTED_TEXT } from '@directus/constants';
 import { toArray } from '@directus/utils';
-import { merge } from 'lodash-es';
-import { pino } from 'pino';
-import type { LoggerOptions } from 'pino';
 import type { Request, RequestHandler } from 'express';
+import { merge } from 'lodash-es';
+import type { LoggerOptions } from 'pino';
+import { pino } from 'pino';
 import { pinoHttp, stdSerializers } from 'pino-http';
 import { URL } from 'url';
 import env from './env.js';
-import { REDACT_TEXT } from './constants.js';
 import { getConfigFromEnv } from './utils/get-config-from-env.js';
 
 const pinoOptions: LoggerOptions = {
 	level: env['LOG_LEVEL'] || 'info',
 	redact: {
 		paths: ['req.headers.authorization', 'req.headers.cookie'],
-		censor: REDACT_TEXT,
+		censor: REDACTED_TEXT,
 	},
 };
 
@@ -21,7 +21,7 @@ export const httpLoggerOptions: LoggerOptions = {
 	level: env['LOG_LEVEL'] || 'info',
 	redact: {
 		paths: ['req.headers.authorization', 'req.headers.cookie'],
-		censor: REDACT_TEXT,
+		censor: REDACTED_TEXT,
 	},
 };
 
@@ -56,13 +56,13 @@ if (env['LOG_STYLE'] === 'raw') {
 
 			if (path === 'res.headers') {
 				if ('set-cookie' in value) {
-					value['set-cookie'] = REDACT_TEXT;
+					value['set-cookie'] = REDACTED_TEXT;
 				}
 
 				return value;
 			}
 
-			return REDACT_TEXT;
+			return REDACTED_TEXT;
 		},
 	};
 }
@@ -121,7 +121,7 @@ function redactQuery(originalPath: string) {
 	const url = new URL(originalPath, 'http://example.com/');
 
 	if (url.searchParams.has('access_token')) {
-		url.searchParams.set('access_token', REDACT_TEXT);
+		url.searchParams.set('access_token', REDACTED_TEXT);
 	}
 
 	return url.pathname + url.search;

api/src/utils/redact.test.ts

@@ -0,0 +1,144 @@
+import { REDACTED_TEXT } from '@directus/constants';
+import { merge } from 'lodash-es';
+import { expect, test } from 'vitest';
+import { redact } from './redact.js';
+
+const input = {
+	$trigger: {
+		event: 'users.create',
+		payload: {
+			first_name: 'Example',
+			last_name: 'User',
+			email: 'user@example.com',
+			password: 'secret',
+		},
+		key: 'eb641950-fffa-4388-8606-aede594ae487',
+		collection: 'directus_users',
+	},
+	exec_fm27u: {
+		$trigger: {
+			event: 'users.create',
+			payload: {
+				first_name: 'Example',
+				last_name: 'User',
+				email: 'user@example.com',
+				password: 'secret',
+			},
+			key: 'eb641950-fffa-4388-8606-aede594ae487',
+			collection: 'directus_users',
+		},
+		$last: {
+			event: 'users.create',
+			payload: {
+				first_name: 'Example',
+				last_name: 'User',
+				email: 'user@example.com',
+				password: 'secret',
+			},
+			key: 'eb641950-fffa-4388-8606-aede594ae487',
+			collection: 'directus_users',
+		},
+	},
+};
+
+test('should not mutate input', () => {
+	const result = redact(input, [['$trigger']], REDACTED_TEXT);
+
+	expect(result).not.toBe(input);
+});
+
+test('should support single level path', () => {
+	const result = redact(input, [['$trigger']], REDACTED_TEXT);
+
+	expect(result).toEqual(
+		merge({}, input, {
+			$trigger: REDACTED_TEXT,
+		})
+	);
+});
+
+test('should support multi level path', () => {
+	const result = redact(input, [['$trigger', 'payload', 'password']], REDACTED_TEXT);
+
+	expect(result).toEqual(
+		merge({}, input, {
+			$trigger: {
+				payload: { password: REDACTED_TEXT },
+			},
+		})
+	);
+});
+
+test('should support wildcard path', () => {
+	const result = redact(input, [['*', 'payload']], REDACTED_TEXT);
+
+	expect(result).toEqual(
+		merge({}, input, {
+			$trigger: {
+				payload: REDACTED_TEXT,
+			},
+		})
+	);
+});
+
+test('should support deep path', () => {
+	const result = redact(input, [['**', 'password']], REDACTED_TEXT);
+
+	expect(result).toMatchObject(
+		merge({}, input, {
+			$trigger: {
+				payload: {
+					password: REDACTED_TEXT,
+				},
+			},
+			exec_fm27u: {
+				$trigger: {
+					payload: {
+						password: REDACTED_TEXT,
+					},
+				},
+				$last: {
+					payload: {
+						password: REDACTED_TEXT,
+					},
+				},
+			},
+		})
+	);
+});
+
+test('should support multiple paths', () => {
+	const result = redact(
+		input,
+		[
+			['$trigger', 'key'],
+			['*', 'payload', 'email'],
+			['**', 'password'],
+		],
+		REDACTED_TEXT
+	);
+
+	expect(result).toEqual(
+		merge({}, input, {
+			$trigger: {
+				key: REDACTED_TEXT,
+				payload: {
+					email: REDACTED_TEXT,
+					password: REDACTED_TEXT,
+				},
+			},
+			exec_fm27u: {
+				$trigger: {
+					payload: {
+						password: REDACTED_TEXT,
+					},
+				},
+				$last: {
+					payload: {
+						password: REDACTED_TEXT,
+					},
+				},
+			},
+		})
+	);
+});

api/src/utils/redact.ts

@@ -0,0 +1,88 @@
+import type { UnknownObject } from '@directus/types';
+import { isObject } from '@directus/utils';
+
+type Paths = string[][];
+
+/**
+ * Redact values at certain paths in an object.
+ * @param input Input object in which values should be redacted.
+ * @param paths Nested array of object paths to be redacted (supports `*` for shallow matching, `**` for deep matching).
+ * @param replacement Replacement the values are redacted by.
+ * @returns Redacted object.
+ */
+export function redact(input: UnknownObject, paths: Paths, replacement: string): UnknownObject {
+	const wildcardChars = ['*', '**'];
+
+	const clone = structuredClone(input);
+	const visited = new WeakSet<UnknownObject>();
+	traverse(clone, paths);
+
+	return clone;
+
+	function traverse(object: UnknownObject, checkPaths: Paths): void {
+		if (checkPaths.length === 0) {
+			return;
+		}
+
+		visited.add(object);
+
+		const globalCheckPaths = [];
+
+		for (const key of Object.keys(object)) {
+			const localCheckPaths = [];
+
+			for (const [index, path] of [...checkPaths].entries()) {
+				const [current, ...remaining] = path;
+
+				const escapedKey = wildcardChars.includes(key) ? `\\${key}` : key;
+
+				switch (current) {
+					case escapedKey:
+						if (remaining.length > 0) {
+							localCheckPaths.push(remaining);
+						} else {
+							object[key] = replacement;
+							checkPaths.splice(index, 1);
+						}
+
+						break;
+					case '*':
+						if (remaining.length > 0) {
+							globalCheckPaths.push(remaining);
+							checkPaths.splice(index, 1);
+						} else {
+							object[key] = replacement;
+						}
+
+						break;
+					case '**':
+						if (remaining.length > 0) {
+							const [next, ...nextRemaining] = remaining;
+
+							if (next === escapedKey) {
+								if (nextRemaining.length === 0) {
+									object[key] = replacement;
+								} else {
+									localCheckPaths.push(nextRemaining);
+								}
+							} else if (next !== undefined && wildcardChars.includes(next)) {
+								localCheckPaths.push(remaining);
+							} else {
+								localCheckPaths.push(path);
+							}
+						} else {
+							object[key] = replacement;
+						}
+
+						break;
+				}
+			}
+
+			const value = object[key];
+
+			if (isObject(value) && !visited.has(value)) {
+				traverse(value, [...globalCheckPaths, ...localCheckPaths]);
+			}
+		}
+	}
+}