Redact env values in logs for Flows (#19513)

* Redact env values in revisions for Flows

* Add unit tests

* Create cold-maps-teach.md

* Redact string type envs only

Co-authored-by: Brainslug <br41nslug@users.noreply.github.com>

* Fix linting

* Update test for non-string env

* Ignore zero length strings

Co-authored-by: Brainslug <br41nslug@users.noreply.github.com>

* Add replacementFn to include key of redacted value

* Update cold-maps-teach.md

* Remove case insensitivity

* Update changeset

* Rework

* Add utils to changeset

* Add unit test

* Rename to getRedactedString and add REDACTED_TEXT

* Consistent naming

---------

Co-authored-by: Brainslug <br41nslug@users.noreply.github.com>
Co-authored-by: Pascal Jufer <pascal-jufer@bluewin.ch>

api/src/flows.ts

@@ -1,4 +1,4 @@
-import { Action, REDACTED_TEXT } from '@directus/constants';
+import { Action } from '@directus/constants';
 import type {
 	Accountability,
 	ActionHandler,
@@ -8,7 +8,7 @@ import type {
 	OperationHandler,
 	SchemaOverview,
 } from '@directus/types';
-import { applyOptionsData, isValidJSON, parseJSON, toArray } from '@directus/utils';
+import { applyOptionsData, getRedactedString, isValidJSON, parseJSON, toArray } from '@directus/utils';
 import type { Knex } from 'knex';
 import { omit, pick } from 'lodash-es';
 import { get } from 'micromustache';
@@ -27,7 +27,7 @@ import { constructFlowTree } from './utils/construct-flow-tree.js';
 import { getSchema } from './utils/get-schema.js';
 import { JobQueue } from './utils/job-queue.js';
 import { mapValuesDeep } from './utils/map-values-deep.js';
-import { redact } from './utils/redact.js';
+import { redactObject } from './utils/redact-object.js';
 import { sanitizeError } from './utils/sanitize-error.js';
 import { scheduleSynchronizedJob, validateCron } from './utils/schedule.js';
 
@@ -63,9 +63,11 @@ class FlowManager {
 	private webhookFlowHandlers: Record<string, any> = {};
 
 	private reloadQueue: JobQueue;
+	private envs: Record<string, any>;
 
 	constructor() {
 		this.reloadQueue = new JobQueue();
+		this.envs = env['FLOWS_ENV_ALLOW_LIST'] ? pick(env, toArray(env['FLOWS_ENV_ALLOW_LIST'])) : {};
 
 		const messenger = getMessenger();
 
@@ -308,7 +310,7 @@ class FlowManager {
 			[TRIGGER_KEY]: data,
 			[LAST_KEY]: data,
 			[ACCOUNTABILITY_KEY]: context?.['accountability'] ?? null,
-			[ENV_KEY]: pick(env, env['FLOWS_ENV_ALLOW_LIST'] ? toArray(env['FLOWS_ENV_ALLOW_LIST']) : []),
+			[ENV_KEY]: this.envs,
 		};
 
 		let nextOperation = flow.operation;
@@ -361,16 +363,19 @@ class FlowManager {
 					collection: 'directus_flows',
 					item: flow.id,
 					data: {
-						steps: steps,
-						data: redact(
+						steps: steps.map((step) => redactObject(step, { values: this.envs }, getRedactedString)),
+						data: redactObject(
 							omit(keyedData, '$accountability.permissions'), // Permissions is a ton of data, and is just a copy of what's in the directus_permissions table
-							[
-								['**', 'headers', 'authorization'],
-								['**', 'headers', 'cookie'],
-								['**', 'query', 'access_token'],
-								['**', 'payload', 'password'],
-							],
-							REDACTED_TEXT
+							{
+								keys: [
+									['**', 'headers', 'authorization'],
+									['**', 'headers', 'cookie'],
+									['**', 'query', 'access_token'],
+									['**', 'payload', 'password'],
+								],
+								values: this.envs,
+							},
+							getRedactedString
 						),
 					},
 				});

api/src/logger.test.ts

@@ -1,4 +1,4 @@
-import { REDACTED_TEXT } from '@directus/constants';
+import { REDACTED_TEXT } from '@directus/utils';
 import { Writable } from 'node:stream';
 import { pino } from 'pino';
 import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';

api/src/logger.ts

@@ -1,5 +1,4 @@
-import { REDACTED_TEXT } from '@directus/constants';
-import { toArray } from '@directus/utils';
+import { REDACTED_TEXT, toArray } from '@directus/utils';
 import type { Request, RequestHandler } from 'express';
 import { merge } from 'lodash-es';
 import type { LoggerOptions } from 'pino';

api/src/utils/redact-object.test.ts

@@ -1,7 +1,7 @@
-import { REDACTED_TEXT } from '@directus/constants';
+import { getRedactedString, REDACTED_TEXT } from '@directus/utils';
 import { merge } from 'lodash-es';
 import { describe, expect, test } from 'vitest';
-import { errorReplacer, redact } from './redact.js';
+import { getReplacer, redactObject } from './redact-object.js';
 
 const input = {
 	$trigger: {
@@ -42,13 +42,13 @@ const input = {
 };
 
 test('should not mutate input', () => {
-	const result = redact(input, [['$trigger']], REDACTED_TEXT);
+	const result = redactObject(input, { keys: [['$trigger']] }, getRedactedString);
 
 	expect(result).not.toBe(input);
 });
 
 test('should support single level path', () => {
-	const result = redact(input, [['$trigger']], REDACTED_TEXT);
+	const result = redactObject(input, { keys: [['$trigger']] }, getRedactedString);
 
 	expect(result).toEqual(
 		merge({}, input, {
@@ -58,7 +58,7 @@ test('should support single level path', () => {
 });
 
 test('should support multi level path', () => {
-	const result = redact(input, [['$trigger', 'payload', 'password']], REDACTED_TEXT);
+	const result = redactObject(input, { keys: [['$trigger', 'payload', 'password']] }, getRedactedString);
 
 	expect(result).toEqual(
 		merge({}, input, {
@@ -70,7 +70,7 @@ test('should support multi level path', () => {
 });
 
 test('should support wildcard path', () => {
-	const result = redact(input, [['*', 'payload']], REDACTED_TEXT);
+	const result = redactObject(input, { keys: [['*', 'payload']] }, getRedactedString);
 
 	expect(result).toEqual(
 		merge({}, input, {
@@ -82,7 +82,7 @@ test('should support wildcard path', () => {
 });
 
 test('should support deep path', () => {
-	const result = redact(input, [['**', 'password']], REDACTED_TEXT);
+	const result = redactObject(input, { keys: [['**', 'password']] }, getRedactedString);
 
 	expect(result).toMatchObject(
 		merge({}, input, {
@@ -108,14 +108,16 @@ test('should support deep path', () => {
 });
 
 test('should support multiple paths', () => {
-	const result = redact(
+	const result = redactObject(
 		input,
-		[
-			['$trigger', 'key'],
-			['*', 'payload', 'email'],
-			['**', 'password'],
-		],
-		REDACTED_TEXT
+		{
+			keys: [
+				['$trigger', 'key'],
+				['*', 'payload', 'email'],
+				['**', 'password'],
+			],
+		},
+		getRedactedString
 	);
 
 	expect(result).toEqual(
@@ -143,11 +145,12 @@ test('should support multiple paths', () => {
 	);
 });
 
-describe('errorReplacer tests', () => {
+describe('getReplacer tests', () => {
 	test('Returns parsed error object', () => {
 		const errorMessage = 'Error Message';
 		const errorCause = 'Error Cause';
-		const result = errorReplacer('', new Error(errorMessage, { cause: errorCause }));
+		const replacer = getReplacer(getRedactedString);
+		const result: any = replacer('', new Error(errorMessage, { cause: errorCause }));
 		expect(result.name).toBe('Error');
 		expect(result.message).toBe(errorMessage);
 		expect(result.stack).toBeDefined();
@@ -173,8 +176,10 @@ describe('errorReplacer tests', () => {
 			},
 		];
 
+		const replacer = getReplacer(getRedactedString);
+
 		for (const value of values) {
-			expect(errorReplacer('', value)).toBe(value);
+			expect(replacer('', value)).toBe(value);
 		}
 	});
 
@@ -199,7 +204,60 @@ describe('errorReplacer tests', () => {
 			error: { name: 'Error', message: errorMessage, cause: errorCause },
 		};
 
-		const result = JSON.parse(JSON.stringify(objWithError, errorReplacer));
+		const result = JSON.parse(JSON.stringify(objWithError, getReplacer(getRedactedString)));
+
+		// Stack changes depending on env
+		expect(result.error.stack).toBeDefined();
+		delete result.error.stack;
+
+		expect(result).toStrictEqual(expectedResult);
+	});
+
+	test('Correctly redacts values when used with JSON.stringify()', () => {
+		const baseValue = {
+			num: 123,
+			bool: true,
+			null: null,
+			string_ignore: `No error Cause it's case sensitive~~`,
+		};
+
+		const objWithError = {
+			...baseValue,
+			string: `Replace cause case matches Errors~~`,
+			nested: { another_str: 'just because of safety 123456' },
+			nested_array: [{ str_a: 'cause surely' }, { str_b: 'not an Error' }, { str_ignore: 'nothing here' }],
+			array: ['something', 'no Error', 'just because', 'all is good'],
+			error: new Error('This is an Error message.', { cause: 'Here is an Error cause!' }),
+		};
+
+		const expectedResult = {
+			...baseValue,
+			string: `Replace ${getRedactedString('cause')} case matches ${getRedactedString('ERROR')}s~~`,
+			nested: { another_str: `just be${getRedactedString('cause')} of safety 123456` },
+			nested_array: [
+				{ str_a: `${getRedactedString('cause')} surely` },
+				{ str_b: `not an ${getRedactedString('ERROR')}` },
+				{ str_ignore: 'nothing here' },
+			],
+			array: ['something', `no ${getRedactedString('ERROR')}`, `just be${getRedactedString('cause')}`, 'all is good'],
+			error: {
+				name: getRedactedString('ERROR'),
+				message: `This is an ${getRedactedString('ERROR')} message.`,
+				cause: `Here is an ${getRedactedString('ERROR')} ${getRedactedString('cause')}!`,
+			},
+		};
+
+		const result = JSON.parse(
+			JSON.stringify(
+				objWithError,
+				getReplacer(getRedactedString, {
+					empty: '',
+					ERROR: 'Error',
+					cause: 'cause',
+					number: 123456,
+				})
+			)
+		);
 
 		// Stack changes depending on env
 		expect(result.error.stack).toBeDefined();

api/src/utils/redact-object.ts

@@ -0,0 +1,136 @@
+import type { UnknownObject } from '@directus/types';
+import { isObject } from '@directus/utils';
+
+type Keys = string[][];
+type Values = Record<string, any>;
+type Replacement = (key?: string) => string;
+
+/**
+ * Redact values in an object.
+ *
+ * @param input Input object in which values should be redacted.
+ * @param redact The key paths at which and values itself which should be redacted.
+ * @param redact.keys Nested array of key paths at which values should be redacted. (Supports `*` for shallow matching, `**` for deep matching.)
+ * @param redact.values Value names and the corresponding values that should be redacted.
+ * @param replacement Replacement function with which the values are redacted.
+ * @returns Redacted object.
+ */
+export function redactObject(
+	input: UnknownObject,
+	redact: {
+		keys?: Keys;
+		values?: Values;
+	},
+	replacement: Replacement
+): UnknownObject {
+	const wildcardChars = ['*', '**'];
+
+	const clone = JSON.parse(JSON.stringify(input, getReplacer(replacement, redact.values)));
+	const visited = new WeakSet<UnknownObject>();
+
+	if (redact.keys) {
+		traverse(clone, redact.keys);
+	}
+
+	return clone;
+
+	function traverse(object: UnknownObject, checkKeyPaths: Keys): void {
+		if (checkKeyPaths.length === 0) {
+			return;
+		}
+
+		visited.add(object);
+
+		const REDACTED_TEXT = replacement();
+		const globalCheckPaths = [];
+
+		for (const key of Object.keys(object)) {
+			const localCheckPaths = [];
+
+			for (const [index, path] of [...checkKeyPaths].entries()) {
+				const [current, ...remaining] = path;
+
+				const escapedKey = wildcardChars.includes(key) ? `\\${key}` : key;
+
+				switch (current) {
+					case escapedKey:
+						if (remaining.length > 0) {
+							localCheckPaths.push(remaining);
+						} else {
+							object[key] = REDACTED_TEXT;
+							checkKeyPaths.splice(index, 1);
+						}
+
+						break;
+					case '*':
+						if (remaining.length > 0) {
+							globalCheckPaths.push(remaining);
+							checkKeyPaths.splice(index, 1);
+						} else {
+							object[key] = REDACTED_TEXT;
+						}
+
+						break;
+					case '**':
+						if (remaining.length > 0) {
+							const [next, ...nextRemaining] = remaining;
+
+							if (next === escapedKey) {
+								if (nextRemaining.length === 0) {
+									object[key] = REDACTED_TEXT;
+								} else {
+									localCheckPaths.push(nextRemaining);
+								}
+							} else if (next !== undefined && wildcardChars.includes(next)) {
+								localCheckPaths.push(remaining);
+							} else {
+								localCheckPaths.push(path);
+							}
+						} else {
+							object[key] = REDACTED_TEXT;
+						}
+
+						break;
+				}
+			}
+
+			const value = object[key];
+
+			if (isObject(value) && !visited.has(value)) {
+				traverse(value, [...globalCheckPaths, ...localCheckPaths]);
+			}
+		}
+	}
+}
+
+/**
+ * Replace values and extract Error objects for use with JSON.stringify()
+ */
+export function getReplacer(replacement: Replacement, values?: Values) {
+	const filteredValues = values
+		? Object.entries(values).filter(([_k, v]) => typeof v === 'string' && v.length > 0)
+		: [];
+
+	return (_key: string, value: unknown) => {
+		if (value instanceof Error) {
+			return {
+				name: value.name,
+				message: value.message,
+				stack: value.stack,
+				cause: value.cause,
+			};
+		}
+
+		if (!values || filteredValues.length === 0 || typeof value !== 'string') return value;
+
+		let finalValue = value;
+
+		for (const [redactKey, valueToRedact] of filteredValues) {
+			if (finalValue.includes(valueToRedact)) {
+				finalValue = finalValue.replace(new RegExp(valueToRedact, 'g'), replacement(redactKey));
+			}
+		}
+
+		return finalValue;
+	};
+}

api/src/utils/redact.ts

@@ -1,104 +0,0 @@
-import type { UnknownObject } from '@directus/types';
-import { isObject } from '@directus/utils';
-
-type Paths = string[][];
-
-/**
- * Redact values at certain paths in an object.
- * @param input Input object in which values should be redacted.
- * @param paths Nested array of object paths to be redacted (supports `*` for shallow matching, `**` for deep matching).
- * @param replacement Replacement the values are redacted by.
- * @returns Redacted object.
- */
-export function redact(input: UnknownObject, paths: Paths, replacement: string): UnknownObject {
-	const wildcardChars = ['*', '**'];
-
-	const clone = JSON.parse(JSON.stringify(input, errorReplacer));
-	const visited = new WeakSet<UnknownObject>();
-	traverse(clone, paths);
-
-	return clone;
-
-	function traverse(object: UnknownObject, checkPaths: Paths): void {
-		if (checkPaths.length === 0) {
-			return;
-		}
-
-		visited.add(object);
-
-		const globalCheckPaths = [];
-
-		for (const key of Object.keys(object)) {
-			const localCheckPaths = [];
-
-			for (const [index, path] of [...checkPaths].entries()) {
-				const [current, ...remaining] = path;
-
-				const escapedKey = wildcardChars.includes(key) ? `\\${key}` : key;
-
-				switch (current) {
-					case escapedKey:
-						if (remaining.length > 0) {
-							localCheckPaths.push(remaining);
-						} else {
-							object[key] = replacement;
-							checkPaths.splice(index, 1);
-						}
-
-						break;
-					case '*':
-						if (remaining.length > 0) {
-							globalCheckPaths.push(remaining);
-							checkPaths.splice(index, 1);
-						} else {
-							object[key] = replacement;
-						}
-
-						break;
-					case '**':
-						if (remaining.length > 0) {
-							const [next, ...nextRemaining] = remaining;
-
-							if (next === escapedKey) {
-								if (nextRemaining.length === 0) {
-									object[key] = replacement;
-								} else {
-									localCheckPaths.push(nextRemaining);
-								}
-							} else if (next !== undefined && wildcardChars.includes(next)) {
-								localCheckPaths.push(remaining);
-							} else {
-								localCheckPaths.push(path);
-							}
-						} else {
-							object[key] = replacement;
-						}
-
-						break;
-				}
-			}
-
-			const value = object[key];
-
-			if (isObject(value) && !visited.has(value)) {
-				traverse(value, [...globalCheckPaths, ...localCheckPaths]);
-			}
-		}
-	}
-}
-
-/**
- * Extract values from Error objects for use with JSON.stringify()
- */
-export function errorReplacer(_key: string, value: unknown) {
-	if (value instanceof Error) {
-		return {
-			name: value.name,
-			message: value.message,
-			stack: value.stack,
-			cause: value.cause,
-		};
-	}
-
-	return value;
-}