Add getMilliseconds util for safer value interpretation (#17498)

* Add `getMilliseconds` util for safer value interpretation

* Test more data types

* Remove remnant

* Customizable fallback with default of undefined

* Clean-up

* Transform getMilliseconds to named export

---------

Co-authored-by: ian <licitdev@gmail.com>
This commit is contained in:
Pascal Jufer
2023-02-17 17:06:26 +01:00
committed by GitHub
parent 2ff9a64c87
commit e3c755dcf0
17 changed files with 133 additions and 87 deletions

View File

@@ -1,34 +1,34 @@
import { Router } from 'express';
import Joi from 'joi';
import ldap, {
Client,
Error,
EqualityFilter,
Error,
InappropriateAuthenticationError,
InsufficientAccessRightsError,
InvalidCredentialsError,
LDAPResult,
SearchCallbackResponse,
SearchEntry,
LDAPResult,
InappropriateAuthenticationError,
InvalidCredentialsError,
InsufficientAccessRightsError,
} from 'ldapjs';
import ms from 'ms';
import { getIPFromReq } from '../../utils/get-ip-from-req';
import Joi from 'joi';
import { AuthDriver } from '../auth';
import { AuthDriverOptions, User } from '../../types';
import env from '../../env';
import {
InvalidConfigException,
InvalidCredentialsException,
InvalidPayloadException,
InvalidProviderException,
ServiceUnavailableException,
InvalidConfigException,
UnexpectedResponseException,
} from '../../exceptions';
import { RecordNotUniqueException } from '../../exceptions/database/record-not-unique';
import { AuthenticationService, UsersService } from '../../services';
import asyncHandler from '../../utils/async-handler';
import env from '../../env';
import { respond } from '../../middleware/respond';
import logger from '../../logger';
import { respond } from '../../middleware/respond';
import { AuthenticationService, UsersService } from '../../services';
import { AuthDriverOptions, User } from '../../types';
import asyncHandler from '../../utils/async-handler';
import { getIPFromReq } from '../../utils/get-ip-from-req';
import { getMilliseconds } from '../../utils/get-milliseconds';
import { AuthDriver } from '../auth';
interface UserInfo {
dn: string;
@@ -408,7 +408,7 @@ export function createLDAPAuthRouter(provider: string): Router {
res.cookie(env.REFRESH_TOKEN_COOKIE_NAME, refreshToken, {
httpOnly: true,
domain: env.REFRESH_TOKEN_COOKIE_DOMAIN,
maxAge: ms(env.REFRESH_TOKEN_TTL as string),
maxAge: getMilliseconds(env.REFRESH_TOKEN_TTL),
secure: env.REFRESH_TOKEN_COOKIE_SECURE ?? false,
sameSite: (env.REFRESH_TOKEN_COOKIE_SAME_SITE as 'lax' | 'strict' | 'none') || 'strict',
});

View File

@@ -3,7 +3,6 @@ import { parseJSON } from '@directus/shared/utils';
import express, { Router } from 'express';
import flatten from 'flat';
import jwt from 'jsonwebtoken';
import ms from 'ms';
import { Client, errors, generators, Issuer } from 'openid-client';
import { getAuthProvider } from '../../auth';
import env from '../../env';
@@ -22,6 +21,7 @@ import { AuthData, AuthDriverOptions, User } from '../../types';
import asyncHandler from '../../utils/async-handler';
import { getConfigFromEnv } from '../../utils/get-config-from-env';
import { getIPFromReq } from '../../utils/get-ip-from-req';
import { getMilliseconds } from '../../utils/get-milliseconds';
import { Url } from '../../utils/url';
import { LocalAuthDriver } from './local';
@@ -327,7 +327,7 @@ export function createOAuth2AuthRouter(providerName: string): Router {
res.cookie(env.REFRESH_TOKEN_COOKIE_NAME, refreshToken, {
httpOnly: true,
domain: env.REFRESH_TOKEN_COOKIE_DOMAIN,
maxAge: ms(env.REFRESH_TOKEN_TTL as string),
maxAge: getMilliseconds(env.REFRESH_TOKEN_TTL),
secure: env.REFRESH_TOKEN_COOKIE_SECURE ?? false,
sameSite: (env.REFRESH_TOKEN_COOKIE_SAME_SITE as 'lax' | 'strict' | 'none') || 'strict',
});

View File

@@ -3,7 +3,6 @@ import { parseJSON } from '@directus/shared/utils';
import express, { Router } from 'express';
import flatten from 'flat';
import jwt from 'jsonwebtoken';
import ms from 'ms';
import { Client, errors, generators, Issuer } from 'openid-client';
import { getAuthProvider } from '../../auth';
import env from '../../env';
@@ -22,6 +21,7 @@ import { AuthData, AuthDriverOptions, User } from '../../types';
import asyncHandler from '../../utils/async-handler';
import { getConfigFromEnv } from '../../utils/get-config-from-env';
import { getIPFromReq } from '../../utils/get-ip-from-req';
import { getMilliseconds } from '../../utils/get-milliseconds';
import { Url } from '../../utils/url';
import { LocalAuthDriver } from './local';
@@ -356,7 +356,7 @@ export function createOpenIDAuthRouter(providerName: string): Router {
res.cookie(env.REFRESH_TOKEN_COOKIE_NAME, refreshToken, {
httpOnly: true,
domain: env.REFRESH_TOKEN_COOKIE_DOMAIN,
maxAge: ms(env.REFRESH_TOKEN_TTL as string),
maxAge: getMilliseconds(env.REFRESH_TOKEN_TTL),
secure: env.REFRESH_TOKEN_COOKIE_SECURE ?? false,
sameSite: (env.REFRESH_TOKEN_COOKIE_SAME_SITE as 'lax' | 'strict' | 'none') || 'strict',
});

View File

@@ -1,10 +1,10 @@
import Keyv, { Options } from 'keyv';
import ms from 'ms';
import env from './env';
import logger from './logger';
import { getConfigFromEnv } from './utils/get-config-from-env';
import { validateEnv } from './utils/validate-env';
import { compress, decompress } from './utils/compress';
import { getConfigFromEnv } from './utils/get-config-from-env';
import { getMilliseconds } from './utils/get-milliseconds';
import { validateEnv } from './utils/validate-env';
let cache: Keyv | null = null;
let systemCache: Keyv | null = null;
@@ -13,12 +13,12 @@ let lockCache: Keyv | null = null;
export function getCache(): { cache: Keyv | null; systemCache: Keyv; lockCache: Keyv } {
if (env.CACHE_ENABLED === true && cache === null) {
validateEnv(['CACHE_NAMESPACE', 'CACHE_TTL', 'CACHE_STORE']);
cache = getKeyvInstance(env.CACHE_TTL ? ms(env.CACHE_TTL as string) : undefined);
cache = getKeyvInstance(getMilliseconds(env.CACHE_TTL));
cache.on('error', (err) => logger.warn(err, `[cache] ${err}`));
}
if (systemCache === null) {
systemCache = getKeyvInstance(env.CACHE_SYSTEM_TTL ? ms(env.CACHE_SYSTEM_TTL as string) : undefined, '_system');
systemCache = getKeyvInstance(getMilliseconds(env.CACHE_SYSTEM_TTL), '_system');
systemCache.on('error', (err) => logger.warn(err, `[cache] ${err}`));
}

View File

@@ -1,6 +1,7 @@
import { TransformationParams } from './types';
import { CookieOptions } from 'express';
import env from './env';
import ms from 'ms';
import { TransformationParams } from './types';
import { getMilliseconds } from './utils/get-milliseconds';
export const SYSTEM_ASSET_ALLOW_LIST: TransformationParams[] = [
{
@@ -52,10 +53,10 @@ export const GENERATE_SPECIAL = ['uuid', 'date-created', 'role-created', 'user-c
export const UUID_REGEX = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}';
export const COOKIE_OPTIONS = {
export const COOKIE_OPTIONS: CookieOptions = {
httpOnly: true,
domain: env.REFRESH_TOKEN_COOKIE_DOMAIN,
maxAge: ms(env.REFRESH_TOKEN_TTL as string),
maxAge: getMilliseconds(env.REFRESH_TOKEN_TTL),
secure: env.REFRESH_TOKEN_COOKIE_SECURE ?? false,
sameSite: (env.REFRESH_TOKEN_COOKIE_SAME_SITE as 'lax' | 'strict' | 'none') || 'strict',
};

View File

@@ -4,7 +4,6 @@ import type { Range } from '@directus/storage';
import { parseJSON } from '@directus/shared/utils';
import { Router } from 'express';
import { merge, pick } from 'lodash';
import ms from 'ms';
import { ASSET_TRANSFORM_QUERY_KEYS, SYSTEM_ASSET_ALLOW_LIST } from '../constants';
import getDatabase from '../database';
import env from '../env';
@@ -16,6 +15,7 @@ import { TransformationMethods, TransformationParams, TransformationPreset } fro
import asyncHandler from '../utils/async-handler';
import { getCacheControlHeader } from '../utils/get-cache-headers';
import { getConfigFromEnv } from '../utils/get-config-from-env';
import { getMilliseconds } from '../utils/get-milliseconds';
const router = Router();
@@ -163,7 +163,7 @@ router.get(
res.attachment(req.params.filename ?? file.filename_download);
res.setHeader('Content-Type', file.type);
res.setHeader('Accept-Ranges', 'bytes');
res.setHeader('Cache-Control', getCacheControlHeader(req, ms(env.ASSETS_CACHE_TTL as string), false, true));
res.setHeader('Cache-Control', getCacheControlHeader(req, getMilliseconds(env.ASSETS_CACHE_TTL), false, true));
const unixTime = Date.parse(file.modified_on);
if (!Number.isNaN(unixTime)) {

View File

@@ -1,14 +1,14 @@
import { EXTENSION_TYPES } from '@directus/shared/constants';
import { Plural } from '@directus/shared/types';
import { depluralize, isIn } from '@directus/shared/utils';
import { Router } from 'express';
import asyncHandler from '../utils/async-handler';
import env from '../env';
import { RouteNotFoundException } from '../exceptions';
import { getExtensionManager } from '../extensions';
import ms from 'ms';
import env from '../env';
import { getCacheControlHeader } from '../utils/get-cache-headers';
import { respond } from '../middleware/respond';
import { depluralize, isIn } from '@directus/shared/utils';
import { Plural } from '@directus/shared/types';
import { EXTENSION_TYPES } from '@directus/shared/constants';
import asyncHandler from '../utils/async-handler';
import { getCacheControlHeader } from '../utils/get-cache-headers';
import { getMilliseconds } from '../utils/get-milliseconds';
const router = Router();
@@ -45,7 +45,7 @@ router.get(
}
res.setHeader('Content-Type', 'application/javascript; charset=UTF-8');
res.setHeader('Cache-Control', getCacheControlHeader(req, ms(env.EXTENSIONS_CACHE_TTL as string), false, false));
res.setHeader('Cache-Control', getCacheControlHeader(req, getMilliseconds(env.EXTENSIONS_CACHE_TTL), false, false));
res.setHeader('Vary', 'Origin, Cache-Control');
res.end(extensionSource);
})

View File

@@ -41,7 +41,7 @@ const checkCacheMiddleware: RequestHandler = asyncHandler(async (req, res, next)
return next();
}
const cacheTTL = cacheExpiryDate ? cacheExpiryDate - Date.now() : null;
const cacheTTL = cacheExpiryDate ? cacheExpiryDate - Date.now() : undefined;
res.setHeader('Cache-Control', getCacheControlHeader(req, cacheTTL, true, true));
res.setHeader('Vary', 'Origin, Cache-Control');

View File

@@ -1,15 +1,15 @@
import { parse as parseBytesConfiguration } from 'bytes';
import { RequestHandler } from 'express';
import ms from 'ms';
import { getCache, setCacheValue } from '../cache';
import env from '../env';
import asyncHandler from '../utils/async-handler';
import { getCacheKey } from '../utils/get-cache-key';
import { getCacheControlHeader } from '../utils/get-cache-headers';
import logger from '../logger';
import { ExportService } from '../services';
import asyncHandler from '../utils/async-handler';
import { getCacheControlHeader } from '../utils/get-cache-headers';
import { getCacheKey } from '../utils/get-cache-key';
import { getDateFormatted } from '../utils/get-date-formatted';
import { getMilliseconds } from '../utils/get-milliseconds';
import { stringByteSize } from '../utils/get-string-byte-size';
import { parse as parseBytesConfiguration } from 'bytes';
export const respond: RequestHandler = asyncHandler(async (req, res) => {
const { cache } = getCache();
@@ -33,13 +33,13 @@ export const respond: RequestHandler = asyncHandler(async (req, res) => {
const key = getCacheKey(req);
try {
await setCacheValue(cache, key, res.locals.payload, ms(env.CACHE_TTL as string));
await setCacheValue(cache, `${key}__expires_at`, { exp: Date.now() + ms(env.CACHE_TTL as string) });
await setCacheValue(cache, key, res.locals.payload, getMilliseconds(env.CACHE_TTL));
await setCacheValue(cache, `${key}__expires_at`, { exp: Date.now() + getMilliseconds(env.CACHE_TTL, 0) });
} catch (err: any) {
logger.warn(err, `[cache] Couldn't set key ${key}. ${err}`);
}
res.setHeader('Cache-Control', getCacheControlHeader(req, ms(env.CACHE_TTL as string), true, true));
res.setHeader('Cache-Control', getCacheControlHeader(req, getMilliseconds(env.CACHE_TTL), true, true));
res.setHeader('Vary', 'Origin, Cache-Control');
} else {
// Don't cache anything by default

View File

@@ -2,7 +2,6 @@ import { Accountability, Action, SchemaOverview } from '@directus/shared/types';
import jwt from 'jsonwebtoken';
import { Knex } from 'knex';
import { clone, cloneDeep } from 'lodash';
import ms from 'ms';
import { performance } from 'perf_hooks';
import { getAuthProvider } from '../auth';
import { DEFAULT_AUTH_PROVIDER } from '../constants';
@@ -17,6 +16,7 @@ import {
} from '../exceptions';
import { createRateLimiter } from '../rate-limiter';
import { AbstractServiceOptions, DirectusTokenPayload, LoginResult, Session, User } from '../types';
import { getMilliseconds } from '../utils/get-milliseconds';
import { stall } from '../utils/stall';
import { ActivityService } from './activity';
import { SettingsService } from './settings';
@@ -209,7 +209,7 @@ export class AuthenticationService {
});
const refreshToken = nanoid(64);
const refreshTokenExpiration = new Date(Date.now() + ms(env.REFRESH_TOKEN_TTL as string));
const refreshTokenExpiration = new Date(Date.now() + getMilliseconds(env.REFRESH_TOKEN_TTL, 0));
await this.knex('directus_sessions').insert({
token: refreshToken,
@@ -247,7 +247,7 @@ export class AuthenticationService {
return {
accessToken,
refreshToken,
expires: ms(env.ACCESS_TOKEN_TTL as string),
expires: getMilliseconds(env.ACCESS_TOKEN_TTL),
id: user.id,
};
}
@@ -364,7 +364,7 @@ export class AuthenticationService {
});
const newRefreshToken = nanoid(64);
const refreshTokenExpiration = new Date(Date.now() + ms(env.REFRESH_TOKEN_TTL as string));
const refreshTokenExpiration = new Date(Date.now() + getMilliseconds(env.REFRESH_TOKEN_TTL, 0));
await this.knex('directus_sessions')
.update({
@@ -380,7 +380,7 @@ export class AuthenticationService {
return {
accessToken,
refreshToken: newRefreshToken,
expires: ms(env.ACCESS_TOKEN_TTL as string),
expires: getMilliseconds(env.ACCESS_TOKEN_TTL),
id: record.user_id,
};
}

View File

@@ -1,5 +1,5 @@
import { BaseException } from '@directus/shared/exceptions';
import { Accountability, Action, Aggregate, Filter, Query, SchemaOverview, PrimaryKey } from '@directus/shared/types';
import { Accountability, Action, Aggregate, Filter, PrimaryKey, Query, SchemaOverview } from '@directus/shared/types';
import { parseFilterFunctionPath } from '@directus/shared/utils';
import argon2 from 'argon2';
import {
@@ -36,15 +36,13 @@ import {
InputTypeComposer,
InputTypeComposerFieldConfigMapDefinition,
ObjectTypeComposer,
ObjectTypeComposerFieldConfigDefinition,
ObjectTypeComposerFieldConfigMapDefinition,
SchemaComposer,
toInputObjectType,
ObjectTypeComposerFieldConfigDefinition,
} from 'graphql-compose';
import processError from './utils/process-error';
import { Knex } from 'knex';
import { flatten, get, mapKeys, merge, omit, pick, set, transform, uniq } from 'lodash';
import ms from 'ms';
import { clearSystemCache, getCache } from '../../cache';
import { DEFAULT_AUTH_PROVIDER, GENERATE_SPECIAL } from '../../constants';
import getDatabase from '../../database';
@@ -80,16 +78,18 @@ import { TFAService } from '../tfa';
import { UsersService } from '../users';
import { UtilsService } from '../utils';
import { WebhooksService } from '../webhooks';
import processError from './utils/process-error';
import { GraphQLDate } from './types/date';
import { GraphQLGeoJSON } from './types/geojson';
import { GraphQLStringOrFloat } from './types/string-or-float';
import { GraphQLVoid } from './types/void';
import { addPathToValidationError } from './utils/add-path-to-validation-error';
import { GraphQLHash } from './types/hash';
import { GraphQLBigInt } from './types/bigint';
import { FUNCTIONS } from '@directus/shared/constants';
import { getMilliseconds } from '../../utils/get-milliseconds';
import { GraphQLBigInt } from './types/bigint';
import { GraphQLHash } from './types/hash';
import { addPathToValidationError } from './utils/add-path-to-validation-error';
const validationRules = Array.from(specifiedRules);
@@ -2000,7 +2000,7 @@ export class GraphQLService {
res?.cookie(env.REFRESH_TOKEN_COOKIE_NAME, result.refreshToken, {
httpOnly: true,
domain: env.REFRESH_TOKEN_COOKIE_DOMAIN,
maxAge: ms(env.REFRESH_TOKEN_TTL as string),
maxAge: getMilliseconds(env.REFRESH_TOKEN_TTL),
secure: env.REFRESH_TOKEN_COOKIE_SECURE ?? false,
sameSite: (env.REFRESH_TOKEN_COOKIE_SAME_SITE as 'lax' | 'strict' | 'none') || 'strict',
});
@@ -2038,7 +2038,7 @@ export class GraphQLService {
res?.cookie(env.REFRESH_TOKEN_COOKIE_NAME, result.refreshToken, {
httpOnly: true,
domain: env.REFRESH_TOKEN_COOKIE_DOMAIN,
maxAge: ms(env.REFRESH_TOKEN_TTL as string),
maxAge: getMilliseconds(env.REFRESH_TOKEN_TTL),
secure: env.REFRESH_TOKEN_COOKIE_SECURE ?? false,
sameSite: (env.REFRESH_TOKEN_COOKIE_SAME_SITE as 'lax' | 'strict' | 'none') || 'strict',
});

View File

@@ -1,24 +1,24 @@
import {
AbstractServiceOptions,
ShareData,
LoginResult,
Item,
PrimaryKey,
MutationOptions,
DirectusTokenPayload,
} from '../types';
import { ItemsService } from './items';
import argon2 from 'argon2';
import jwt from 'jsonwebtoken';
import ms from 'ms';
import { InvalidCredentialsException, ForbiddenException } from '../exceptions';
import env from '../env';
import { AuthorizationService } from './authorization';
import { UsersService } from './users';
import { MailService } from './mail';
import { userName } from '../utils/user-name';
import { ForbiddenException, InvalidCredentialsException } from '../exceptions';
import {
AbstractServiceOptions,
DirectusTokenPayload,
Item,
LoginResult,
MutationOptions,
PrimaryKey,
ShareData,
} from '../types';
import { getMilliseconds } from '../utils/get-milliseconds';
import { md } from '../utils/md';
import { Url } from '../utils/url';
import { userName } from '../utils/user-name';
import { AuthorizationService } from './authorization';
import { ItemsService } from './items';
import { MailService } from './mail';
import { UsersService } from './users';
export class SharesService extends ItemsService {
authorizationService: AuthorizationService;
@@ -95,7 +95,7 @@ export class SharesService extends ItemsService {
});
const refreshToken = nanoid(64);
const refreshTokenExpiration = new Date(Date.now() + ms(env.REFRESH_TOKEN_TTL as string));
const refreshTokenExpiration = new Date(Date.now() + getMilliseconds(env.REFRESH_TOKEN_TTL, 0));
await this.knex('directus_sessions').insert({
token: refreshToken,
@@ -111,7 +111,7 @@ export class SharesService extends ItemsService {
return {
accessToken,
refreshToken,
expires: ms(env.ACCESS_TOKEN_TTL as string),
expires: getMilliseconds(env.ACCESS_TOKEN_TTL),
};
}

View File

@@ -56,12 +56,11 @@ const scenarios = [
// Test the ttl value
{
name: 'when ttl is null',
name: 'when ttl is undefined',
input: {
env: {},
headers: {},
accountability: null,
ttl: null,
globalCacheSettings: false,
personalized: false,
},

View File

@@ -11,7 +11,7 @@ import { Request } from 'express';
*/
export function getCacheControlHeader(
req: Request,
ttl: number | null,
ttl: number | undefined,
globalCacheSettings: boolean,
personalized: boolean
): string {
@@ -22,7 +22,7 @@ export function getCacheControlHeader(
if (noCacheRequested) return 'no-store';
// When the resource / current request shouldn't be cached
if (ttl === null || ttl < 0) return 'no-cache';
if (ttl === undefined || ttl < 0) return 'no-cache';
// When the API cache can invalidate at any moment
if (globalCacheSettings && env.CACHE_AUTO_PURGE === true) return 'no-cache';

View File

@@ -0,0 +1,34 @@
import { expect, test } from 'vitest';
import { getMilliseconds } from './get-milliseconds';
test.each([
// accept human readable time format and plain number
['1d', 86400000],
['1000', 1000],
[1000, 1000],
// accept negative values
['-1 minutes', -60000],
[-1, -1],
[0, 0],
// fallback to undefined
[null, undefined],
[undefined, undefined],
['', undefined],
['invalid string', undefined],
[false, undefined],
[[], undefined],
[{}, undefined],
[Symbol(123), undefined],
[
() => {
return 456;
},
undefined,
],
])('should result into %s for input "%s"', (input, expected) => {
expect(getMilliseconds(input)).toBe(expected);
});
test('should return custom fallback on invalid value', () => {
expect(getMilliseconds(undefined, 0)).toBe(0);
});

View File

@@ -0,0 +1,12 @@
import ms from 'ms';
/**
* Safely parse human readable time format into milliseconds
*/
export function getMilliseconds<T>(value: unknown, fallback?: T): number | T;
export function getMilliseconds(value: unknown, fallback = undefined): number | undefined {
if ((typeof value !== 'string' && typeof value !== 'number') || value === '') {
return fallback;
}
return ms(String(value)) ?? fallback;
}

View File

@@ -1,11 +1,11 @@
import ms from 'ms';
import { machineId } from 'node-machine-id';
import os from 'os';
// @ts-ignore
import { toArray } from '@directus/shared/utils';
import { version } from '../../package.json';
import env from '../env';
import logger from '../logger';
import { toArray } from '@directus/shared/utils';
import { getMilliseconds } from './get-milliseconds';
export async function track(event: string): Promise<void> {
const axios = (await import('axios')).default;
@@ -44,7 +44,7 @@ async function getEnvInfo(event: string) {
},
cache: {
enabled: env.CACHE_ENABLED,
ttl: ms(env.CACHE_TTL),
ttl: getMilliseconds(env.CACHE_TTL),
store: env.CACHE_STORE,
},
storage: {