From 89fcb19a167db967883bab9672a50e4fd9978271 Mon Sep 17 00:00:00 2001 From: Azri Kahar <42867097+azrikahar@users.noreply.github.com> Date: Wed, 1 Feb 2023 01:36:49 +0800 Subject: [PATCH] Expose schema endpoints (#14713) * Expose schema endpoints * respond with 204 * fix payload for export flag * allow export as YAML * use SchemaService * move getStringFromStream to utils * clearer exceptions with logs * check empty request body * specs * add bypassCache flag and remove flushCaches calls * Identify integer typed fields in Oracle * Fix uuid foreign keys type for Oracle * Detect autoincrement for CockroachDB * Bypass cache for fields and relations services * Add same DB tests * Test if working for all vendors * Add schema snapshot testing into sequential flow * Fix schema re-apply attempts when it's a different database vendor (#14816) * prevent diffing on certain properties * fix apply-snapshot test * prevent updates to autoincrement fields' schema * sanitize relations * remove `colleciton.schema.sql` parsing in test It is now being sanitized by `sanitizeCollection()` function in get-snapshot-diff.ts * Change concurrency group to not cancel test running on main * remove multipart for apply & add hash checking * check incoming version & add force query param * refine error message & stub for nested hash * add `vendor` top-level property as an additional safety net for cross db vendor applies * sanitize generated schema snapshots * snapshot joi validation * diff joi validation * minor cleanup * extract applyDiff & use deep-diff applyChange * use applyDiff in schema service * Mark vendor as optional * Update tests to apply diffs * move force flag check into service * Patch mssql vendor typo * Set relation no action in cyclic relations * Update mysql timestamp default value * Oracle cannot define no action * Update oracle timestamp type default values * add hash checking for outdated diffs * fix diff service & endpoint * Add hashes when returning after * Fix self referencing relations for Oracle * Add temp fix for CURRENT_TIMESTAMP defaults in Oracle * clean up driver and database client types * only require diff hash if not kind N * update hash comparison logic for create/delete * Set no action constraint for self referencing M2M in MSSQL * Add basic hash check tests * omit default value when hashing auto increments Specifically for CockroachDB with differing sequences * add vendor check * update specs * Validate vendors with type definition * Spread the vendors input array * re-add Export component * re-add js-yaml to root * Propagate mutation options for schema apply * Verify that snapshots differ and clear cache before hash tests * Fix unit test * Revert temp fix for CURRENT_TIMESTAMP defaults in Oracle * Define and reuse type for export format * Define and reuse list of database clients * change `were` to `was` * change `where` to `were` * add some empty lines for readability * tweak exception message * fix test * use object-hash * use boolean to check whether file is included * simplify request content type check * throw error when multiple files were uploaded * use nullish coalesce instead of short circuit * Update api/src/services/schema.ts Co-authored-by: Rijk van Zanten * Remove unnecessary `Omit` on `SnapshotWithHash` type * Revert "Remove unnecessary `Omit` on `SnapshotWithHash` type" This reverts commit d22ac771ec55f07d4b5373204ad803126750f0c9. * check empty snapshot earlier * use allow-list logic via pick instead of omit * Update api/src/services/schema.ts Co-authored-by: Pascal Jufer * Move to own /schema endpoint controller * Fix refs to schema endpoint * move streamToString to utils package * move get-versioned-hash and add test * extract kind into an enum * Fix mysql5 timestamp value * Fix test collection existing on rerun * resolve TODO comment in blackbox test * Drop deep level hashes in diff These hashes are used only for more accurate error reporting but are otherwise superfluous, since changes are already detected by the top level hash. Therefore we remove them in favor of a simpler diff format and implementation. * Revert schema "fix" for createItem, add comment * Strict diff schema validation * Revert CrDB auto-increment detection patch in 816c998 * Clear systemCache to expose newly added fields * Use DiffKind constants * Extract diff & snapshot validation into own tested utils * Apply suggestions from @azrikahar * Update knex-schema-inspector to 3.0.1 Includes the fix for CrDB auto-increment detection (knex/knex-schema-inspector#135) * Update knex-schema-inspector in packages * Update lock file * add test for schema service * add test for export service * add relevant tests to modified util functions * fix csv test to account for os end of line * fix files controller test * dedupe test data for schema service * Align schema specs with docs * Update api/src/controllers/schema.ts * Revert testing for all vendors --------- Co-authored-by: Rijk van Zanten Co-authored-by: ian Co-authored-by: Pascal Jufer --- .gitignore | 4 +- api/package.json | 3 +- api/src/app.ts | 2 + api/src/cli/commands/schema/apply.ts | 39 +- api/src/cli/utils/create-db-connection.ts | 6 +- api/src/cli/utils/drivers.ts | 8 +- api/src/controllers/files.test.ts | 8 + api/src/controllers/files.ts | 2 +- api/src/controllers/schema.ts | 120 ++++ .../helpers/schema/dialects/oracle.ts | 25 + api/src/database/helpers/schema/types.ts | 18 +- api/src/database/index.ts | 5 +- api/src/middleware/respond.ts | 6 + api/src/services/fields.ts | 2 + api/src/services/import-export.test.ts | 68 ++- api/src/services/import-export.ts | 14 +- api/src/services/index.ts | 1 + api/src/services/relations.ts | 4 + api/src/services/schema.test.ts | 188 ++++++ api/src/services/schema.ts | 67 +++ api/src/types/database.ts | 4 + api/src/types/index.ts | 1 + api/src/types/snapshot.ts | 20 + api/src/utils/apply-diff.test.ts | 28 + api/src/utils/apply-diff.ts | 317 ++++++++++ api/src/utils/apply-snapshot.test.ts | 15 +- api/src/utils/apply-snapshot.ts | 291 +-------- api/src/utils/get-snapshot-diff.ts | 23 +- api/src/utils/get-snapshot.ts | 13 +- api/src/utils/get-versioned-hash.test.ts | 118 ++++ api/src/utils/get-versioned-hash.ts | 6 + api/src/utils/sanitize-schema.test.ts | 347 +++++++++++ api/src/utils/sanitize-schema.ts | 81 +++ api/src/utils/validate-diff.test.ts | 152 +++++ api/src/utils/validate-diff.ts | 150 +++++ api/src/utils/validate-query.test.ts | 24 + api/src/utils/validate-query.ts | 2 +- api/src/utils/validate-snapshot.test.ts | 55 ++ api/src/utils/validate-snapshot.ts | 80 +++ app/src/lang/translations/en-US.yaml | 1 + .../components/export-sidebar-detail.vue | 4 + package.json | 2 + packages/schema/package.json | 2 +- packages/shared/package.json | 2 +- packages/specs/src/components/diff.yaml | 46 ++ packages/specs/src/components/schema.yaml | 21 + packages/specs/src/openapi.yaml | 16 + packages/specs/src/parameters/export.yaml | 11 + packages/specs/src/paths/items/items.yaml | 1 + packages/specs/src/paths/schema/apply.yaml | 22 + packages/specs/src/paths/schema/diff.yaml | 45 ++ packages/specs/src/paths/schema/snapshot.yaml | 24 + packages/utils/src/node/index.ts | 1 + .../node/readable-stream-to-string.test.ts | 11 + .../src/node/readable-stream-to-string.ts | 11 + pnpm-lock.yaml | 30 +- tests-blackbox/common/config.ts | 1 + tests-blackbox/common/functions.ts | 4 +- tests-blackbox/common/seed-functions.ts | 40 +- .../routes/items/seed-all-field-types.ts | 26 +- tests-blackbox/routes/schema/schema.seed.ts | 250 ++++++++ tests-blackbox/routes/schema/schema.test.ts | 559 ++++++++++++++++++ tests-blackbox/setup/sequentialTests.js | 1 + 63 files changed, 3072 insertions(+), 376 deletions(-) create mode 100644 api/src/controllers/schema.ts create mode 100644 api/src/services/schema.test.ts create mode 100644 api/src/services/schema.ts create mode 100644 api/src/types/database.ts create mode 100644 api/src/utils/apply-diff.test.ts create mode 100644 api/src/utils/apply-diff.ts create mode 100644 api/src/utils/get-versioned-hash.test.ts create mode 100644 api/src/utils/get-versioned-hash.ts create mode 100644 api/src/utils/sanitize-schema.test.ts create mode 100644 api/src/utils/sanitize-schema.ts create mode 100644 api/src/utils/validate-diff.test.ts create mode 100644 api/src/utils/validate-diff.ts create mode 100644 api/src/utils/validate-query.test.ts create mode 100644 api/src/utils/validate-snapshot.test.ts create mode 100644 api/src/utils/validate-snapshot.ts create mode 100644 packages/specs/src/components/diff.yaml create mode 100644 packages/specs/src/components/schema.yaml create mode 100644 packages/specs/src/parameters/export.yaml create mode 100644 packages/specs/src/paths/schema/apply.yaml create mode 100644 packages/specs/src/paths/schema/diff.yaml create mode 100644 packages/specs/src/paths/schema/snapshot.yaml create mode 100644 packages/utils/src/node/readable-stream-to-string.test.ts create mode 100644 packages/utils/src/node/readable-stream-to-string.ts create mode 100644 tests-blackbox/routes/schema/schema.seed.ts create mode 100644 tests-blackbox/routes/schema/schema.test.ts diff --git a/.gitignore b/.gitignore index 832008f29e..2c27a2ccd4 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,8 @@ dist .e2e-containers.json coverage TODO -schema.yaml -schema.json +/schema.yaml +/schema.json .*.swp debug debug.ts diff --git a/api/package.json b/api/package.json index d37c4e27e2..2ae7cf6aec 100644 --- a/api/package.json +++ b/api/package.json @@ -87,6 +87,7 @@ "@directus/storage-driver-gcs": "workspace:*", "@directus/storage-driver-local": "workspace:*", "@directus/storage-driver-s3": "workspace:*", + "@directus/utils": "workspace:*", "@godaddy/terminus": "4.11.2", "@rollup/plugin-alias": "4.0.2", "@rollup/plugin-virtual": "3.0.1", @@ -129,7 +130,7 @@ "jsonwebtoken": "9.0.0", "keyv": "4.5.2", "knex": "2.4.1", - "knex-schema-inspector": "3.0.0", + "knex-schema-inspector": "3.0.1", "ldapjs": "2.3.3", "liquidjs": "10.3.3", "lodash": "4.17.21", diff --git a/api/src/app.ts b/api/src/app.ts index 4f33a954cb..a090758368 100644 --- a/api/src/app.ts +++ b/api/src/app.ts @@ -26,6 +26,7 @@ import presetsRouter from './controllers/presets'; import relationsRouter from './controllers/relations'; import revisionsRouter from './controllers/revisions'; import rolesRouter from './controllers/roles'; +import schemaRouter from './controllers/schema'; import serverRouter from './controllers/server'; import settingsRouter from './controllers/settings'; import usersRouter from './controllers/users'; @@ -253,6 +254,7 @@ export default async function createApp(): Promise { app.use('/relations', relationsRouter); app.use('/revisions', revisionsRouter); app.use('/roles', rolesRouter); + app.use('/schema', schemaRouter); app.use('/server', serverRouter); app.use('/settings', settingsRouter); app.use('/shares', sharesRouter); diff --git a/api/src/cli/commands/schema/apply.ts b/api/src/cli/commands/schema/apply.ts index e5e281dde5..9976b12463 100644 --- a/api/src/cli/commands/schema/apply.ts +++ b/api/src/cli/commands/schema/apply.ts @@ -6,8 +6,9 @@ import { load as loadYaml } from 'js-yaml'; import path from 'path'; import getDatabase, { isInstalled, validateDatabaseConnection } from '../../../database'; import logger from '../../../logger'; -import { Snapshot } from '../../../types'; -import { applySnapshot, isNestedMetaUpdate } from '../../../utils/apply-snapshot'; +import { DiffKind, Snapshot } from '../../../types'; +import { isNestedMetaUpdate } from '../../../utils/apply-diff'; +import { applySnapshot } from '../../../utils/apply-snapshot'; import { getSnapshot } from '../../../utils/get-snapshot'; import { getSnapshotDiff } from '../../../utils/get-snapshot-diff'; @@ -57,20 +58,20 @@ export async function apply(snapshotPath: string, options?: { yes: boolean; dryR message += chalk.black.underline.bold('Collections:'); for (const { collection, diff } of snapshotDiff.collections) { - if (diff[0]?.kind === 'E') { + if (diff[0]?.kind === DiffKind.EDIT) { message += `\n - ${chalk.blue('Update')} ${collection}`; for (const change of diff) { - if (change.kind === 'E') { + if (change.kind === DiffKind.EDIT) { const path = change.path!.slice(1).join('.'); message += `\n - Set ${path} to ${change.rhs}`; } } - } else if (diff[0]?.kind === 'D') { + } else if (diff[0]?.kind === DiffKind.DELETE) { message += `\n - ${chalk.red('Delete')} ${collection}`; - } else if (diff[0]?.kind === 'N') { + } else if (diff[0]?.kind === DiffKind.NEW) { message += `\n - ${chalk.green('Create')} ${collection}`; - } else if (diff[0]?.kind === 'A') { + } else if (diff[0]?.kind === DiffKind.ARRAY) { message += `\n - ${chalk.blue('Update')} ${collection}`; } } @@ -80,24 +81,24 @@ export async function apply(snapshotPath: string, options?: { yes: boolean; dryR message += '\n\n' + chalk.black.underline.bold('Fields:'); for (const { collection, field, diff } of snapshotDiff.fields) { - if (diff[0]?.kind === 'E' || isNestedMetaUpdate(diff[0])) { + if (diff[0]?.kind === DiffKind.EDIT || isNestedMetaUpdate(diff[0])) { message += `\n - ${chalk.blue('Update')} ${collection}.${field}`; for (const change of diff) { const path = change.path!.slice(1).join('.'); - if (change.kind === 'E') { + if (change.kind === DiffKind.EDIT) { message += `\n - Set ${path} to ${change.rhs}`; - } else if (change.kind === 'D') { + } else if (change.kind === DiffKind.DELETE) { message += `\n - Remove ${path}`; - } else if (change.kind === 'N') { + } else if (change.kind === DiffKind.NEW) { message += `\n - Add ${path} and set it to ${change.rhs}`; } } - } else if (diff[0]?.kind === 'D') { + } else if (diff[0]?.kind === DiffKind.DELETE) { message += `\n - ${chalk.red('Delete')} ${collection}.${field}`; - } else if (diff[0]?.kind === 'N') { + } else if (diff[0]?.kind === DiffKind.NEW) { message += `\n - ${chalk.green('Create')} ${collection}.${field}`; - } else if (diff[0]?.kind === 'A') { + } else if (diff[0]?.kind === DiffKind.ARRAY) { message += `\n - ${chalk.blue('Update')} ${collection}.${field}`; } } @@ -107,20 +108,20 @@ export async function apply(snapshotPath: string, options?: { yes: boolean; dryR message += '\n\n' + chalk.black.underline.bold('Relations:'); for (const { collection, field, related_collection, diff } of snapshotDiff.relations) { - if (diff[0]?.kind === 'E') { + if (diff[0]?.kind === DiffKind.EDIT) { message += `\n - ${chalk.blue('Update')} ${collection}.${field}`; for (const change of diff) { - if (change.kind === 'E') { + if (change.kind === DiffKind.EDIT) { const path = change.path!.slice(1).join('.'); message += `\n - Set ${path} to ${change.rhs}`; } } - } else if (diff[0]?.kind === 'D') { + } else if (diff[0]?.kind === DiffKind.DELETE) { message += `\n - ${chalk.red('Delete')} ${collection}.${field}`; - } else if (diff[0]?.kind === 'N') { + } else if (diff[0]?.kind === DiffKind.NEW) { message += `\n - ${chalk.green('Create')} ${collection}.${field}`; - } else if (diff[0]?.kind === 'A') { + } else if (diff[0]?.kind === DiffKind.ARRAY) { message += `\n - ${chalk.blue('Update')} ${collection}.${field}`; } else { continue; diff --git a/api/src/cli/utils/create-db-connection.ts b/api/src/cli/utils/create-db-connection.ts index e03e414b07..b97d51e146 100644 --- a/api/src/cli/utils/create-db-connection.ts +++ b/api/src/cli/utils/create-db-connection.ts @@ -1,6 +1,7 @@ import { knex, Knex } from 'knex'; import path from 'path'; import { promisify } from 'util'; +import { Driver } from '../../types'; export type Credentials = { filename?: string; @@ -12,10 +13,7 @@ export type Credentials = { ssl?: boolean; options__encrypt?: boolean; }; -export default function createDBConnection( - client: 'sqlite3' | 'mysql' | 'pg' | 'oracledb' | 'mssql' | 'cockroachdb', - credentials: Credentials -): Knex { +export default function createDBConnection(client: Driver, credentials: Credentials): Knex { let connection: Knex.Config['connection'] = {}; if (client === 'sqlite3') { diff --git a/api/src/cli/utils/drivers.ts b/api/src/cli/utils/drivers.ts index c5d03258cb..271c57102c 100644 --- a/api/src/cli/utils/drivers.ts +++ b/api/src/cli/utils/drivers.ts @@ -1,4 +1,6 @@ -export const drivers = { +import { Driver } from '../../types'; + +export const drivers: Record = { pg: 'PostgreSQL / Redshift', cockroachdb: 'CockroachDB (Beta)', mysql: 'MySQL / MariaDB / Aurora', @@ -7,9 +9,9 @@ export const drivers = { oracledb: 'Oracle Database', }; -export function getDriverForClient(client: string): keyof typeof drivers | null { +export function getDriverForClient(client: string): Driver | null { for (const [key, value] of Object.entries(drivers)) { - if (value === client) return key as keyof typeof drivers; + if (value === client) return key as Driver; } return null; diff --git a/api/src/controllers/files.test.ts b/api/src/controllers/files.test.ts index a9540a81df..fb48b951a9 100644 --- a/api/src/controllers/files.test.ts +++ b/api/src/controllers/files.test.ts @@ -7,6 +7,14 @@ import { Request, Response } from 'express'; vi.mock('../../src/database'); +vi.mock('../services', () => { + const FilesService = vi.fn(); + FilesService.prototype.uploadOne = vi.fn(); + const MetaService = vi.fn(); + MetaService.prototype.getMetaForQuery = vi.fn().mockResolvedValue({}); + return { FilesService }; +}); + describe('multipartHandler', () => { it(`Errors out if request doesn't contain any files to upload`, () => { const fakeForm = new FormData(); diff --git a/api/src/controllers/files.ts b/api/src/controllers/files.ts index 21bd39473e..ebd38ff897 100644 --- a/api/src/controllers/files.ts +++ b/api/src/controllers/files.ts @@ -110,7 +110,7 @@ export const multipartHandler: RequestHandler = (req, res, next) => { function tryDone() { if (savedFiles.length === fileCount) { if (fileCount === 0) { - return next(new InvalidPayloadException(`No files where included in the body`)); + return next(new InvalidPayloadException(`No files were included in the body`)); } res.locals.savedFiles = savedFiles; diff --git a/api/src/controllers/schema.ts b/api/src/controllers/schema.ts new file mode 100644 index 0000000000..e651150f1e --- /dev/null +++ b/api/src/controllers/schema.ts @@ -0,0 +1,120 @@ +import { parseJSON } from '@directus/shared/utils'; +import Busboy from 'busboy'; +import express, { RequestHandler } from 'express'; +import { load as loadYaml } from 'js-yaml'; +import { InvalidPayloadException, UnsupportedMediaTypeException } from '../exceptions'; +import logger from '../logger'; +import { respond } from '../middleware/respond'; +import { SchemaService } from '../services/schema'; +import { Snapshot } from '../types'; +import asyncHandler from '../utils/async-handler'; +import { getVersionedHash } from '../utils/get-versioned-hash'; + +const router = express.Router(); + +router.get( + '/snapshot', + asyncHandler(async (req, res, next) => { + const service = new SchemaService({ accountability: req.accountability }); + const currentSnapshot = await service.snapshot(); + res.locals.payload = { data: currentSnapshot }; + return next(); + }), + respond +); + +router.post( + '/apply', + asyncHandler(async (req, _res, next) => { + const service = new SchemaService({ accountability: req.accountability }); + await service.apply(req.body); + return next(); + }), + respond +); + +const schemaMultipartHandler: RequestHandler = (req, res, next) => { + if (req.is('application/json')) { + if (Object.keys(req.body).length === 0) throw new InvalidPayloadException(`No data was included in the body`); + res.locals.uploadedSnapshot = req.body; + return next(); + } + + if (!req.is('multipart/form-data')) throw new UnsupportedMediaTypeException(`Unsupported Content-Type header`); + + const headers = req.headers['content-type'] + ? req.headers + : { + ...req.headers, + 'content-type': 'application/octet-stream', + }; + + const busboy = Busboy({ headers }); + + let isFileIncluded = false; + let uploadedSnapshot: Snapshot | null = null; + + busboy.on('file', async (_, fileStream, { mimeType }) => { + if (isFileIncluded) return next(new InvalidPayloadException(`More than one file was included in the body`)); + + isFileIncluded = true; + + const { readableStreamToString } = await import('@directus/utils/node'); + + try { + const uploadedString = await readableStreamToString(fileStream); + + if (mimeType === 'application/json') { + try { + uploadedSnapshot = parseJSON(uploadedString); + } catch (err: any) { + logger.warn(err); + throw new InvalidPayloadException('Invalid JSON schema snapshot'); + } + } else { + try { + uploadedSnapshot = (await loadYaml(uploadedString)) as Snapshot; + } catch (err: any) { + logger.warn(err); + throw new InvalidPayloadException('Invalid YAML schema snapshot'); + } + } + + if (!uploadedSnapshot) throw new InvalidPayloadException(`No file was included in the body`); + + res.locals.uploadedSnapshot = uploadedSnapshot; + + return next(); + } catch (error: any) { + busboy.emit('error', error); + } + }); + + busboy.on('error', (error: Error) => next(error)); + + busboy.on('close', () => { + if (!isFileIncluded) return next(new InvalidPayloadException(`No file was included in the body`)); + }); + + req.pipe(busboy); +}; + +router.post( + '/diff', + asyncHandler(schemaMultipartHandler), + asyncHandler(async (req, res, next) => { + const service = new SchemaService({ accountability: req.accountability }); + const snapshot: Snapshot = res.locals.uploadedSnapshot; + + const currentSnapshot = await service.snapshot(); + const snapshotDiff = await service.diff(snapshot, { currentSnapshot, force: 'force' in req.query }); + if (!snapshotDiff) return next(); + + const currentSnapshotHash = getVersionedHash(currentSnapshot); + res.locals.payload = { data: { hash: currentSnapshotHash, diff: snapshotDiff } }; + return next(); + }), + respond +); + +export default router; diff --git a/api/src/database/helpers/schema/dialects/oracle.ts b/api/src/database/helpers/schema/dialects/oracle.ts index 7d66e1ce39..70741385e4 100644 --- a/api/src/database/helpers/schema/dialects/oracle.ts +++ b/api/src/database/helpers/schema/dialects/oracle.ts @@ -1,4 +1,5 @@ import { KNEX_TYPES } from '@directus/shared/constants'; +import { Field, Relation, Type } from '@directus/shared/types'; import { Options, SchemaHelper } from '../types'; export class SchemaHelperOracle extends SchemaHelper { @@ -14,4 +15,28 @@ export class SchemaHelperOracle extends SchemaHelper { castA2oPrimaryKey(): string { return 'CAST(?? AS VARCHAR2(255))'; } + + preRelationChange(relation: Partial): void { + if (relation.collection === relation.related_collection) { + // Constraints are not allowed on self referencing relationships + // Setting NO ACTION throws - ORA-00905: missing keyword + if (relation.schema?.on_delete) { + relation.schema.on_delete = null; + } + } + } + + processFieldType(field: Field): Type { + if (field.type === 'integer') { + if (field.schema?.numeric_precision === 20) { + return 'bigInteger'; + } else if (field.schema?.numeric_precision === 1) { + return 'boolean'; + } else if (field.schema?.numeric_precision || field.schema?.numeric_scale) { + return 'decimal'; + } + } + + return field.type; + } } diff --git a/api/src/database/helpers/schema/types.ts b/api/src/database/helpers/schema/types.ts index 4113fc81f9..bd1e7ef5e1 100644 --- a/api/src/database/helpers/schema/types.ts +++ b/api/src/database/helpers/schema/types.ts @@ -1,14 +1,14 @@ +import { KNEX_TYPES } from '@directus/shared/constants'; +import { Field, Relation, Type } from '@directus/shared/types'; +import { Knex } from 'knex'; +import { DatabaseClient } from '../../../types'; import { getDatabaseClient } from '../../index'; import { DatabaseHelper } from '../types'; -import { KNEX_TYPES } from '@directus/shared/constants'; -import { Knex } from 'knex'; - -type Clients = 'mysql' | 'postgres' | 'cockroachdb' | 'sqlite' | 'oracle' | 'mssql' | 'redshift'; export type Options = { nullable?: boolean; default?: any; length?: number }; export abstract class SchemaHelper extends DatabaseHelper { - isOneOfClients(clients: Clients[]): boolean { + isOneOfClients(clients: DatabaseClient[]): boolean { return clients.includes(getDatabaseClient(this.knex)); } @@ -92,6 +92,14 @@ export abstract class SchemaHelper extends DatabaseHelper { return; } + preRelationChange(_relation: Partial): void { + return; + } + + processFieldType(field: Field): Type { + return field.type; + } + constraintName(existingName: string): string { // most vendors allow for dropping/creating constraints with the same name // reference issue #14873 diff --git a/api/src/database/index.ts b/api/src/database/index.ts index 8e96b66bf3..47c900c345 100644 --- a/api/src/database/index.ts +++ b/api/src/database/index.ts @@ -10,6 +10,7 @@ import path from 'path'; import { merge } from 'lodash'; import { promisify } from 'util'; import { getHelpers } from './helpers'; +import { DatabaseClient } from '../types'; let database: Knex | null = null; let inspector: ReturnType | null = null; @@ -201,9 +202,7 @@ export async function validateDatabaseConnection(database?: Knex): Promise } } -export function getDatabaseClient( - database?: Knex -): 'mysql' | 'postgres' | 'cockroachdb' | 'sqlite' | 'oracle' | 'mssql' | 'redshift' { +export function getDatabaseClient(database?: Knex): DatabaseClient { database = database ?? getDatabase(); switch (database.client.constructor.name) { diff --git a/api/src/middleware/respond.ts b/api/src/middleware/respond.ts index 90f46abad2..c5c06ed592 100644 --- a/api/src/middleware/respond.ts +++ b/api/src/middleware/respond.ts @@ -77,6 +77,12 @@ export const respond: RequestHandler = asyncHandler(async (req, res) => { res.set('Content-Type', 'text/csv'); return res.status(200).send(exportService.transform(res.locals.payload?.data, 'csv')); } + + if (req.sanitizedQuery.export === 'yaml') { + res.attachment(`${filename}.yaml`); + res.set('Content-Type', 'text/yaml'); + return res.status(200).send(exportService.transform(res.locals.payload?.data, 'yaml')); + } } if (Buffer.isBuffer(res.locals.payload)) { diff --git a/api/src/services/fields.ts b/api/src/services/fields.ts index dbd5552d8e..1a6b367f90 100644 --- a/api/src/services/fields.ts +++ b/api/src/services/fields.ts @@ -178,6 +178,8 @@ export class FieldsService { } else if (field.meta?.special?.includes('cast-datetime')) { field.type = 'dateTime'; } + + field.type = this.helpers.schema.processFieldType(field); } return result; diff --git a/api/src/services/import-export.test.ts b/api/src/services/import-export.test.ts index 5e5d9952b0..dd0a4761d1 100644 --- a/api/src/services/import-export.test.ts +++ b/api/src/services/import-export.test.ts @@ -1,10 +1,12 @@ -import knex, { Knex } from 'knex'; -import { MockClient, Tracker, getTracker } from 'knex-mock-client'; -import { ImportService } from '.'; -import { describe, beforeAll, afterEach, it, expect, vi, beforeEach, MockedFunction } from 'vitest'; -import { Readable } from 'stream'; -import emitter from '../emitter'; import { parse } from 'json2csv'; +import knex, { Knex } from 'knex'; +import { getTracker, MockClient, Tracker } from 'knex-mock-client'; +import { EOL } from 'node:os'; +import { Readable } from 'stream'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, MockedFunction, vi } from 'vitest'; +import { ExportService, ImportService } from '.'; +import { ServiceUnavailableException } from '..'; +import emitter from '../emitter'; vi.mock('../../src/database/index', () => ({ default: vi.fn(), @@ -116,4 +118,58 @@ describe('Integration Tests', () => { }); }); }); + + describe('Services / ExportService', () => { + describe('transform', () => { + it('should return json string with header and footer', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(service.transform(input, 'json')).toBe(`[\n\t{\n\t\t"key": "value"\n\t}\n]`); + }); + + it('should return xml string with header and footer', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(service.transform(input, 'xml')).toBe( + `\n\n \n value\n \n` + ); + }); + + it('should return csv string with header', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(service.transform(input, 'csv')).toBe(`"key"${EOL}"value"`); + }); + + it('should return csv string without header', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(service.transform(input, 'csv', { includeHeader: false })).toBe('\n"value"'); + }); + + it('should return yaml string', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(service.transform(input, 'yaml')).toBe('- key: value\n'); + }); + + it('should throw ServiceUnavailableException error when using a non-existent export type', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(() => service.transform(input, 'invalid-format' as any)).toThrowError(ServiceUnavailableException); + }); + }); + }); }); diff --git a/api/src/services/import-export.ts b/api/src/services/import-export.ts index 9be002480a..ec3b02f05d 100644 --- a/api/src/services/import-export.ts +++ b/api/src/services/import-export.ts @@ -4,6 +4,7 @@ import { queue } from 'async'; import csv from 'csv-parser'; import destroyStream from 'destroy'; import { appendFile, createReadStream } from 'fs-extra'; +import { dump as toYAML } from 'js-yaml'; import { parse as toXML } from 'js2xmlparser'; import { Parser as CSVParser, transforms as CSVTransforms } from 'json2csv'; import { Knex } from 'knex'; @@ -28,6 +29,8 @@ import { NotificationsService } from './notifications'; import emitter from '../emitter'; import type { Readable } from 'node:stream'; +type ExportFormat = 'csv' | 'json' | 'xml' | 'yaml'; + export class ImportService { knex: Knex; accountability: Accountability | null; @@ -190,16 +193,17 @@ export class ExportService { async exportToFile( collection: string, query: Partial, - format: 'xml' | 'csv' | 'json', + format: ExportFormat, options?: { file?: Partial; } ) { try { const mimeTypes = { - xml: 'text/xml', csv: 'text/csv', json: 'application/json', + xml: 'text/xml', + yaml: 'text/yaml', }; const database = getDatabase(); @@ -316,7 +320,7 @@ export class ExportService { */ transform( input: Record[], - format: 'xml' | 'csv' | 'json', + format: ExportFormat, options?: { includeHeader?: boolean; includeFooter?: boolean; @@ -366,6 +370,10 @@ export class ExportService { return string; } + if (format === 'yaml') { + return toYAML(input); + } + throw new ServiceUnavailableException(`Illegal export type used: "${format}"`, { service: 'export' }); } } diff --git a/api/src/services/index.ts b/api/src/services/index.ts index 29c82c9980..1bb8115dfe 100644 --- a/api/src/services/index.ts +++ b/api/src/services/index.ts @@ -23,6 +23,7 @@ export * from './presets'; export * from './relations'; export * from './revisions'; export * from './roles'; +export * from './schema'; export * from './server'; export * from './settings'; export * from './specifications'; diff --git a/api/src/services/relations.ts b/api/src/services/relations.ts index dca529be2e..e699656182 100644 --- a/api/src/services/relations.ts +++ b/api/src/services/relations.ts @@ -171,6 +171,8 @@ export class RelationsService { } const runPostColumnChange = await this.helpers.schema.preColumnChange(); + this.helpers.schema.preRelationChange(relation); + const nestedActionEvents: ActionEventParams[] = []; try { @@ -264,6 +266,8 @@ export class RelationsService { } const runPostColumnChange = await this.helpers.schema.preColumnChange(); + this.helpers.schema.preRelationChange(relation); + const nestedActionEvents: ActionEventParams[] = []; try { diff --git a/api/src/services/schema.test.ts b/api/src/services/schema.test.ts new file mode 100644 index 0000000000..754aeb9b1c --- /dev/null +++ b/api/src/services/schema.test.ts @@ -0,0 +1,188 @@ +import { Diff } from 'deep-diff'; +import knex, { Knex } from 'knex'; +import { getTracker, MockClient, Tracker } from 'knex-mock-client'; +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import { SchemaService } from '.'; +import { ForbiddenException } from '..'; +import { Collection } from '../types/collection'; +import { Snapshot, SnapshotDiffWithHash } from '../types/snapshot'; +import { applyDiff } from '../utils/apply-diff'; +import { getSnapshot } from '../utils/get-snapshot'; + +vi.mock('../../package.json', () => ({ version: '0.0.0' })); + +vi.mock('../../src/database/index', () => { + return { __esModule: true, default: vi.fn(), getDatabaseClient: vi.fn().mockReturnValue('postgres') }; +}); + +vi.mock('../utils/get-snapshot', () => ({ + getSnapshot: vi.fn(), +})); + +vi.mock('../utils/apply-diff', () => ({ + applyDiff: vi.fn(), +})); + +class Client_PG extends MockClient {} + +let db: Knex; +let tracker: Tracker; + +const testSnapshot = { + directus: '0.0.0', + version: 1, + vendor: 'postgres', + collections: [], + fields: [], + relations: [], +} satisfies Snapshot; + +const testCollectionDiff = { + collection: 'test', + diff: [ + { + kind: 'N', + rhs: { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { name: 'test' }, + }, + }, + ] satisfies Diff[], +}; + +beforeAll(() => { + db = knex({ client: Client_PG }); + tracker = getTracker(); +}); + +afterEach(() => { + tracker.reset(); + vi.clearAllMocks(); +}); + +describe('Services / Schema', () => { + describe('snapshot', () => { + it('should throw ForbiddenException for non-admin user', async () => { + vi.mocked(getSnapshot).mockResolvedValueOnce(testSnapshot); + + const service = new SchemaService({ knex: db, accountability: { role: 'test', admin: false } }); + + expect(service.snapshot()).rejects.toThrowError(ForbiddenException); + }); + + it('should return snapshot for admin user', async () => { + vi.mocked(getSnapshot).mockResolvedValueOnce(testSnapshot); + + const service = new SchemaService({ knex: db, accountability: { role: 'admin', admin: true } }); + + expect(service.snapshot()).resolves.toEqual(testSnapshot); + }); + }); + + describe('apply', () => { + const snapshotDiffWithHash = { + hash: '813b3cdf7013310fafde7813b7d5e6bd4eb1e73f', + diff: { + collections: [testCollectionDiff], + fields: [], + relations: [], + }, + } satisfies SnapshotDiffWithHash; + + it('should throw ForbiddenException for non-admin user', async () => { + vi.mocked(getSnapshot).mockResolvedValueOnce(testSnapshot); + + const service = new SchemaService({ knex: db, accountability: { role: 'test', admin: false } }); + + expect(service.apply(snapshotDiffWithHash)).rejects.toThrowError(ForbiddenException); + expect(vi.mocked(applyDiff)).not.toHaveBeenCalledOnce(); + }); + + it('should apply for admin user', async () => { + vi.mocked(getSnapshot).mockResolvedValueOnce(testSnapshot); + + const service = new SchemaService({ knex: db, accountability: { role: 'admin', admin: true } }); + + await service.apply(snapshotDiffWithHash); + + expect(vi.mocked(applyDiff)).toHaveBeenCalledOnce(); + }); + }); + + describe('diff', () => { + const snapshotToApply = { + directus: '0.0.0', + version: 1, + vendor: 'postgres', + collections: [ + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { + name: 'test', + }, + }, + ], + fields: [], + relations: [], + } satisfies Snapshot; + + it('should throw ForbiddenException for non-admin user', async () => { + const service = new SchemaService({ knex: db, accountability: { role: 'test', admin: false } }); + + expect(service.diff(snapshotToApply, { currentSnapshot: testSnapshot, force: true })).rejects.toThrowError( + ForbiddenException + ); + }); + + it('should return diff for admin user', async () => { + const service = new SchemaService({ knex: db, accountability: { role: 'admin', admin: true } }); + + expect(service.diff(snapshotToApply, { currentSnapshot: testSnapshot, force: true })).resolves.toEqual({ + collections: [testCollectionDiff], + fields: [], + relations: [], + }); + }); + + it('should return null for empty diff', async () => { + const service = new SchemaService({ knex: db, accountability: { role: 'admin', admin: true } }); + + expect(service.diff(testSnapshot, { currentSnapshot: testSnapshot, force: true })).resolves.toBeNull(); + }); + }); + + describe('getHashedSnapshot', () => { + it('should return snapshot for admin user', async () => { + const service = new SchemaService({ knex: db, accountability: { role: 'admin', admin: true } }); + + expect(service.getHashedSnapshot(testSnapshot)).toEqual( + expect.objectContaining({ + ...testSnapshot, + hash: expect.any(String), + }) + ); + }); + }); +}); diff --git a/api/src/services/schema.ts b/api/src/services/schema.ts new file mode 100644 index 0000000000..0935e89ab7 --- /dev/null +++ b/api/src/services/schema.ts @@ -0,0 +1,67 @@ +import { Accountability } from '@directus/shared/types'; +import { Knex } from 'knex'; +import getDatabase from '../database'; +import { ForbiddenException } from '../exceptions'; +import { AbstractServiceOptions, Snapshot, SnapshotDiff, SnapshotDiffWithHash, SnapshotWithHash } from '../types'; +import { applyDiff } from '../utils/apply-diff'; +import { getSnapshot } from '../utils/get-snapshot'; +import { getSnapshotDiff } from '../utils/get-snapshot-diff'; +import { getVersionedHash } from '../utils/get-versioned-hash'; +import { validateApplyDiff } from '../utils/validate-diff'; +import { validateSnapshot } from '../utils/validate-snapshot'; + +export class SchemaService { + knex: Knex; + accountability: Accountability | null; + + constructor(options: Omit) { + this.knex = options.knex ?? getDatabase(); + this.accountability = options.accountability ?? null; + } + + async snapshot(): Promise { + if (this.accountability?.admin !== true) throw new ForbiddenException(); + + const currentSnapshot = await getSnapshot({ database: this.knex }); + + return currentSnapshot; + } + + async apply(payload: SnapshotDiffWithHash): Promise { + if (this.accountability?.admin !== true) throw new ForbiddenException(); + + const currentSnapshot = await this.snapshot(); + const snapshotWithHash = this.getHashedSnapshot(currentSnapshot); + + if (!validateApplyDiff(payload, snapshotWithHash)) return; + + await applyDiff(currentSnapshot, payload.diff, { database: this.knex }); + } + + async diff( + snapshot: Snapshot, + options?: { currentSnapshot?: Snapshot; force?: boolean } + ): Promise { + if (this.accountability?.admin !== true) throw new ForbiddenException(); + + validateSnapshot(snapshot, options?.force); + + const currentSnapshot = options?.currentSnapshot ?? (await getSnapshot({ database: this.knex })); + const diff = getSnapshotDiff(currentSnapshot, snapshot); + + if (diff.collections.length === 0 && diff.fields.length === 0 && diff.relations.length === 0) { + return null; + } + + return diff; + } + + getHashedSnapshot(snapshot: Snapshot): SnapshotWithHash { + const snapshotHash = getVersionedHash(snapshot); + + return { + ...snapshot, + hash: snapshotHash, + }; + } +} diff --git a/api/src/types/database.ts b/api/src/types/database.ts new file mode 100644 index 0000000000..a3410db489 --- /dev/null +++ b/api/src/types/database.ts @@ -0,0 +1,4 @@ +export type Driver = 'mysql' | 'pg' | 'cockroachdb' | 'sqlite3' | 'oracledb' | 'mssql'; + +export const DatabaseClients = ['mysql', 'postgres', 'cockroachdb', 'sqlite', 'oracle', 'mssql', 'redshift'] as const; +export type DatabaseClient = typeof DatabaseClients[number]; diff --git a/api/src/types/index.ts b/api/src/types/index.ts index 0660a3610c..7dc87dc525 100644 --- a/api/src/types/index.ts +++ b/api/src/types/index.ts @@ -2,6 +2,7 @@ export * from './assets'; export * from './ast'; export * from './auth'; export * from './collection'; +export * from './database'; export * from './events'; export * from './files'; export * from './graphql'; diff --git a/api/src/types/snapshot.ts b/api/src/types/snapshot.ts index 47e86b2914..b92df5dba0 100644 --- a/api/src/types/snapshot.ts +++ b/api/src/types/snapshot.ts @@ -1,10 +1,12 @@ import { Collection } from './collection'; import { Relation, RelationMeta, Field, FieldMeta } from '@directus/shared/types'; import { Diff } from 'deep-diff'; +import { DatabaseClient } from './database'; export type Snapshot = { version: number; directus: string; + vendor?: DatabaseClient; collections: Collection[]; fields: SnapshotField[]; relations: SnapshotRelation[]; @@ -13,6 +15,8 @@ export type Snapshot = { export type SnapshotField = Field & { meta: Omit }; export type SnapshotRelation = Relation & { meta: Omit }; +export type SnapshotWithHash = Snapshot & { hash: string }; + export type SnapshotDiff = { collections: { collection: string; @@ -30,3 +34,19 @@ export type SnapshotDiff = { diff: Diff[]; }[]; }; + +export type SnapshotDiffWithHash = { hash: string; diff: SnapshotDiff }; + +/** + * Indicates the kind of change based on comparisons by deep-diff package + */ +export const DiffKind = { + /** indicates a newly added property/element */ + NEW: 'N', + /** indicates a property/element was deleted */ + DELETE: 'D', + /** indicates a property/element was edited */ + EDIT: 'E', + /** indicates a change occurred within an array */ + ARRAY: 'A', +} as const; diff --git a/api/src/utils/apply-diff.test.ts b/api/src/utils/apply-diff.test.ts new file mode 100644 index 0000000000..54f53c35b2 --- /dev/null +++ b/api/src/utils/apply-diff.test.ts @@ -0,0 +1,28 @@ +import { Diff } from 'deep-diff'; +import { describe, expect, it } from 'vitest'; +import { SnapshotField } from '../types/snapshot'; + +import { isNestedMetaUpdate } from './apply-diff'; + +describe('isNestedMetaUpdate', () => { + it.each([ + { kind: 'E', path: ['meta', 'options', 'option_a'], rhs: {} }, + { kind: 'A', path: ['meta', 'options', 'option_a'], rhs: [] }, + ] as Diff[])('Returns false when diff is kind $kind', (diff) => { + expect(isNestedMetaUpdate(diff)).toBe(false); + }); + + it.each([ + { kind: 'N', path: ['schema', 'default_value'], rhs: {} }, + { kind: 'D', path: ['schema'], lhs: {} }, + ] as Diff[])('Returns false when diff path is not nested in meta', (diff) => { + expect(isNestedMetaUpdate(diff)).toBe(false); + }); + + it.each([ + { kind: 'N', path: ['meta', 'options', 'option_a'], rhs: { test: 'value' } }, + { kind: 'D', path: ['meta', 'options', 'option_b'], lhs: {} }, + ] as Diff[])('Returns true when diff path is nested in meta', (diff) => { + expect(isNestedMetaUpdate(diff)).toBe(true); + }); +}); diff --git a/api/src/utils/apply-diff.ts b/api/src/utils/apply-diff.ts new file mode 100644 index 0000000000..5afe1354ab --- /dev/null +++ b/api/src/utils/apply-diff.ts @@ -0,0 +1,317 @@ +import { Field, Relation, SchemaOverview } from '@directus/shared/types'; +import { Knex } from 'knex'; +import { CollectionsService, FieldsService, RelationsService } from '../services'; +import { + ActionEventParams, + Collection, + DiffKind, + MutationOptions, + Snapshot, + SnapshotDiff, + SnapshotField, +} from '../types'; +import { getSchema } from './get-schema'; +import getDatabase from '../database'; +import { applyChange, Diff, DiffDeleted, DiffNew } from 'deep-diff'; +import { cloneDeep, merge, set } from 'lodash'; +import logger from '../logger'; +import emitter from '../emitter'; +import { clearSystemCache } from '../cache'; + +type CollectionDelta = { + collection: string; + diff: Diff[]; +}; + +export async function applyDiff( + currentSnapshot: Snapshot, + snapshotDiff: SnapshotDiff, + options?: { database?: Knex; schema?: SchemaOverview } +): Promise { + const database = options?.database ?? getDatabase(); + const schema = options?.schema ?? (await getSchema({ database, bypassCache: true })); + + const nestedActionEvents: ActionEventParams[] = []; + const mutationOptions: MutationOptions = { + autoPurgeSystemCache: false, + bypassEmitAction: (params) => nestedActionEvents.push(params), + }; + + await database.transaction(async (trx) => { + const collectionsService = new CollectionsService({ knex: trx, schema }); + + const getNestedCollectionsToCreate = (currentLevelCollection: string) => + snapshotDiff.collections.filter( + ({ diff }) => (diff[0] as DiffNew).rhs?.meta?.group === currentLevelCollection + ) as CollectionDelta[]; + + const getNestedCollectionsToDelete = (currentLevelCollection: string) => + snapshotDiff.collections.filter( + ({ diff }) => (diff[0] as DiffDeleted).lhs?.meta?.group === currentLevelCollection + ) as CollectionDelta[]; + + const createCollections = async (collections: CollectionDelta[]) => { + for (const { collection, diff } of collections) { + if (diff?.[0].kind === DiffKind.NEW && diff[0].rhs) { + // We'll nest the to-be-created fields in the same collection creation, to prevent + // creating a collection without a primary key + const fields = snapshotDiff.fields + .filter((fieldDiff) => fieldDiff.collection === collection) + .map((fieldDiff) => (fieldDiff.diff[0] as DiffNew).rhs) + .map((fieldDiff) => { + // Casts field type to UUID when applying non-PostgreSQL schema onto PostgreSQL database. + // This is needed because they snapshots UUID fields as char with length 36. + if ( + String(fieldDiff.schema?.data_type).toLowerCase() === 'char' && + fieldDiff.schema?.max_length === 36 && + (fieldDiff.schema?.is_primary_key || + (fieldDiff.schema?.foreign_key_table && fieldDiff.schema?.foreign_key_column)) + ) { + return merge(fieldDiff, { type: 'uuid', schema: { data_type: 'uuid', max_length: null } }); + } else { + return fieldDiff; + } + }); + + try { + await collectionsService.createOne( + { + ...diff[0].rhs, + fields, + }, + mutationOptions + ); + } catch (err: any) { + logger.error(`Failed to create collection "${collection}"`); + throw err; + } + + // Now that the fields are in for this collection, we can strip them from the field edits + snapshotDiff.fields = snapshotDiff.fields.filter((fieldDiff) => fieldDiff.collection !== collection); + + await createCollections(getNestedCollectionsToCreate(collection)); + } + } + }; + + const deleteCollections = async (collections: CollectionDelta[]) => { + for (const { collection, diff } of collections) { + if (diff?.[0].kind === DiffKind.DELETE) { + const relations = schema.relations.filter( + (r) => r.related_collection === collection || r.collection === collection + ); + + if (relations.length > 0) { + const relationsService = new RelationsService({ knex: trx, schema }); + + for (const relation of relations) { + try { + await relationsService.deleteOne(relation.collection, relation.field, mutationOptions); + } catch (err) { + logger.error( + `Failed to delete collection "${collection}" due to relation "${relation.collection}.${relation.field}"` + ); + throw err; + } + } + + // clean up deleted relations from existing schema + schema.relations = schema.relations.filter( + (r) => r.related_collection !== collection && r.collection !== collection + ); + } + + await deleteCollections(getNestedCollectionsToDelete(collection)); + + try { + await collectionsService.deleteOne(collection, mutationOptions); + } catch (err) { + logger.error(`Failed to delete collection "${collection}"`); + throw err; + } + } + } + }; + + // Finds all collections that need to be created + const filterCollectionsForCreation = ({ diff }: { collection: string; diff: Diff[] }) => { + // Check new collections only + const isNewCollection = diff[0].kind === DiffKind.NEW; + if (!isNewCollection) return false; + + // Create now if no group + const groupName = (diff[0] as DiffNew).rhs.meta?.group; + if (!groupName) return true; + + // Check if parent collection already exists in schema + const parentExists = currentSnapshot.collections.find((c) => c.collection === groupName) !== undefined; + // If this is a new collection and the parent collection doesn't exist in current schema -> + // Check if the parent collection will be created as part of applying this snapshot -> + // If yes -> this collection will be created recursively + // If not -> create now + // (ex.) + // TopLevelCollection - I exist in current schema + // NestedCollection - I exist in snapshotDiff as a new collection + // TheCurrentCollectionInIteration - I exist in snapshotDiff as a new collection but will be created as part of NestedCollection + const parentWillBeCreatedInThisApply = + snapshotDiff.collections.filter( + ({ collection, diff }) => diff[0].kind === DiffKind.NEW && collection === groupName + ).length > 0; + // Has group, but parent is not new, parent is also not being created in this snapshot apply + if (parentExists && !parentWillBeCreatedInThisApply) return true; + + return false; + }; + + // Create top level collections (no group, or highest level in existing group) first, + // then continue with nested collections recursively + await createCollections(snapshotDiff.collections.filter(filterCollectionsForCreation)); + + // delete top level collections (no group) first, then continue with nested collections recursively + await deleteCollections( + snapshotDiff.collections.filter( + ({ diff }) => diff[0].kind === DiffKind.DELETE && (diff[0] as DiffDeleted).lhs.meta?.group === null + ) + ); + + for (const { collection, diff } of snapshotDiff.collections) { + if (diff?.[0].kind === DiffKind.EDIT || diff?.[0].kind === DiffKind.ARRAY) { + const currentCollection = currentSnapshot.collections.find((field) => { + return field.collection === collection; + }); + + if (currentCollection) { + try { + const newValues = diff.reduce((acc, currentDiff) => { + applyChange(acc, undefined, currentDiff); + return acc; + }, cloneDeep(currentCollection)); + + await collectionsService.updateOne(collection, newValues, mutationOptions); + } catch (err) { + logger.error(`Failed to update collection "${collection}"`); + throw err; + } + } + } + } + + const fieldsService = new FieldsService({ + knex: trx, + schema: await getSchema({ database: trx, bypassCache: true }), + }); + + for (const { collection, field, diff } of snapshotDiff.fields) { + if (diff?.[0].kind === DiffKind.NEW && !isNestedMetaUpdate(diff?.[0])) { + try { + await fieldsService.createField(collection, (diff[0] as DiffNew).rhs, undefined, mutationOptions); + } catch (err) { + logger.error(`Failed to create field "${collection}.${field}"`); + throw err; + } + } + + if (diff?.[0].kind === DiffKind.EDIT || diff?.[0].kind === DiffKind.ARRAY || isNestedMetaUpdate(diff?.[0])) { + const currentField = currentSnapshot.fields.find((snapshotField) => { + return snapshotField.collection === collection && snapshotField.field === field; + }); + + if (currentField) { + try { + const newValues = diff.reduce((acc, currentDiff) => { + applyChange(acc, undefined, currentDiff); + return acc; + }, cloneDeep(currentField)); + await fieldsService.updateField(collection, newValues, mutationOptions); + } catch (err) { + logger.error(`Failed to update field "${collection}.${field}"`); + throw err; + } + } + } + + if (diff?.[0].kind === DiffKind.DELETE && !isNestedMetaUpdate(diff?.[0])) { + try { + await fieldsService.deleteField(collection, field, mutationOptions); + } catch (err) { + logger.error(`Failed to delete field "${collection}.${field}"`); + throw err; + } + + // Field deletion also cleans up the relationship. We should ignore any relationship + // changes attached to this now non-existing field + snapshotDiff.relations = snapshotDiff.relations.filter( + (relation) => (relation.collection === collection && relation.field === field) === false + ); + } + } + + const relationsService = new RelationsService({ + knex: trx, + schema: await getSchema({ database: trx, bypassCache: true }), + }); + + for (const { collection, field, diff } of snapshotDiff.relations) { + const structure = {}; + + for (const diffEdit of diff) { + set(structure, diffEdit.path!, undefined); + } + + if (diff?.[0].kind === DiffKind.NEW) { + try { + await relationsService.createOne((diff[0] as DiffNew).rhs, mutationOptions); + } catch (err) { + logger.error(`Failed to create relation "${collection}.${field}"`); + throw err; + } + } + + if (diff?.[0].kind === DiffKind.EDIT || diff?.[0].kind === DiffKind.ARRAY) { + const currentRelation = currentSnapshot.relations.find((relation) => { + return relation.collection === collection && relation.field === field; + }); + + if (currentRelation) { + try { + const newValues = diff.reduce((acc, currentDiff) => { + applyChange(acc, undefined, currentDiff); + return acc; + }, cloneDeep(currentRelation)); + await relationsService.updateOne(collection, field, newValues, mutationOptions); + } catch (err) { + logger.error(`Failed to update relation "${collection}.${field}"`); + throw err; + } + } + } + + if (diff?.[0].kind === DiffKind.DELETE) { + try { + await relationsService.deleteOne(collection, field, mutationOptions); + } catch (err) { + logger.error(`Failed to delete relation "${collection}.${field}"`); + throw err; + } + } + } + }); + + await clearSystemCache(); + + if (nestedActionEvents.length > 0) { + const updatedSchema = await getSchema({ database, bypassCache: true }); + + for (const nestedActionEvent of nestedActionEvents) { + nestedActionEvent.context.schema = updatedSchema; + emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context); + } + } +} + +export function isNestedMetaUpdate(diff: Diff): boolean { + if (!diff) return false; + if (diff.kind !== DiffKind.NEW && diff.kind !== DiffKind.DELETE) return false; + if (!diff.path || diff.path.length < 2 || diff.path[0] !== 'meta') return false; + return true; +} diff --git a/api/src/utils/apply-snapshot.test.ts b/api/src/utils/apply-snapshot.test.ts index dec2deca3e..db95f20eb8 100644 --- a/api/src/utils/apply-snapshot.test.ts +++ b/api/src/utils/apply-snapshot.test.ts @@ -50,7 +50,7 @@ describe('applySnapshot', () => { singleton: false, translations: {}, }, - schema: { comment: null, name: 'test_table_2', schema: 'public' }, + schema: { name: 'test_table_2' }, fields: [ { collection: 'test_table_2', @@ -76,11 +76,9 @@ describe('applySnapshot', () => { width: 'full', }, schema: { - comment: null, data_type: 'uuid', default_value: null, foreign_key_column: null, - foreign_key_schema: null, foreign_key_table: null, generation_expression: null, has_auto_increment: false, @@ -92,7 +90,6 @@ describe('applySnapshot', () => { name: 'id', numeric_precision: null, numeric_scale: null, - schema: 'public', table: 'test_table_2', }, type: 'uuid', @@ -134,7 +131,7 @@ describe('applySnapshot', () => { singleton: false, translations: {}, }, - schema: { comment: null, name: 'test_table_2', schema: 'public' }, + schema: { name: 'test_table_2' }, fields: [ { collection: 'test_table_2', @@ -160,11 +157,9 @@ describe('applySnapshot', () => { width: 'full', }, schema: { - comment: null, data_type: 'uuid', default_value: null, foreign_key_column: null, - foreign_key_schema: null, foreign_key_table: null, generation_expression: null, has_auto_increment: false, @@ -176,7 +171,6 @@ describe('applySnapshot', () => { name: 'id', numeric_precision: null, numeric_scale: null, - schema: 'public', table: 'test_table_2', }, type: 'uuid', @@ -211,11 +205,9 @@ describe('applySnapshot', () => { width: 'full', }, schema: { - comment: null, data_type: 'uuid', default_value: null, foreign_key_column: null, - foreign_key_schema: null, foreign_key_table: null, generation_expression: null, has_auto_increment: false, @@ -227,7 +219,6 @@ describe('applySnapshot', () => { name: 'id', numeric_precision: null, numeric_scale: null, - schema: 'public', table: 'test_table_3', }, type: 'uuid', @@ -244,7 +235,7 @@ describe('applySnapshot', () => { singleton: false, translations: {}, }, - schema: { comment: null, name: 'test_table_3', schema: 'public' }, + schema: { name: 'test_table_3' }, }; // Stop call to db later on in apply-snapshot diff --git a/api/src/utils/apply-snapshot.ts b/api/src/utils/apply-snapshot.ts index b6356bc484..3e1be93dd3 100644 --- a/api/src/utils/apply-snapshot.ts +++ b/api/src/utils/apply-snapshot.ts @@ -1,21 +1,12 @@ -import { Field, Relation, SchemaOverview } from '@directus/shared/types'; -import { Diff, DiffDeleted, DiffNew } from 'deep-diff'; +import { SchemaOverview } from '@directus/shared/types'; import { Knex } from 'knex'; -import { merge, set } from 'lodash'; +import { getCache } from '../cache'; import getDatabase from '../database'; -import logger from '../logger'; -import { CollectionsService, FieldsService, RelationsService } from '../services'; -import { ActionEventParams, Collection, MutationOptions, Snapshot, SnapshotDiff, SnapshotField } from '../types'; +import { Snapshot, SnapshotDiff } from '../types'; +import { applyDiff } from './apply-diff'; import { getSchema } from './get-schema'; import { getSnapshot } from './get-snapshot'; import { getSnapshotDiff } from './get-snapshot-diff'; -import { getCache } from '../cache'; -import emitter from '../emitter'; - -type CollectionDelta = { - collection: string; - diff: Diff[]; -}; export async function applySnapshot( snapshot: Snapshot, @@ -28,279 +19,7 @@ export async function applySnapshot( const current = options?.current ?? (await getSnapshot({ database, schema })); const snapshotDiff = options?.diff ?? getSnapshotDiff(current, snapshot); - const nestedActionEvents: ActionEventParams[] = []; - const mutationOptions: MutationOptions = { - autoPurgeSystemCache: false, - bypassEmitAction: (params) => nestedActionEvents.push(params), - }; - - await database.transaction(async (trx) => { - const collectionsService = new CollectionsService({ knex: trx, schema }); - - const getNestedCollectionsToCreate = (currentLevelCollection: string) => - snapshotDiff.collections.filter( - ({ diff }) => (diff[0] as DiffNew).rhs?.meta?.group === currentLevelCollection - ) as CollectionDelta[]; - - const getNestedCollectionsToDelete = (currentLevelCollection: string) => - snapshotDiff.collections.filter( - ({ diff }) => (diff[0] as DiffDeleted).lhs?.meta?.group === currentLevelCollection - ) as CollectionDelta[]; - - const createCollections = async (collections: CollectionDelta[]) => { - for (const { collection, diff } of collections) { - if (diff?.[0].kind === 'N' && diff[0].rhs) { - // We'll nest the to-be-created fields in the same collection creation, to prevent - // creating a collection without a primary key - const fields = snapshotDiff.fields - .filter((fieldDiff) => fieldDiff.collection === collection) - .map((fieldDiff) => (fieldDiff.diff[0] as DiffNew).rhs) - .map((fieldDiff) => { - // Casts field type to UUID when applying non-PostgreSQL schema onto PostgreSQL database. - // This is needed because they snapshots UUID fields as char with length 36. - if ( - fieldDiff.schema?.data_type === 'char' && - fieldDiff.schema?.max_length === 36 && - (fieldDiff.schema?.is_primary_key || - (fieldDiff.schema?.foreign_key_table && fieldDiff.schema?.foreign_key_column)) - ) { - return merge(fieldDiff, { type: 'uuid', schema: { data_type: 'uuid', max_length: null } }); - } else { - return fieldDiff; - } - }); - - try { - await collectionsService.createOne( - { - ...diff[0].rhs, - fields, - }, - mutationOptions - ); - } catch (err: any) { - logger.error(`Failed to create collection "${collection}"`); - throw err; - } - - // Now that the fields are in for this collection, we can strip them from the field edits - snapshotDiff.fields = snapshotDiff.fields.filter((fieldDiff) => fieldDiff.collection !== collection); - - await createCollections(getNestedCollectionsToCreate(collection)); - } - } - }; - - const deleteCollections = async (collections: CollectionDelta[]) => { - for (const { collection, diff } of collections) { - if (diff?.[0].kind === 'D') { - const relations = schema.relations.filter( - (r) => r.related_collection === collection || r.collection === collection - ); - - if (relations.length > 0) { - const relationsService = new RelationsService({ knex: trx, schema }); - - for (const relation of relations) { - try { - await relationsService.deleteOne(relation.collection, relation.field, mutationOptions); - } catch (err) { - logger.error( - `Failed to delete collection "${collection}" due to relation "${relation.collection}.${relation.field}"` - ); - throw err; - } - } - - // clean up deleted relations from existing schema - schema.relations = schema.relations.filter( - (r) => r.related_collection !== collection && r.collection !== collection - ); - } - - await deleteCollections(getNestedCollectionsToDelete(collection)); - - try { - await collectionsService.deleteOne(collection, mutationOptions); - } catch (err) { - logger.error(`Failed to delete collection "${collection}"`); - throw err; - } - } - } - }; - - // Finds all collections that need to be created - const filterCollectionsForCreation = ({ diff }: { collection: string; diff: Diff[] }) => { - // Check new collections only - const isNewCollection = diff[0].kind === 'N'; - if (!isNewCollection) return false; - - // Create now if no group - const groupName = (diff[0] as DiffNew).rhs.meta?.group; - if (!groupName) return true; - - // Check if parent collection already exists in schema - const parentExists = current.collections.find((c) => c.collection === groupName) !== undefined; - // If this is a new collection and the parent collection doesn't exist in current schema -> - // Check if the parent collection will be created as part of applying this snapshot -> - // If yes -> this collection will be created recursively - // If not -> create now - // (ex.) - // TopLevelCollection - I exist in current schema - // NestedCollection - I exist in snapshotDiff as a new collection - // TheCurrentCollectionInIteration - I exist in snapshotDiff as a new collection but will be created as part of NestedCollection - const parentWillBeCreatedInThisApply = - snapshotDiff.collections.filter(({ collection, diff }) => diff[0].kind === 'N' && collection === groupName) - .length > 0; - // Has group, but parent is not new, parent is also not being created in this snapshot apply - if (parentExists && !parentWillBeCreatedInThisApply) return true; - - return false; - }; - - // Create top level collections (no group, or highest level in existing group) first, - // then continue with nested collections recursively - await createCollections(snapshotDiff.collections.filter(filterCollectionsForCreation)); - - // delete top level collections (no group) first, then continue with nested collections recursively - await deleteCollections( - snapshotDiff.collections.filter( - ({ diff }) => diff[0].kind === 'D' && (diff[0] as DiffDeleted).lhs.meta?.group === null - ) - ); - - for (const { collection, diff } of snapshotDiff.collections) { - if (diff?.[0].kind === 'E' || diff?.[0].kind === 'A') { - const newValues = snapshot.collections.find((field) => { - return field.collection === collection; - }); - - if (newValues) { - try { - await collectionsService.updateOne(collection, newValues, mutationOptions); - } catch (err) { - logger.error(`Failed to update collection "${collection}"`); - throw err; - } - } - } - } - - const fieldsService = new FieldsService({ - knex: trx, - schema: await getSchema({ database: trx, bypassCache: true }), - }); - - for (const { collection, field, diff } of snapshotDiff.fields) { - if (diff?.[0].kind === 'N' && !isNestedMetaUpdate(diff?.[0])) { - try { - await fieldsService.createField(collection, (diff[0] as DiffNew).rhs, undefined, mutationOptions); - } catch (err) { - logger.error(`Failed to create field "${collection}.${field}"`); - throw err; - } - } - - if (diff?.[0].kind === 'E' || diff?.[0].kind === 'A' || isNestedMetaUpdate(diff?.[0])) { - const newValues = snapshot.fields.find((snapshotField) => { - return snapshotField.collection === collection && snapshotField.field === field; - }); - - if (newValues) { - try { - await fieldsService.updateField( - collection, - { - ...newValues, - }, - mutationOptions - ); - } catch (err) { - logger.error(`Failed to update field "${collection}.${field}"`); - throw err; - } - } - } - - if (diff?.[0].kind === 'D' && !isNestedMetaUpdate(diff?.[0])) { - try { - await fieldsService.deleteField(collection, field, mutationOptions); - } catch (err) { - logger.error(`Failed to delete field "${collection}.${field}"`); - throw err; - } - - // Field deletion also cleans up the relationship. We should ignore any relationship - // changes attached to this now non-existing field - snapshotDiff.relations = snapshotDiff.relations.filter( - (relation) => (relation.collection === collection && relation.field === field) === false - ); - } - } - - const relationsService = new RelationsService({ - knex: trx, - schema: await getSchema({ database: trx, bypassCache: true }), - }); - - for (const { collection, field, diff } of snapshotDiff.relations) { - const structure = {}; - - for (const diffEdit of diff) { - set(structure, diffEdit.path!, undefined); - } - - if (diff?.[0].kind === 'N') { - try { - await relationsService.createOne((diff[0] as DiffNew).rhs, mutationOptions); - } catch (err) { - logger.error(`Failed to create relation "${collection}.${field}"`); - throw err; - } - } - - if (diff?.[0].kind === 'E' || diff?.[0].kind === 'A') { - const newValues = snapshot.relations.find((relation) => { - return relation.collection === collection && relation.field === field; - }); - - if (newValues) { - try { - await relationsService.updateOne(collection, field, newValues, mutationOptions); - } catch (err) { - logger.error(`Failed to update relation "${collection}.${field}"`); - throw err; - } - } - } - - if (diff?.[0].kind === 'D') { - try { - await relationsService.deleteOne(collection, field, mutationOptions); - } catch (err) { - logger.error(`Failed to delete relation "${collection}.${field}"`); - throw err; - } - } - } - }); + await applyDiff(current, snapshotDiff, { database, schema }); await systemCache?.clear(); - - if (nestedActionEvents.length > 0) { - const updatedSchema = await getSchema({ database, bypassCache: true }); - - for (const nestedActionEvent of nestedActionEvents) { - nestedActionEvent.context.schema = updatedSchema; - emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context); - } - } -} - -export function isNestedMetaUpdate(diff: Diff): boolean { - if (!diff) return false; - if (diff.kind !== 'N' && diff.kind !== 'D') return false; - if (!diff.path || diff.path.length < 2 || diff.path[0] !== 'meta') return false; - return true; } diff --git a/api/src/utils/get-snapshot-diff.ts b/api/src/utils/get-snapshot-diff.ts index c790e84d3b..6b7ca3d00c 100644 --- a/api/src/utils/get-snapshot-diff.ts +++ b/api/src/utils/get-snapshot-diff.ts @@ -1,6 +1,7 @@ -import { Snapshot, SnapshotDiff } from '../types'; import { diff } from 'deep-diff'; import { orderBy } from 'lodash'; +import { Snapshot, SnapshotDiff, DiffKind } from '../types'; +import { sanitizeCollection, sanitizeField, sanitizeRelation } from './sanitize-schema'; export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDiff { const diffedSnapshot: SnapshotDiff = { @@ -13,7 +14,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif return { collection: currentCollection.collection, - diff: diff(currentCollection, afterCollection), + diff: diff(sanitizeCollection(currentCollection), sanitizeCollection(afterCollection)), }; }), ...after.collections @@ -26,7 +27,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif }) .map((afterCollection) => ({ collection: afterCollection.collection, - diff: diff(undefined, afterCollection), + diff: diff(undefined, sanitizeCollection(afterCollection)), })), ].filter((obj) => Array.isArray(obj.diff)) as SnapshotDiff['collections'], 'collection' @@ -38,10 +39,16 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif (afterField) => afterField.collection === currentField.collection && afterField.field === currentField.field ); + const isAutoIncrementPrimaryKey = + !!currentField.schema?.is_primary_key && !!currentField.schema?.has_auto_increment; + return { collection: currentField.collection, field: currentField.field, - diff: diff(currentField, afterField), + diff: diff( + sanitizeField(currentField, isAutoIncrementPrimaryKey), + sanitizeField(afterField, isAutoIncrementPrimaryKey) + ), }; }), ...after.fields @@ -56,7 +63,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif .map((afterField) => ({ collection: afterField.collection, field: afterField.field, - diff: diff(undefined, afterField), + diff: diff(undefined, sanitizeField(afterField)), })), ].filter((obj) => Array.isArray(obj.diff)) as SnapshotDiff['fields'], ['collection'] @@ -73,7 +80,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif collection: currentRelation.collection, field: currentRelation.field, related_collection: currentRelation.related_collection, - diff: diff(currentRelation, afterRelation), + diff: diff(sanitizeRelation(currentRelation), sanitizeRelation(afterRelation)), }; }), ...after.relations @@ -89,7 +96,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif collection: afterRelation.collection, field: afterRelation.field, related_collection: afterRelation.related_collection, - diff: diff(undefined, afterRelation), + diff: diff(undefined, sanitizeRelation(afterRelation)), })), ].filter((obj) => Array.isArray(obj.diff)) as SnapshotDiff['relations'], ['collection'] @@ -101,7 +108,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif */ const deletedCollections = diffedSnapshot.collections - .filter((collection) => collection.diff?.[0].kind === 'D') + .filter((collection) => collection.diff?.[0].kind === DiffKind.DELETE) .map(({ collection }) => collection); diffedSnapshot.fields = diffedSnapshot.fields.filter( diff --git a/api/src/utils/get-snapshot.ts b/api/src/utils/get-snapshot.ts index c54aa83f22..3e4d5e0b19 100644 --- a/api/src/utils/get-snapshot.ts +++ b/api/src/utils/get-snapshot.ts @@ -1,14 +1,16 @@ -import getDatabase from '../database'; +import getDatabase, { getDatabaseClient } from '../database'; import { getSchema } from './get-schema'; import { CollectionsService, FieldsService, RelationsService } from '../services'; import { version } from '../../package.json'; -import { Snapshot, SnapshotField, SnapshotRelation } from '../types'; +import { Collection, Snapshot, SnapshotField, SnapshotRelation } from '../types'; import { Knex } from 'knex'; import { omit, sortBy, toPairs, fromPairs, mapValues, isPlainObject, isArray } from 'lodash'; import { SchemaOverview } from '@directus/shared/types'; +import { sanitizeCollection, sanitizeField, sanitizeRelation } from './sanitize-schema'; export async function getSnapshot(options?: { database?: Knex; schema?: SchemaOverview }): Promise { const database = options?.database ?? getDatabase(); + const vendor = getDatabaseClient(database); const schema = options?.schema ?? (await getSchema({ database, bypassCache: true })); const collectionsService = new CollectionsService({ knex: database, schema }); @@ -32,9 +34,10 @@ export async function getSnapshot(options?: { database?: Knex; schema?: SchemaOv return { version: 1, directus: version, - collections: collectionsSorted, - fields: fieldsSorted, - relations: relationsSorted, + vendor, + collections: collectionsSorted.map((collection) => sanitizeCollection(collection)) as Collection[], + fields: fieldsSorted.map((field) => sanitizeField(field)) as SnapshotField[], + relations: relationsSorted.map((relation) => sanitizeRelation(relation)) as SnapshotRelation[], }; } diff --git a/api/src/utils/get-versioned-hash.test.ts b/api/src/utils/get-versioned-hash.test.ts new file mode 100644 index 0000000000..de8bacd38c --- /dev/null +++ b/api/src/utils/get-versioned-hash.test.ts @@ -0,0 +1,118 @@ +import { test, expect, describe, vi } from 'vitest'; + +import { getVersionedHash } from './get-versioned-hash'; + +vi.mock('../../package.json', () => ({ + version: '10.10.10', +})); + +describe('getVersionedHash', () => { + test.each([ + { + input: { + collection: 'test', + meta: { + accountability: 'all', + archive_app_filter: true, + archive_field: 'status', + archive_value: 'archived', + collapse: 'open', + collection: 'test', + color: null, + display_template: null, + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + sort: null, + sort_field: null, + translations: null, + unarchive_value: 'draft', + }, + schema: { name: 'test' }, + }, + expected: '8acdde88d26ea4142e224fe0b4bfdaab9ac9c923', + }, + { + input: { + collection: 'test', + field: 'id', + meta: { + collection: 'test', + conditions: null, + display: null, + display_options: null, + field: 'id', + group: null, + hidden: true, + interface: 'input', + note: null, + options: null, + readonly: true, + required: false, + sort: null, + special: null, + translations: null, + validation: null, + validation_message: null, + width: 'full', + }, + schema: { + comment: null, + data_type: 'integer', + default_value: "nextval('test_id_seq'::regclass)", + foreign_key_column: null, + foreign_key_schema: null, + foreign_key_table: null, + generation_expression: null, + has_auto_increment: true, + is_generated: false, + is_nullable: false, + is_primary_key: true, + is_unique: true, + max_length: null, + name: 'id', + numeric_precision: 32, + numeric_scale: 0, + schema: 'public', + table: 'test', + }, + type: 'integer', + }, + expected: '60f0d169cfa32799cae884e1bf33e0e49c3ff383', + }, + { + input: { + collection: 'test_example', + field: 'm2m', + related_collection: 'test', + meta: { + id: 1, + junction_field: 'example_id', + many_collection: 'test_example', + many_field: 'test_id', + one_allowed_collections: null, + one_collection: 'test', + one_collection_field: null, + one_deselect_action: 'nullify', + one_field: 'm2m', + sort_field: null, + }, + schema: { + table: 'test_example', + column: 'test_id', + foreign_key_table: 'test', + foreign_key_column: 'id', + constraint_name: 'test_example_test_id_foreign', + on_update: 'NO ACTION', + on_delete: 'SET NULL', + }, + }, + expected: 'fa17767ef6646a72a6cfc211d36886d06896d0fc', + }, + ])('should return $expected', ({ input, expected }) => { + expect(getVersionedHash(input)).toBe(expected); + }); +}); diff --git a/api/src/utils/get-versioned-hash.ts b/api/src/utils/get-versioned-hash.ts new file mode 100644 index 0000000000..8d96b0aa83 --- /dev/null +++ b/api/src/utils/get-versioned-hash.ts @@ -0,0 +1,6 @@ +import hash from 'object-hash'; +import { version } from '../../package.json'; + +export function getVersionedHash(item: Record): string { + return hash({ item, version }); +} diff --git a/api/src/utils/sanitize-schema.test.ts b/api/src/utils/sanitize-schema.test.ts new file mode 100644 index 0000000000..4b48afb40b --- /dev/null +++ b/api/src/utils/sanitize-schema.test.ts @@ -0,0 +1,347 @@ +import { Field, Relation } from '@directus/shared/types'; +import { expect, test, describe } from 'vitest'; +import { Collection } from '../types'; + +import { sanitizeCollection, sanitizeField, sanitizeRelation } from './sanitize-schema'; + +describe('sanitizeCollection', () => { + test.each([ + // Not supported in SQLite + comment in MSSQL + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { comment: null, name: 'test', schema: 'public' }, + }, + // MySQL Only + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { collation: 'latin1_swedish_ci', name: 'test', engine: 'InnoDB' }, + }, + // Postgres Only + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { name: 'test', owner: 'postgres' }, + }, + // SQLite Only + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { name: 'test', sql: 'CREATE TABLE `test` (`id` integer not null primary key autoincrement)' }, + }, + // MSSQL only + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { name: 'test', catalog: 'test-db' }, + }, + ] satisfies Collection[])('should only contain name property in collection schema', (testCollection) => { + const result = sanitizeCollection(testCollection); + + expect(result).toEqual({ + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { name: 'test' }, + }); + }); +}); + +describe('sanitizeField', () => { + test('should only contain certain properties in field schema when sanitizeAllSchema is false', () => { + const testField = { + collection: 'test', + field: 'id', + name: 'id', + meta: { + id: 1, + collection: 'test', + conditions: null, + display: null, + display_options: null, + field: 'id', + group: null, + hidden: true, + interface: 'input', + note: null, + options: null, + readonly: true, + required: false, + sort: null, + special: null, + translations: null, + validation: null, + validation_message: null, + width: 'full', + }, + schema: { + comment: null, + data_type: 'integer', + default_value: "nextval('test_id_seq'::regclass)", + foreign_key_column: null, + foreign_key_schema: null, + foreign_key_table: null, + generation_expression: null, + has_auto_increment: true, + is_generated: false, + is_nullable: false, + is_primary_key: true, + is_unique: true, + max_length: null, + name: 'id', + numeric_precision: 32, + numeric_scale: 0, + schema: 'public', + table: 'test', + }, + type: 'integer', + } satisfies Field; + + const result = sanitizeField(testField); + + expect(result).toEqual({ + collection: 'test', + field: 'id', + name: 'id', + meta: { + id: 1, + collection: 'test', + conditions: null, + display: null, + display_options: null, + field: 'id', + group: null, + hidden: true, + interface: 'input', + note: null, + options: null, + readonly: true, + required: false, + sort: null, + special: null, + translations: null, + validation: null, + validation_message: null, + width: 'full', + }, + schema: { + data_type: 'integer', + default_value: "nextval('test_id_seq'::regclass)", + foreign_key_column: null, + foreign_key_table: null, + generation_expression: null, + has_auto_increment: true, + is_generated: false, + is_nullable: false, + is_primary_key: true, + is_unique: true, + max_length: null, + name: 'id', + numeric_precision: 32, + numeric_scale: 0, + + table: 'test', + }, + type: 'integer', + }); + }); + + test('should not contain field schema when sanitizeAllSchema is true', () => { + const testField = { + collection: 'test', + field: 'id', + name: 'id', + meta: { + id: 1, + collection: 'test', + conditions: null, + display: null, + display_options: null, + field: 'id', + group: null, + hidden: true, + interface: 'input', + note: null, + options: null, + readonly: true, + required: false, + sort: null, + special: null, + translations: null, + validation: null, + validation_message: null, + width: 'full', + }, + schema: { + data_type: 'integer', + default_value: "nextval('test_id_seq'::regclass)", + foreign_key_column: null, + foreign_key_table: null, + generation_expression: null, + has_auto_increment: true, + is_generated: false, + is_nullable: false, + is_primary_key: true, + is_unique: true, + max_length: null, + name: 'id', + numeric_precision: 32, + numeric_scale: 0, + table: 'test', + }, + type: 'integer', + } satisfies Field; + + const result = sanitizeField(testField, true); + + expect(result).toEqual({ + collection: 'test', + field: 'id', + name: 'id', + meta: { + id: 1, + collection: 'test', + conditions: null, + display: null, + display_options: null, + field: 'id', + group: null, + hidden: true, + interface: 'input', + note: null, + options: null, + readonly: true, + required: false, + sort: null, + special: null, + translations: null, + validation: null, + validation_message: null, + width: 'full', + }, + type: 'integer', + }); + }); +}); + +describe('sanitizeRelation', () => { + test.each([ + // Postgres + MSSSQL + { + collection: 'test_example', + field: 'm2m', + related_collection: 'test', + meta: { + id: 1, + junction_field: 'example_id', + many_collection: 'test_example', + many_field: 'test_id', + one_allowed_collections: null, + one_collection: 'test', + one_collection_field: null, + one_deselect_action: 'nullify', + one_field: 'm2m', + sort_field: null, + }, + schema: { + table: 'test_example', + column: 'test_id', + foreign_key_table: 'test', + foreign_key_column: 'id', + foreign_key_schema: 'public', + constraint_name: 'test_example_test_id_foreign', + on_update: 'NO ACTION', + on_delete: 'SET NULL', + }, + }, + ] satisfies Relation[])('should only contain certain properties in relation schema', (testRelation) => { + const result = sanitizeRelation(testRelation); + + expect(result).toEqual({ + collection: 'test_example', + field: 'm2m', + related_collection: 'test', + meta: { + id: 1, + junction_field: 'example_id', + many_collection: 'test_example', + many_field: 'test_id', + one_allowed_collections: null, + one_collection: 'test', + one_collection_field: null, + one_deselect_action: 'nullify', + one_field: 'm2m', + sort_field: null, + }, + schema: { + table: 'test_example', + column: 'test_id', + foreign_key_table: 'test', + foreign_key_column: 'id', + constraint_name: 'test_example_test_id_foreign', + on_update: 'NO ACTION', + on_delete: 'SET NULL', + }, + }); + }); +}); diff --git a/api/src/utils/sanitize-schema.ts b/api/src/utils/sanitize-schema.ts new file mode 100644 index 0000000000..14e4e28742 --- /dev/null +++ b/api/src/utils/sanitize-schema.ts @@ -0,0 +1,81 @@ +import { Field, Relation } from '@directus/shared/types'; +import { pick } from 'lodash'; +import { Collection } from '../types'; + +/** + * Pick certain database vendor specific collection properties that should be compared when performing diff + * + * @param collection collection to sanitize + * @returns sanitized collection + * + * @see {@link https://github.com/knex/knex-schema-inspector/blob/master/lib/types/table.ts} + */ + +export function sanitizeCollection(collection: Collection | undefined) { + if (!collection) return collection; + + return pick(collection, ['collection', 'fields', 'meta', 'schema.name']); +} + +/** + * Pick certain database vendor specific field properties that should be compared when performing diff + * + * @param field field to sanitize + * @param sanitizeAllSchema Whether or not the whole field schema should be sanitized. Mainly used to prevent modifying autoincrement fields + * @returns sanitized field + * + * @see {@link https://github.com/knex/knex-schema-inspector/blob/master/lib/types/column.ts} + */ +export function sanitizeField(field: Field | undefined, sanitizeAllSchema = false) { + if (!field) return field; + + const defaultPaths = ['collection', 'field', 'type', 'meta', 'name', 'children']; + const pickedPaths = sanitizeAllSchema + ? defaultPaths + : [ + ...defaultPaths, + 'schema.name', + 'schema.table', + 'schema.data_type', + 'schema.default_value', + 'schema.max_length', + 'schema.numeric_precision', + 'schema.numeric_scale', + 'schema.is_nullable', + 'schema.is_unique', + 'schema.is_primary_key', + 'schema.is_generated', + 'schema.generation_expression', + 'schema.has_auto_increment', + 'schema.foreign_key_table', + 'schema.foreign_key_column', + ]; + + return pick(field, pickedPaths); +} + +/** + * Pick certain database vendor specific relation properties that should be compared when performing diff + * + * @param relation relation to sanitize + * @returns sanitized relation + * + * @see {@link https://github.com/knex/knex-schema-inspector/blob/master/lib/types/foreign-key.ts} + */ +export function sanitizeRelation(relation: Relation | undefined) { + if (!relation) return relation; + + return pick(relation, [ + 'collection', + 'field', + 'related_collection', + 'meta', + 'schema.table', + 'schema.column', + 'schema.foreign_key_table', + 'schema.foreign_key_column', + 'schema.constraint_name', + 'schema.on_update', + 'schema.on_delete', + ]); +} diff --git a/api/src/utils/validate-diff.test.ts b/api/src/utils/validate-diff.test.ts new file mode 100644 index 0000000000..8aa8de142d --- /dev/null +++ b/api/src/utils/validate-diff.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, test } from 'vitest'; +import { Collection } from '../types/collection'; +import { + Snapshot, + SnapshotDiff, + SnapshotDiffWithHash, + SnapshotField, + SnapshotRelation, + SnapshotWithHash, +} from '../types/snapshot'; +import { validateApplyDiff } from './validate-diff'; + +test('should fail on invalid diff schema', () => { + const diff = {} as SnapshotDiffWithHash; + const snapshot = {} as SnapshotWithHash; + + expect(() => validateApplyDiff(diff, snapshot)).toThrowError('"hash" is required'); +}); + +test('should fail on invalid hash', () => { + const diff = { + hash: 'abc', + diff: { collections: [{ collection: 'test', diff: [] }], fields: [], relations: [] }, + } as SnapshotDiffWithHash; + const snapshot = { hash: 'xyz' } as SnapshotWithHash; + + expect(() => validateApplyDiff(diff, snapshot)).toThrowError( + "Provided hash does not match the current instance's schema hash" + ); +}); + +describe('should throw accurate error', () => { + const baseDiff = (partialDiff: Partial): SnapshotDiffWithHash => { + return { + hash: 'abc', + diff: { + fields: [], + collections: [], + relations: [], + ...partialDiff, + }, + }; + }; + const baseSnapshot = (partialSnapshot?: Partial) => { + return { + hash: 'xyz', + collections: [] as Collection[], + fields: [] as SnapshotField[], + relations: [] as SnapshotRelation[], + ...partialSnapshot, + } as SnapshotWithHash; + }; + + test('creating collection which already exists', () => { + const diff = baseDiff({ + collections: [{ collection: 'test', diff: [{ kind: 'N', rhs: {} as Collection }] }], + }); + const snapshot = baseSnapshot({ collections: [{ collection: 'test' } as Collection] }); + + expect(() => validateApplyDiff(diff, snapshot)).toThrowError( + 'Provided diff is trying to create collection "test" but it already exists' + ); + }); + + test('deleting collection which does not exist', () => { + const diff = baseDiff({ + collections: [{ collection: 'test', diff: [{ kind: 'D', lhs: {} as Collection }] }], + }); + + expect(() => validateApplyDiff(diff, baseSnapshot())).toThrowError( + 'Provided diff is trying to delete collection "test" but it does not exist' + ); + }); + + test('creating field which already exists', () => { + const diff = baseDiff({ + fields: [{ collection: 'test', field: 'test', diff: [{ kind: 'N', rhs: {} as SnapshotField }] }], + }); + const snapshot = baseSnapshot({ fields: [{ collection: 'test', field: 'test' } as SnapshotField] }); + + expect(() => validateApplyDiff(diff, snapshot)).toThrowError( + 'Provided diff is trying to create field "test.test" but it already exists' + ); + }); + + test('deleting field which does not exist', () => { + const diff = baseDiff({ + fields: [{ collection: 'test', field: 'test', diff: [{ kind: 'D', lhs: {} as SnapshotField }] }], + }); + + expect(() => validateApplyDiff(diff, baseSnapshot())).toThrowError( + 'Provided diff is trying to delete field "test.test" but it does not exist' + ); + }); + + test('creating relation which already exists', () => { + const diff = baseDiff({ + relations: [ + { + collection: 'test', + field: 'test', + related_collection: 'relation', + diff: [{ kind: 'N', rhs: {} as SnapshotRelation }], + }, + ], + }); + const snapshot = baseSnapshot({ + relations: [{ collection: 'test', field: 'test', related_collection: 'relation' } as SnapshotRelation], + }); + + expect(() => validateApplyDiff(diff, snapshot)).toThrowError( + 'Provided diff is trying to create relation "test.test-> relation" but it already exists' + ); + }); + + test('deleting relation which does not exist', () => { + const diff = baseDiff({ + relations: [ + { + collection: 'test', + field: 'test', + related_collection: 'relation', + diff: [{ kind: 'D', lhs: {} as SnapshotRelation }], + }, + ], + }); + + expect(() => validateApplyDiff(diff, baseSnapshot())).toThrowError( + 'Provided diff is trying to delete relation "test.test-> relation" but it does not exist' + ); + }); +}); + +test('should detect empty diff', () => { + const diff = { + hash: 'abc', + diff: { collections: [], fields: [], relations: [] }, + }; + const snapshot = {} as SnapshotWithHash; + + expect(validateApplyDiff(diff, snapshot)).toBe(false); +}); + +test('should pass on valid diff', () => { + const diff = { + hash: 'abc', + diff: { collections: [{ collection: 'test', diff: [] }], fields: [], relations: [] }, + }; + const snapshot = { hash: 'abc' } as SnapshotWithHash; + + expect(validateApplyDiff(diff, snapshot)).toBe(true); +}); diff --git a/api/src/utils/validate-diff.ts b/api/src/utils/validate-diff.ts new file mode 100644 index 0000000000..1c1f5dad02 --- /dev/null +++ b/api/src/utils/validate-diff.ts @@ -0,0 +1,150 @@ +import Joi from 'joi'; +import { InvalidPayloadException } from '../index'; +import { DiffKind, SnapshotDiffWithHash, SnapshotWithHash } from '../types/snapshot'; + +const deepDiffSchema = Joi.object({ + kind: Joi.string() + .valid(...Object.values(DiffKind)) + .required(), + path: Joi.array().items(Joi.string()), + lhs: Joi.object().when('kind', { is: [DiffKind.DELETE, DiffKind.EDIT], then: Joi.required() }), + rhs: Joi.object().when('kind', { is: [DiffKind.NEW, DiffKind.EDIT], then: Joi.required() }), + index: Joi.number().when('kind', { is: DiffKind.ARRAY, then: Joi.required() }), + item: Joi.link('/').when('kind', { is: DiffKind.ARRAY, then: Joi.required() }), +}); + +const applyJoiSchema = Joi.object({ + hash: Joi.string().required(), + diff: Joi.object({ + collections: Joi.array() + .items( + Joi.object({ + collection: Joi.string().required(), + diff: Joi.array().items(deepDiffSchema).required(), + }) + ) + .required(), + fields: Joi.array() + .items( + Joi.object({ + collection: Joi.string().required(), + field: Joi.string().required(), + diff: Joi.array().items(deepDiffSchema).required(), + }) + ) + .required(), + relations: Joi.array() + .items( + Joi.object({ + collection: Joi.string().required(), + field: Joi.string().required(), + related_collection: Joi.string(), + diff: Joi.array().items(deepDiffSchema).required(), + }) + ) + .required(), + }).required(), +}); + +/** + * Validates the diff against the current schema snapshot. + * + * @returns True if the diff can be applied (valid & not empty). + */ +export function validateApplyDiff(applyDiff: SnapshotDiffWithHash, currentSnapshotWithHash: SnapshotWithHash) { + const { error } = applyJoiSchema.validate(applyDiff); + if (error) throw new InvalidPayloadException(error.message); + + // No changes to apply + if ( + applyDiff.diff.collections.length === 0 && + applyDiff.diff.fields.length === 0 && + applyDiff.diff.relations.length === 0 + ) { + return false; + } + + // Diff can be applied due to matching hash + if (applyDiff.hash === currentSnapshotWithHash.hash) return true; + + for (const diffCollection of applyDiff.diff.collections) { + const collection = diffCollection.collection; + + if (diffCollection.diff[0]?.kind === DiffKind.NEW) { + const existingCollection = currentSnapshotWithHash.collections.find( + (c) => c.collection === diffCollection.collection + ); + + if (existingCollection) { + throw new InvalidPayloadException( + `Provided diff is trying to create collection "${collection}" but it already exists. Please generate a new diff and try again.` + ); + } + } else if (diffCollection.diff[0]?.kind === DiffKind.DELETE) { + const existingCollection = currentSnapshotWithHash.collections.find( + (c) => c.collection === diffCollection.collection + ); + + if (!existingCollection) { + throw new InvalidPayloadException( + `Provided diff is trying to delete collection "${collection}" but it does not exist. Please generate a new diff and try again.` + ); + } + } + } + for (const diffField of applyDiff.diff.fields) { + const field = `${diffField.collection}.${diffField.field}`; + + if (diffField.diff[0]?.kind === DiffKind.NEW) { + const existingField = currentSnapshotWithHash.fields.find( + (f) => f.collection === diffField.collection && f.field === diffField.field + ); + + if (existingField) { + throw new InvalidPayloadException( + `Provided diff is trying to create field "${field}" but it already exists. Please generate a new diff and try again.` + ); + } + } else if (diffField.diff[0]?.kind === DiffKind.DELETE) { + const existingField = currentSnapshotWithHash.fields.find( + (f) => f.collection === diffField.collection && f.field === diffField.field + ); + + if (!existingField) { + throw new InvalidPayloadException( + `Provided diff is trying to delete field "${field}" but it does not exist. Please generate a new diff and try again.` + ); + } + } + } + for (const diffRelation of applyDiff.diff.relations) { + let relation = `${diffRelation.collection}.${diffRelation.field}`; + if (diffRelation.related_collection) relation += `-> ${diffRelation.related_collection}`; + + if (diffRelation.diff[0]?.kind === DiffKind.NEW) { + const existingRelation = currentSnapshotWithHash.relations.find( + (r) => r.collection === diffRelation.collection && r.field === diffRelation.field + ); + + if (existingRelation) { + throw new InvalidPayloadException( + `Provided diff is trying to create relation "${relation}" but it already exists. Please generate a new diff and try again.` + ); + } + } else if (diffRelation.diff[0]?.kind === DiffKind.DELETE) { + const existingRelation = currentSnapshotWithHash.relations.find( + (r) => r.collection === diffRelation.collection && r.field === diffRelation.field + ); + + if (!existingRelation) { + throw new InvalidPayloadException( + `Provided diff is trying to delete relation "${relation}" but it does not exist. Please generate a new diff and try again.` + ); + } + } + } + + throw new InvalidPayloadException( + `Provided hash does not match the current instance's schema hash, indicating the schema has changed after this diff was generated. Please generate a new diff and try again.` + ); +} diff --git a/api/src/utils/validate-query.test.ts b/api/src/utils/validate-query.test.ts new file mode 100644 index 0000000000..d5924ff82a --- /dev/null +++ b/api/src/utils/validate-query.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, test, vi } from 'vitest'; +import { validateQuery } from './validate-query'; + +vi.mock('../env', async () => { + const actual = (await vi.importActual('../env')) as { default: Record }; + const MOCK_ENV = { + ...actual.default, + MAX_QUERY_LIMIT: 100, + }; + return { + default: MOCK_ENV, + getEnv: () => MOCK_ENV, + }; +}); + +describe('export', () => { + test.each(['csv', 'json', 'xml', 'yaml'])('should accept format %i', (format) => { + expect(() => validateQuery({ export: format } as any)).not.toThrowError(); + }); + + test('should error with invalid-format', () => { + expect(() => validateQuery({ export: 'invalid-format' } as any)).toThrowError('"export" must be one of'); + }); +}); diff --git a/api/src/utils/validate-query.ts b/api/src/utils/validate-query.ts index f4900e2b97..5b03158444 100644 --- a/api/src/utils/validate-query.ts +++ b/api/src/utils/validate-query.ts @@ -16,7 +16,7 @@ const querySchema = Joi.object({ page: Joi.number().integer().min(0), meta: Joi.array().items(Joi.string().valid('total_count', 'filter_count')), search: Joi.string(), - export: Joi.string().valid('json', 'csv', 'xml'), + export: Joi.string().valid('csv', 'json', 'xml', 'yaml'), aggregate: Joi.object(), deep: Joi.object(), alias: Joi.object(), diff --git a/api/src/utils/validate-snapshot.test.ts b/api/src/utils/validate-snapshot.test.ts new file mode 100644 index 0000000000..07be6b6d16 --- /dev/null +++ b/api/src/utils/validate-snapshot.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, test, vi } from 'vitest'; +import { Snapshot } from '../types/snapshot'; +import { validateSnapshot } from './validate-snapshot'; + +vi.mock('../../package.json', () => ({ + version: '9.22.4', +})); + +vi.mock('../database', () => ({ + getDatabaseClient: () => 'sqlite', +})); + +describe('should fail on invalid snapshot schema', () => { + test('empty snapshot', () => { + const snapshot = {} as Snapshot; + + expect(() => validateSnapshot(snapshot)).toThrowError('"version" is required'); + }); + + test('invalid version', () => { + const snapshot = { version: 0 } as Snapshot; + + expect(() => validateSnapshot(snapshot)).toThrowError('"version" must be [1]'); + }); + + test('invalid schema', () => { + const snapshot = { version: 1, directus: '9.22.4', collections: {} } as Snapshot; + + expect(() => validateSnapshot(snapshot)).toThrowError('"collections" must be an array'); + }); +}); + +describe('should require force option on version / vendor mismatch', () => { + test('directus version mismatch', () => { + const snapshot = { version: 1, directus: '9.22.3' } as Snapshot; + + expect(() => validateSnapshot(snapshot)).toThrowError( + "Provided snapshot's directus version 9.22.3 does not match the current instance's version 9.22.4" + ); + }); + + test('db vendor mismatch', () => { + const snapshot = { version: 1, directus: '9.22.4', vendor: 'postgres' } as Snapshot; + + expect(() => validateSnapshot(snapshot)).toThrowError( + "Provided snapshot's vendor postgres does not match the current instance's vendor sqlite." + ); + }); +}); + +test('should allow bypass on version / vendor mismatch via force option ', () => { + const snapshot = { version: 1, directus: '9.22.3', vendor: 'postgres' } as Snapshot; + + expect(validateSnapshot(snapshot, true)).toBeUndefined(); +}); diff --git a/api/src/utils/validate-snapshot.ts b/api/src/utils/validate-snapshot.ts new file mode 100644 index 0000000000..2a41440850 --- /dev/null +++ b/api/src/utils/validate-snapshot.ts @@ -0,0 +1,80 @@ +import { version as currentDirectusVersion } from '../../package.json'; +import { InvalidPayloadException } from '../exceptions'; +import { getDatabaseClient } from '../database'; +import Joi from 'joi'; +import { TYPES } from '@directus/shared/constants'; +import { ALIAS_TYPES } from '../constants'; +import { DatabaseClients, Snapshot } from '../types'; + +const snapshotJoiSchema = Joi.object({ + version: Joi.number().valid(1).required(), + directus: Joi.string().required(), + vendor: Joi.string() + .valid(...DatabaseClients) + .optional(), + collections: Joi.array().items( + Joi.object({ + collection: Joi.string(), + meta: Joi.any(), + schema: Joi.object({ + name: Joi.string(), + }), + }) + ), + fields: Joi.array().items( + Joi.object({ + collection: Joi.string(), + field: Joi.string(), + meta: Joi.any(), + schema: Joi.object({ + default_value: Joi.any(), + max_length: [Joi.number(), Joi.string(), Joi.valid(null)], + is_nullable: Joi.bool(), + }) + .unknown() + .allow(null), + type: Joi.string() + .valid(...TYPES, ...ALIAS_TYPES) + .allow(null), + }) + ), + relations: Joi.array().items( + Joi.object({ + collection: Joi.string(), + field: Joi.string(), + meta: Joi.any(), + related_collection: Joi.any(), + schema: Joi.any(), + }) + ), +}); + +/** + * Validates the snapshot against the current instance. + **/ +export function validateSnapshot(snapshot: Snapshot, force = false) { + const { error } = snapshotJoiSchema.validate(snapshot); + if (error) throw new InvalidPayloadException(error.message); + + // Bypass checks when "force" option is enabled + if (force) return; + + if (snapshot.directus !== currentDirectusVersion) { + throw new InvalidPayloadException( + `Provided snapshot's directus version ${snapshot.directus} does not match the current instance's version ${currentDirectusVersion}. You can bypass this check by passing the "force" query parameter.` + ); + } + + if (!snapshot.vendor) { + throw new InvalidPayloadException( + 'Provided snapshot does not contain the "vendor" property. You can bypass this check by passing the "force" query parameter.' + ); + } + + const currentVendor = getDatabaseClient(); + if (snapshot.vendor !== currentVendor) { + throw new InvalidPayloadException( + `Provided snapshot's vendor ${snapshot.vendor} does not match the current instance's vendor ${currentVendor}. You can bypass this check by passing the "force" query parameter.` + ); + } +} diff --git a/app/src/lang/translations/en-US.yaml b/app/src/lang/translations/en-US.yaml index 51743b3982..633ba88a1d 100644 --- a/app/src/lang/translations/en-US.yaml +++ b/app/src/lang/translations/en-US.yaml @@ -250,6 +250,7 @@ float: Float integer: Integer json: JSON xml: XML +yaml: YAML string: String text: Text time: Time diff --git a/app/src/views/private/components/export-sidebar-detail.vue b/app/src/views/private/components/export-sidebar-detail.vue index b1df838470..c5ef5b38f6 100644 --- a/app/src/views/private/components/export-sidebar-detail.vue +++ b/app/src/views/private/components/export-sidebar-detail.vue @@ -104,6 +104,10 @@ text: t('xml'), value: 'xml', }, + { + text: t('yaml'), + value: 'yaml', + }, ]" /> diff --git a/package.json b/package.json index b18546d8c3..df6b9f69ce 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "devDependencies": { "@directus/shared": "workspace:*", "@types/jest": "29.2.4", + "@types/js-yaml": "4.0.5", "@types/lodash": "4.14.191", "@types/seedrandom": "3.0.2", "@types/supertest": "2.0.12", @@ -42,6 +43,7 @@ "globby": "11.1.0", "jest": "29.3.1", "jest-environment-node": "29.3.1", + "js-yaml": "4.1.0", "json-to-graphql-query": "2.2.4", "knex": "2.4.1", "lint-staged": "13.1.0", diff --git a/packages/schema/package.json b/packages/schema/package.json index 6980d0ebd6..5ab61e7eac 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -43,7 +43,7 @@ }, "dependencies": { "knex": "2.4.1", - "knex-schema-inspector": "3.0.0" + "knex-schema-inspector": "3.0.1" }, "devDependencies": { "typescript": "4.9.4" diff --git a/packages/shared/package.json b/packages/shared/package.json index 6301d2e902..1b35a755c1 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -71,7 +71,7 @@ "geojson": "0.5.0", "joi": "17.7.0", "knex": "2.4.1", - "knex-schema-inspector": "3.0.0", + "knex-schema-inspector": "3.0.1", "lodash": "4.17.21", "micromustache": "8.0.3", "nanoid": "4.0.0", diff --git a/packages/specs/src/components/diff.yaml b/packages/specs/src/components/diff.yaml new file mode 100644 index 0000000000..a8cd9d7ef5 --- /dev/null +++ b/packages/specs/src/components/diff.yaml @@ -0,0 +1,46 @@ +type: object +properties: + hash: + type: string + diff: + type: object + properties: + collections: + type: array + items: + type: object + properties: + collection: + type: string + diff: + type: array + items: + type: object + fields: + type: array + items: + type: object + properties: + collection: + type: string + field: + type: string + diff: + type: array + items: + type: object + relations: + type: array + items: + type: object + properties: + collection: + type: string + field: + type: string + related_collection: + type: string + diff: + type: array + items: + type: object diff --git a/packages/specs/src/components/schema.yaml b/packages/specs/src/components/schema.yaml new file mode 100644 index 0000000000..9be3a47b7c --- /dev/null +++ b/packages/specs/src/components/schema.yaml @@ -0,0 +1,21 @@ +type: object +properties: + version: + type: integer + example: 1 + directus: + type: string + vendor: + type: string + collections: + type: array + items: + $ref: '../openapi.yaml#/components/schemas/Collections' + fields: + type: array + items: + $ref: '../openapi.yaml#/components/schemas/Fields' + relations: + type: array + items: + $ref: '../openapi.yaml#/components/schemas/Relations' diff --git a/packages/specs/src/openapi.yaml b/packages/specs/src/openapi.yaml index 99671d5b60..446409203a 100644 --- a/packages/specs/src/openapi.yaml +++ b/packages/specs/src/openapi.yaml @@ -73,6 +73,8 @@ tags: - name: Roles description: Roles are groups of users that share permissions. x-collection: directus_roles + - name: Schema + description: Retrieve and update the schema of an instance. - name: Server description: Access to where Directus runs. Allows you to make sure your server has everything needed to run the platform, and @@ -205,6 +207,14 @@ paths: /roles/{id}: $ref: './paths/roles/role.yaml' + # Schema + /schema/snapshot: + $ref: './paths/schema/snapshot.yaml' + /schema/apply: + $ref: './paths/schema/apply.yaml' + /schema/diff: + $ref: './paths/schema/diff.yaml' + # Server /server/info: $ref: './paths/server/info.yaml' @@ -263,6 +273,8 @@ components: $ref: './components/preset.yaml' Collections: $ref: './components/collection.yaml' + Diff: + $ref: './components/diff.yaml' Fields: $ref: './components/field.yaml' Files: @@ -283,6 +295,8 @@ components: $ref: './components/revision.yaml' Roles: $ref: './components/role.yaml' + Schema: + $ref: './components/schema.yaml' Settings: $ref: './components/setting.yaml' Users: @@ -330,6 +344,8 @@ components: $ref: './parameters/fields.yaml' Mode: $ref: './parameters/mode.yaml' + Export: + $ref: './parameters/export.yaml' responses: NotFoundError: $ref: './responses/notFoundError.yaml' diff --git a/packages/specs/src/parameters/export.yaml b/packages/specs/src/parameters/export.yaml new file mode 100644 index 0000000000..6d26116281 --- /dev/null +++ b/packages/specs/src/parameters/export.yaml @@ -0,0 +1,11 @@ +name: export +description: Saves the API response to a file. Accepts one of "csv", "json", "xml", "yaml". +in: query +required: false +schema: + type: string + enum: + - csv + - json + - xml + - yaml diff --git a/packages/specs/src/paths/items/items.yaml b/packages/specs/src/paths/items/items.yaml index 30a9802708..5e44ba8c6b 100644 --- a/packages/specs/src/paths/items/items.yaml +++ b/packages/specs/src/paths/items/items.yaml @@ -43,6 +43,7 @@ post: requestBody: content: application/json: + # Intentionally empty, see https://github.com/directus/directus/pull/16294 schema: responses: '200': diff --git a/packages/specs/src/paths/schema/apply.yaml b/packages/specs/src/paths/schema/apply.yaml new file mode 100644 index 0000000000..5c44f062f6 --- /dev/null +++ b/packages/specs/src/paths/schema/apply.yaml @@ -0,0 +1,22 @@ +post: + summary: Apply Schema Difference + description: + Update the instance's schema by passing the diff previously retrieved via `/schema/diff` endpoint in the request + body. This endpoint is only available to admin users. + operationId: schemaApply + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + data: + $ref: '../../openapi.yaml#/components/schemas/Diff' + responses: + '204': + description: Successful request + '403': + $ref: '../../openapi.yaml#/components/responses/UnauthorizedError' + tags: + - Schema diff --git a/packages/specs/src/paths/schema/diff.yaml b/packages/specs/src/paths/schema/diff.yaml new file mode 100644 index 0000000000..e85ebd3a3d --- /dev/null +++ b/packages/specs/src/paths/schema/diff.yaml @@ -0,0 +1,45 @@ +post: + summary: Retrieve Schema Difference + description: + Compare the current instance's schema against the schema snapshot in JSON request body or a JSON/YAML file and + retrieve the difference. This endpoint is only available to admin users. + operationId: schemaDiff + parameters: + - name: force + description: Bypass version and database vendor restrictions. + in: query + required: false + schema: + type: boolean + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + data: + $ref: '../../openapi.yaml#/components/schemas/Schema' + multipart/form-data: + schema: + type: object + properties: + file: + type: string + format: binary + responses: + '200': + description: Successful request + content: + application/json: + schema: + type: object + properties: + data: + $ref: '../../openapi.yaml#/components/schemas/Diff' + '204': + description: No schema difference. + '403': + $ref: '../../openapi.yaml#/components/responses/UnauthorizedError' + tags: + - Schema diff --git a/packages/specs/src/paths/schema/snapshot.yaml b/packages/specs/src/paths/schema/snapshot.yaml new file mode 100644 index 0000000000..e7fbc7d19f --- /dev/null +++ b/packages/specs/src/paths/schema/snapshot.yaml @@ -0,0 +1,24 @@ +get: + summary: Retrieve Schema Snapshot + description: Retrieve the current schema. This endpoint is only available to admin users. + operationId: schemaSnapshot + parameters: + - $ref: '../../openapi.yaml#/components/parameters/Export' + responses: + '200': + description: Successful request + content: + application/json: + schema: + type: object + properties: + data: + $ref: '../../openapi.yaml#/components/schemas/Schema' + text/yaml: + schema: + type: string + format: binary + '403': + $ref: '../../openapi.yaml#/components/responses/UnauthorizedError' + tags: + - Schema diff --git a/packages/utils/src/node/index.ts b/packages/utils/src/node/index.ts index 13eb4d4af8..71648c45a1 100644 --- a/packages/utils/src/node/index.ts +++ b/packages/utils/src/node/index.ts @@ -1 +1,2 @@ +export * from './readable-stream-to-string.js'; export * from './is-readable-stream.js'; diff --git a/packages/utils/src/node/readable-stream-to-string.test.ts b/packages/utils/src/node/readable-stream-to-string.test.ts new file mode 100644 index 0000000000..2987cb1eaa --- /dev/null +++ b/packages/utils/src/node/readable-stream-to-string.test.ts @@ -0,0 +1,11 @@ +import { test, expect } from 'vitest'; +import { Readable } from 'node:stream'; + +import { readableStreamToString } from './readable-stream-to-string.js'; + +test.each([Readable.from('test', { encoding: 'utf8' }), Readable.from(Buffer.from([0x74, 0x65, 0x73, 0x74]))])( + 'Returns readable stream as string', + async (readableStream) => { + expect(readableStreamToString(readableStream)).resolves.toBe('test'); + } +); diff --git a/packages/utils/src/node/readable-stream-to-string.ts b/packages/utils/src/node/readable-stream-to-string.ts new file mode 100644 index 0000000000..71dcc40d7e --- /dev/null +++ b/packages/utils/src/node/readable-stream-to-string.ts @@ -0,0 +1,11 @@ +import type { Readable } from 'node:stream'; + +export const readableStreamToString = async (stream: Readable): Promise => { + const chunks = []; + + for await (const chunk of stream) { + chunks.push(Buffer.from(chunk)); + } + + return Buffer.concat(chunks).toString('utf8'); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 975166529d..42c2ba2850 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,6 +6,7 @@ importers: specifiers: '@directus/shared': workspace:* '@types/jest': 29.2.4 + '@types/js-yaml': 4.0.5 '@types/lodash': 4.14.191 '@types/seedrandom': 3.0.2 '@types/supertest': 2.0.12 @@ -21,6 +22,7 @@ importers: globby: 11.1.0 jest: 29.3.1 jest-environment-node: 29.3.1 + js-yaml: 4.1.0 json-to-graphql-query: 2.2.4 knex: 2.4.1 lint-staged: 13.1.0 @@ -36,6 +38,7 @@ importers: devDependencies: '@directus/shared': link:packages/shared '@types/jest': 29.2.4 + '@types/js-yaml': 4.0.5 '@types/lodash': 4.14.191 '@types/seedrandom': 3.0.2 '@types/supertest': 2.0.12 @@ -51,6 +54,7 @@ importers: globby: 11.1.0 jest: 29.3.1 jest-environment-node: 29.3.1 + js-yaml: 4.1.0 json-to-graphql-query: 2.2.4 knex: 2.4.1 lint-staged: 13.1.0 @@ -80,6 +84,7 @@ importers: '@directus/storage-driver-gcs': workspace:* '@directus/storage-driver-local': workspace:* '@directus/storage-driver-s3': workspace:* + '@directus/utils': workspace:* '@godaddy/terminus': 4.11.2 '@keyv/redis': 2.5.4 '@ngneat/falso': 6.3.2 @@ -164,7 +169,7 @@ importers: keyv-memcache: 1.3.3 knex: 2.4.1 knex-mock-client: 1.11.0 - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 ldapjs: 2.3.3 liquidjs: 10.3.3 lodash: 4.17.21 @@ -226,6 +231,7 @@ importers: '@directus/storage-driver-gcs': link:../packages/storage-driver-gcs '@directus/storage-driver-local': link:../packages/storage-driver-local '@directus/storage-driver-s3': link:../packages/storage-driver-s3 + '@directus/utils': link:../packages/utils '@godaddy/terminus': 4.11.2 '@rollup/plugin-alias': 4.0.2_rollup@3.7.5 '@rollup/plugin-virtual': 3.0.1_rollup@3.7.5 @@ -268,7 +274,7 @@ importers: jsonwebtoken: 9.0.0 keyv: 4.5.2 knex: 2.4.1_i7frrkiwsbjh7pywr5yuup4dni - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 ldapjs: 2.3.3 liquidjs: 10.3.3 lodash: 4.17.21 @@ -648,11 +654,11 @@ importers: packages/schema: specifiers: knex: 2.4.1 - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 typescript: 4.9.4 dependencies: knex: 2.4.1 - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 devDependencies: typescript: 4.9.4 @@ -672,7 +678,7 @@ importers: geojson: 0.5.0 joi: 17.7.0 knex: 2.4.1 - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 lodash: 4.17.21 micromustache: 8.0.3 nanoid: 4.0.0 @@ -692,7 +698,7 @@ importers: geojson: 0.5.0 joi: 17.7.0 knex: 2.4.1 - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 lodash: 4.17.21 micromustache: 8.0.3 nanoid: 4.0.0 @@ -9040,7 +9046,7 @@ packages: mississippi: 3.0.0 mkdirp: 0.5.6 move-concurrently: 1.0.1 - promise-inflight: 1.0.1 + promise-inflight: 1.0.1_bluebird@3.7.2 rimraf: 2.7.1 ssri: 6.0.2 unique-filename: 1.1.1 @@ -9064,7 +9070,7 @@ packages: minipass-pipeline: 1.2.4 mkdirp: 1.0.4 p-map: 4.0.0 - promise-inflight: 1.0.1 + promise-inflight: 1.0.1_bluebird@3.7.2 rimraf: 3.0.2 ssri: 8.0.1 tar: 6.1.13 @@ -14284,8 +14290,8 @@ packages: lodash.clonedeep: 4.5.0 dev: true - /knex-schema-inspector/3.0.0: - resolution: {integrity: sha512-J3Aeh4kIZD2sYUjnpak+sdVtegGVkkhCF5/VSziocSZpseW4MwTcnxHbttsaT3tA85INuPxxUBxFgyCm9tMZDQ==} + /knex-schema-inspector/3.0.1: + resolution: {integrity: sha512-ofglN/HoHYhNxQJbuuKEj0vtVsBpEVlbvfoFd9lPn1ABzJHdwZmKLquCYbVLTXZIN0O6y6DDEuN4EqBKMhH+xQ==} dependencies: lodash.flatten: 4.4.0 lodash.isnil: 4.0.0 @@ -17120,13 +17126,15 @@ packages: engines: {node: '>=0.4.0'} dev: true - /promise-inflight/1.0.1: + /promise-inflight/1.0.1_bluebird@3.7.2: resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} peerDependencies: bluebird: '*' peerDependenciesMeta: bluebird: optional: true + dependencies: + bluebird: 3.7.2 /promise-retry/2.0.1: resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==} diff --git a/tests-blackbox/common/config.ts b/tests-blackbox/common/config.ts index 16acab0b09..73fd110001 100644 --- a/tests-blackbox/common/config.ts +++ b/tests-blackbox/common/config.ts @@ -56,6 +56,7 @@ const directusConfig = { LOG_LEVEL: logLevel, SERVE_APP: 'false', DB_EXCLUDE_TABLES: 'knex_migrations,knex_migrations_lock,spatial_ref_sys,sysdiagrams', + MAX_PAYLOAD_SIZE: '10mb', EXTENSIONS_PATH: './tests-blackbox/extensions', ...directusAuthConfig, }; diff --git a/tests-blackbox/common/functions.ts b/tests-blackbox/common/functions.ts index d51242ea60..15dbb499df 100644 --- a/tests-blackbox/common/functions.ts +++ b/tests-blackbox/common/functions.ts @@ -442,6 +442,8 @@ export async function CreateFieldM2M(vendor: string, options: OptionsCreateField schema: options.fieldSchema, }; + const isSelfReferencing = options.collection === options.otherCollection; + if (!fieldOptions.meta.special) { fieldOptions.meta.special = ['m2m']; } else if (!fieldOptions.meta.special.includes('m2m')) { @@ -483,7 +485,7 @@ export async function CreateFieldM2M(vendor: string, options: OptionsCreateField const junctionField = await CreateField(vendor, junctionFieldOptions); - const otherJunctionFieldName = `${options.otherCollection}_id`; + const otherJunctionFieldName = `${options.otherCollection}_id${isSelfReferencing ? '2' : ''}`; const otherJunctionFieldOptions: OptionsCreateField = { collection: options.junctionCollection, field: otherJunctionFieldName, diff --git a/tests-blackbox/common/seed-functions.ts b/tests-blackbox/common/seed-functions.ts index 0ec039205a..819bbb69dc 100644 --- a/tests-blackbox/common/seed-functions.ts +++ b/tests-blackbox/common/seed-functions.ts @@ -7,7 +7,9 @@ const FIVE_YEARS_IN_MILLISECONDS = 5 * 365 * 24 * 60 * 60 * 1000; type OptionsSeedGenerateBase = { quantity: number; - seed?: string | undefined; + seed?: string; + vendor?: string; + isDefaultValue?: boolean; }; export type OptionsSeedGeneratePrimaryKeys = OptionsSeedGenerateBase & OptionsSeedGenerateInteger; @@ -262,6 +264,18 @@ function generateDate(options: OptionsSeedGenerateDate) { } } + if (options.isDefaultValue && options.vendor === 'oracle') { + for (let i = 0; i < values.length; i++) { + values[i] = new Date(values[i]) + .toLocaleDateString('en-GB', { + day: 'numeric', + month: 'short', + year: 'numeric', + }) + .replace(/ /g, '-'); + } + } + return values; } @@ -272,6 +286,12 @@ function generateDateTime(options: OptionsSeedGenerateDateTime) { values[i] = values[i].slice(0, -5); } + if (options.isDefaultValue && options.vendor === 'oracle') { + for (let index = 0; index < values.length; index++) { + values[index] = 'CURRENT_TIMESTAMP'; + } + } + return values; } @@ -315,6 +335,12 @@ function generateTime(options: OptionsSeedGenerateTime) { } } + if (options.isDefaultValue && options.vendor === 'oracle') { + for (let index = 0; index < values.length; index++) { + values[index] = 'CURRENT_TIMESTAMP'; + } + } + return values; } @@ -354,5 +380,17 @@ function generateTimestamp(options: OptionsSeedGenerateTimestamp) { values[index] = values[index].slice(0, 20) + '000Z'; } + if (options.isDefaultValue && options.vendor) { + if (['mysql', 'mysql5', 'maria'].includes(options.vendor)) { + for (let index = 0; index < values.length; index++) { + values[index] = new Date(values[index]).toISOString().replace(/([^T]+)T([^.]+).*/g, '$1 $2'); + } + } else if (options.vendor === 'oracle') { + for (let index = 0; index < values.length; index++) { + values[index] = 'CURRENT_TIMESTAMP'; + } + } + } + return values; } diff --git a/tests-blackbox/routes/items/seed-all-field-types.ts b/tests-blackbox/routes/items/seed-all-field-types.ts index b34f0d4231..ff276b8b49 100644 --- a/tests-blackbox/routes/items/seed-all-field-types.ts +++ b/tests-blackbox/routes/items/seed-all-field-types.ts @@ -21,16 +21,38 @@ export function getTestsAllTypesSchema(): TestsFieldSchema { return fieldSchema; } -export const seedAllFieldTypesStructure = async (vendor: string, collection: string) => { +export const seedAllFieldTypesStructure = async (vendor: string, collection: string, setDefaultValues = false) => { try { const fieldSchema = getTestsAllTypesSchema(); // Create fields for (const key of Object.keys(fieldSchema)) { + let meta = {}; + let schema = {}; + + const fieldType = fieldSchema[key].type; + + if (fieldType === 'uuid') { + meta = { special: ['uuid'] }; + } + + if (setDefaultValues && SeedFunctions.generateValues[fieldType as keyof typeof SeedFunctions.generateValues]) { + schema = { + default_value: SeedFunctions.generateValues[fieldType as keyof typeof SeedFunctions.generateValues]({ + quantity: 1, + seed: `${collection}_${fieldType}`, + vendor, + isDefaultValue: true, + })[0], + }; + } + await CreateField(vendor, { collection: collection, field: fieldSchema[key].field.toLowerCase(), - type: fieldSchema[key].type, + type: fieldType, + meta, + schema, }); } diff --git a/tests-blackbox/routes/schema/schema.seed.ts b/tests-blackbox/routes/schema/schema.seed.ts new file mode 100644 index 0000000000..6389d89ece --- /dev/null +++ b/tests-blackbox/routes/schema/schema.seed.ts @@ -0,0 +1,250 @@ +import vendors from '@common/get-dbs-to-test'; +import { + CreateCollection, + CreateField, + DeleteCollection, + PRIMARY_KEY_TYPES, + CreateFieldM2M, + CreateFieldM2O, + CreateFieldO2M, + DeleteField, + PrimaryKeyType, +} from '@common/index'; +import { seedAllFieldTypesStructure } from '../items/seed-all-field-types'; + +export const collectionAll = 'test_schema_all'; +export const collectionM2O = 'test_schema_m2o'; +export const collectionM2O2 = 'test_schema_m2o2'; +export const collectionO2M = 'test_schema_o2m'; +export const collectionO2M2 = 'test_schema_o2m2'; +export const collectionM2M = 'test_schema_m2m'; +export const collectionM2M2 = 'test_schema_m2m2'; +export const junctionM2M = 'test_schema_jm2m'; +export const junctionM2M2 = 'test_schema_jm2m2'; +export const collectionSelf = 'test_schema_self'; +export const junctionSelfM2M = 'test_schema_jm2m_self'; +export const tempTestCollection = 'temp_test_collection'; + +export type SampleItem = { + id?: number | string; + name: string; +}; + +export type Ingredient = { + id?: number | string; + name: string; +}; + +export type Supplier = { + id?: number | string; + name: string; +}; + +export const deleteAllCollections = async (vendor: string, pkType: PrimaryKeyType, setDefaultValues: boolean) => { + const suffix = setDefaultValues ? '2' : ''; + + // Setup + const localCollectionAll = `${collectionAll}_${pkType}${suffix}`; + const localCollectionM2M = `${collectionM2M}_${pkType}${suffix}`; + const localCollectionM2M2 = `${collectionM2M2}_${pkType}${suffix}`; + const localJunctionAllM2M = `${junctionM2M}_${pkType}${suffix}`; + const localJunctionM2MM2M2 = `${junctionM2M2}_${pkType}${suffix}`; + const localCollectionM2O = `${collectionM2O}_${pkType}${suffix}`; + const localCollectionM2O2 = `${collectionM2O2}_${pkType}${suffix}`; + const localCollectionO2M = `${collectionO2M}_${pkType}${suffix}`; + const localCollectionO2M2 = `${collectionO2M2}_${pkType}${suffix}`; + const localCollectionSelf = `${collectionSelf}_${pkType}${suffix}`; + const localJunctionSelfM2M = `${junctionSelfM2M}_${pkType}${suffix}`; + + // Delete existing collections + await DeleteField(vendor, { collection: localCollectionSelf, field: 'self_id' }); + await DeleteField(vendor, { collection: localCollectionSelf, field: 'self_id' }); + await DeleteCollection(vendor, { collection: localJunctionSelfM2M }); + await DeleteCollection(vendor, { collection: localCollectionSelf }); + await DeleteCollection(vendor, { collection: localCollectionO2M2 }); + await DeleteCollection(vendor, { collection: localCollectionO2M }); + await DeleteCollection(vendor, { collection: localJunctionM2MM2M2 }); + await DeleteCollection(vendor, { collection: localJunctionAllM2M }); + await DeleteCollection(vendor, { collection: localCollectionM2M2 }); + await DeleteCollection(vendor, { collection: localCollectionM2M }); + await DeleteCollection(vendor, { collection: localCollectionAll }); + await DeleteCollection(vendor, { collection: localCollectionM2O }); + await DeleteCollection(vendor, { collection: localCollectionM2O2 }); +}; + +export const seedDBStructure = () => { + it.each(vendors)( + '%s', + async (vendor) => { + for (const setDefaultValues of [false, true]) { + const suffix = setDefaultValues ? '2' : ''; + + for (const pkType of PRIMARY_KEY_TYPES) { + try { + const localCollectionAll = `${collectionAll}_${pkType}${suffix}`; + const localCollectionM2M = `${collectionM2M}_${pkType}${suffix}`; + const localCollectionM2M2 = `${collectionM2M2}_${pkType}${suffix}`; + const localJunctionAllM2M = `${junctionM2M}_${pkType}${suffix}`; + const localJunctionM2MM2M2 = `${junctionM2M2}_${pkType}${suffix}`; + const localCollectionM2O = `${collectionM2O}_${pkType}${suffix}`; + const localCollectionM2O2 = `${collectionM2O2}_${pkType}${suffix}`; + const localCollectionO2M = `${collectionO2M}_${pkType}${suffix}`; + const localCollectionO2M2 = `${collectionO2M2}_${pkType}${suffix}`; + const localCollectionSelf = `${collectionSelf}_${pkType}${suffix}`; + const localJunctionSelfM2M = `${junctionSelfM2M}_${pkType}${suffix}`; + + // Delete existing collections + await deleteAllCollections(vendor, pkType, setDefaultValues); + + // Delete the temp collection created in previous test run + await DeleteCollection(vendor, { collection: tempTestCollection }); + + // Create All collection + await CreateCollection(vendor, { + collection: localCollectionAll, + primaryKeyType: pkType, + }); + + await CreateField(vendor, { + collection: localCollectionAll, + field: 'name', + type: 'string', + }); + + // Create M2M collection + await CreateCollection(vendor, { + collection: localCollectionM2M, + primaryKeyType: pkType, + }); + + // Create nested M2M collection + await CreateCollection(vendor, { + collection: localCollectionM2M2, + primaryKeyType: pkType, + }); + + // Create M2M relationships + await CreateFieldM2M(vendor, { + collection: localCollectionAll, + field: 'all_m2m', + otherCollection: localCollectionM2M, + otherField: 'm2m_all', + junctionCollection: localJunctionAllM2M, + primaryKeyType: pkType, + }); + + await CreateFieldM2M(vendor, { + collection: localCollectionM2M, + field: 'm2m_m2m2', + otherCollection: localCollectionM2M2, + otherField: 'm2m2_m2m', + junctionCollection: localJunctionM2MM2M2, + primaryKeyType: pkType, + }); + + // Create M2O collection + await CreateCollection(vendor, { + collection: localCollectionM2O, + primaryKeyType: pkType, + }); + + // Create nested M2O collection + await CreateCollection(vendor, { + collection: localCollectionM2O2, + primaryKeyType: pkType, + }); + + // Create M2O relationships + await CreateFieldM2O(vendor, { + collection: localCollectionAll, + field: 'all_id', + primaryKeyType: pkType, + otherCollection: localCollectionM2O, + }); + + await CreateFieldM2O(vendor, { + collection: localCollectionM2O, + field: 'm2o_id', + primaryKeyType: pkType, + otherCollection: localCollectionM2O2, + }); + + // Create O2M collection + await CreateCollection(vendor, { + collection: localCollectionO2M, + primaryKeyType: pkType, + }); + + // Create nested O2M collection + await CreateCollection(vendor, { + collection: localCollectionO2M2, + primaryKeyType: pkType, + }); + + // Create O2M relationships + await CreateFieldO2M(vendor, { + collection: localCollectionAll, + field: 'o2m', + otherCollection: localCollectionO2M, + otherField: 'all_id', + primaryKeyType: pkType, + }); + + await CreateFieldO2M(vendor, { + collection: localCollectionO2M, + field: 'o2m2', + otherCollection: localCollectionO2M2, + otherField: 'o2m_id', + primaryKeyType: pkType, + }); + + // Create Self collection + await CreateCollection(vendor, { + collection: localCollectionSelf, + primaryKeyType: pkType, + }); + + // Create self M2M relationship + await CreateFieldM2M(vendor, { + collection: localCollectionSelf, + field: 'm2m', + otherCollection: localCollectionSelf, + otherField: 'm2m2', + junctionCollection: localJunctionSelfM2M, + primaryKeyType: pkType, + relationSchema: { + on_delete: 'SET NULL', + }, + otherRelationSchema: + vendor === 'mssql' + ? { on_delete: 'NO ACTION' } + : { + on_delete: 'SET NULL', + }, + }); + + // Create self O2M relationship + await CreateFieldO2M(vendor, { + collection: localCollectionSelf, + field: 'o2m', + otherCollection: localCollectionSelf, + otherField: 'self_id', + primaryKeyType: pkType, + relationSchema: { + on_delete: 'NO ACTION', + }, + }); + + await seedAllFieldTypesStructure(vendor, localCollectionAll, setDefaultValues); + await seedAllFieldTypesStructure(vendor, localCollectionSelf, setDefaultValues); + + expect(true).toBeTruthy(); + } catch (error) { + expect(error).toBeFalsy(); + } + } + } + }, + 1800000 + ); +}; diff --git a/tests-blackbox/routes/schema/schema.test.ts b/tests-blackbox/routes/schema/schema.test.ts new file mode 100644 index 0000000000..dfe24c0b5c --- /dev/null +++ b/tests-blackbox/routes/schema/schema.test.ts @@ -0,0 +1,559 @@ +import request from 'supertest'; +import { getUrl } from '@common/config'; +import vendors from '@common/get-dbs-to-test'; +import * as common from '@common/index'; +import { + collectionAll, + collectionM2M, + collectionM2M2, + collectionM2O, + collectionM2O2, + collectionO2M, + collectionO2M2, + collectionSelf, + deleteAllCollections, + junctionM2M, + junctionM2M2, + junctionSelfM2M, + tempTestCollection, +} from './schema.seed'; +import { cloneDeep } from 'lodash'; +import { PrimaryKeyType, PRIMARY_KEY_TYPES } from '@common/index'; +import { load as loadYaml } from 'js-yaml'; +import { version as currentDirectusVersion } from '../../../api/package.json'; + +describe('Schema Snapshots', () => { + const snapshotsCacheOriginal: { + [vendor: string]: any; + } = {}; + + const snapshotsCacheOriginalYaml: { + [vendor: string]: any; + } = {}; + + const snapshotsCacheEmpty: { + [vendor: string]: any; + } = {}; + + describe('GET /schema/snapshot', () => { + common.DisableTestCachingSetup(); + + describe('denies non-admin users', () => { + it.each(vendors)('%s', async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.APP_ACCESS.TOKEN}`); + const response2 = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.API_ONLY.TOKEN}`); + const response3 = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.NO_ROLE.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(403); + expect(response2.statusCode).toEqual(403); + expect(response3.statusCode).toEqual(403); + }); + }); + + describe('retrieves a snapshot (JSON)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(200); + + snapshotsCacheOriginal[vendor] = response.body.data; + }, + 300000 + ); + }); + + describe('retrieves a snapshot (YAML)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .query({ export: 'yaml' }) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(200); + + snapshotsCacheOriginalYaml[vendor] = response.text; + }, + 300000 + ); + }); + + describe('remove tables', () => { + describe.each(common.PRIMARY_KEY_TYPES)('%s primary keys', (pkType) => { + it.each(vendors)( + '%s', + async (vendor) => { + for (const setDefaultValues of [false, true]) { + // Delete existing collections + await deleteAllCollections(vendor, pkType, setDefaultValues); + } + + await assertCollectionsDeleted(vendor, pkType); + }, + 300000 + ); + }); + }); + }); + + common.ClearCaches(); + + describe('retrieves empty snapshot', () => { + it.each(vendors)( + '%s', + async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(200); + expect(snapshotsCacheEmpty[vendor]).not.toEqual(snapshotsCacheOriginal[vendor]); + + snapshotsCacheEmpty[vendor] = response.body.data; + }, + 300000 + ); + }); + + describe('POST /schema/diff', () => { + describe('denies non-admin users', () => { + it.each(vendors)('%s', async (vendor) => { + // Action + const currentVendor = vendor.replace(/[0-9]/g, ''); + const response = await request(getUrl(vendor)) + .post('/schema/diff') + .send({ + version: 1, + directus: currentDirectusVersion, + vendor: currentVendor, + collections: [], + fields: [], + relations: [], + }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.APP_ACCESS.TOKEN}`); + const response2 = await request(getUrl(vendor)) + .post('/schema/diff') + .send({ + version: 1, + directus: currentDirectusVersion, + vendor: currentVendor, + collections: [], + fields: [], + relations: [], + }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.API_ONLY.TOKEN}`); + const response3 = await request(getUrl(vendor)) + .post('/schema/diff') + .send({ + version: 1, + directus: currentDirectusVersion, + vendor: currentVendor, + collections: [], + fields: [], + relations: [], + }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.NO_ROLE.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(403); + expect(response2.statusCode).toEqual(403); + expect(response3.statusCode).toEqual(403); + }); + }); + + describe('returns diffs with empty snapshot', () => { + it.each(vendors)( + '%s', + async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheEmpty[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + expect(response.statusCode).toEqual(204); + }, + 300000 + ); + }); + + describe('returns diffs with original snapshot', () => { + it.each(vendors)( + '%s', + async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheOriginal[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(200); + expect(response.body.data?.diff?.collections?.length).toBe(66); + expect(response.body.data?.diff?.fields?.length).toBe(342); + expect(response.body.data?.diff?.relations?.length).toBe(66); + }, + 300000 + ); + }); + }); + + describe('POST /schema/apply', () => { + describe('denies non-admin users', () => { + it.each(vendors)('%s', async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send({ data: true }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.APP_ACCESS.TOKEN}`); + const response2 = await request(getUrl(vendor)) + .post('/schema/apply') + .send({ data: true }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.API_ONLY.TOKEN}`); + const response3 = await request(getUrl(vendor)) + .post('/schema/apply') + .send({ data: true }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.NO_ROLE.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(403); + expect(response2.statusCode).toEqual(403); + expect(response3.statusCode).toEqual(403); + }); + }); + + describe('applies a snapshot (JSON)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheOriginal[vendor]).toBeDefined(); + + // Action + const responseDiff = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheOriginal[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send(responseDiff.body.data) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(204); + }, + 1200000 + ); + }); + + describe('retrieves the same snapshot after applying a snapshot (JSON)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheOriginal[vendor]).toBeDefined(); + + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const curSnapshot = cloneDeep(response.body.data); + const oldSnapshot = cloneDeep(snapshotsCacheOriginal[vendor]); + + parseSnapshot(vendor, curSnapshot); + parseSnapshot(vendor, oldSnapshot); + + // Assert + expect(response.statusCode).toEqual(200); + expect(curSnapshot).toStrictEqual(oldSnapshot); + }, + 300000 + ); + }); + + describe('applies empty snapshot', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheEmpty[vendor]).toBeDefined(); + + // Action + const responseDiff = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheEmpty[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send(responseDiff.body.data) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(204); + }, + 1200000 + ); + }); + + describe('ensure that tables are removed', () => { + describe.each(common.PRIMARY_KEY_TYPES)('%s primary keys', (pkType) => { + it.each(vendors)( + '%s', + async (vendor) => { + await assertCollectionsDeleted(vendor, pkType); + }, + 600000 + ); + }); + }); + + describe('applies a snapshot (YAML)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheOriginalYaml[vendor]).toBeDefined(); + + // Action + const responseDiff = await request(getUrl(vendor)) + .post('/schema/diff') + .attach('file', Buffer.from(snapshotsCacheOriginalYaml[vendor])) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send(responseDiff.body.data) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(204); + }, + 1200000 + ); + }); + + describe('retrieves the same snapshot after applying a snapshot (YAML)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheOriginalYaml[vendor]).toBeDefined(); + + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .query({ export: 'yaml' }) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const curSnapshot = await loadYaml(response.text); + const oldSnapshot = cloneDeep(snapshotsCacheOriginal[vendor]); + + parseSnapshot(vendor, curSnapshot); + parseSnapshot(vendor, oldSnapshot); + + // Assert + expect(response.statusCode).toEqual(200); + expect(curSnapshot).toStrictEqual(oldSnapshot); + }, + 300000 + ); + }); + }); + + common.ClearCaches(); + + describe('Hash Tests', () => { + describe('with deleted fields', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheEmpty[vendor]).toBeDefined(); + + // Action + const responseDiff = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheEmpty[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + for (const pkType of PRIMARY_KEY_TYPES) { + await request(getUrl(vendor)) + .delete(`/fields/${collectionSelf}_${pkType}/self_id`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + } + + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send(responseDiff.body.data) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(400); + expect(response.text).toContain('Please generate a new diff and try again.'); + }, + 1200000 + ); + }); + + describe('with new collection', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheEmpty[vendor]).toBeDefined(); + + // Action + const responseDiff = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheEmpty[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + await request(getUrl(vendor)) + .post(`/collections`) + .send({ + collection: tempTestCollection, + fields: [ + { + field: 'id', + type: 'integer', + meta: { hidden: true, interface: 'input', readonly: true }, + schema: { is_primary_key: true, has_auto_increment: true }, + }, + ], + schema: {}, + meta: { singleton: false }, + }) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send(responseDiff.body.data) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(400); + expect(response.text).toContain('Please generate a new diff and try again.'); + }, + 1200000 + ); + }); + }); +}); + +function parseSnapshot(vendor: string, snapshot: any) { + if (vendor === 'cockroachdb') { + if (snapshot.fields) { + for (const field of snapshot.fields) { + if ( + field.schema?.default_value && + ['integer', 'bigInteger'].includes(field.type) && + typeof field.schema.default_value === 'string' && + field.schema.default_value.startsWith('nextval(') + ) { + field.schema.default_value = field.schema.default_value.replace( + /(_seq)\d*('::STRING::REGCLASS\))/, + `_seq'::STRING::REGCLASS)` + ); + } + } + } + } +} + +async function assertCollectionsDeleted(vendor: string, pkType: PrimaryKeyType) { + for (const setDefaultValues of [false, true]) { + const suffix = setDefaultValues ? '2' : ''; + + // Setup + const localCollectionAll = `${collectionAll}_${pkType}${suffix}`; + const localCollectionM2M = `${collectionM2M}_${pkType}${suffix}`; + const localCollectionM2M2 = `${collectionM2M2}_${pkType}${suffix}`; + const localJunctionAllM2M = `${junctionM2M}_${pkType}${suffix}`; + const localJunctionM2MM2M2 = `${junctionM2M2}_${pkType}${suffix}`; + const localCollectionM2O = `${collectionM2O}_${pkType}${suffix}`; + const localCollectionM2O2 = `${collectionM2O2}_${pkType}${suffix}`; + const localCollectionO2M = `${collectionO2M}_${pkType}${suffix}`; + const localCollectionO2M2 = `${collectionO2M2}_${pkType}${suffix}`; + const localCollectionSelf = `${collectionSelf}_${pkType}${suffix}`; + const localJunctionSelfM2M = `${junctionSelfM2M}_${pkType}${suffix}`; + + const response = await request(getUrl(vendor)) + .get(`/items/${localJunctionSelfM2M}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response2 = await request(getUrl(vendor)) + .get(`/items/${localCollectionSelf}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response3 = await request(getUrl(vendor)) + .get(`/items/${localCollectionO2M2}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response4 = await request(getUrl(vendor)) + .get(`/items/${localCollectionO2M}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response5 = await request(getUrl(vendor)) + .get(`/items/${localJunctionM2MM2M2}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response6 = await request(getUrl(vendor)) + .get(`/items/${localJunctionAllM2M}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response7 = await request(getUrl(vendor)) + .get(`/items/${localCollectionM2M2}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response8 = await request(getUrl(vendor)) + .get(`/items/${localCollectionM2M}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response9 = await request(getUrl(vendor)) + .get(`/items/${localCollectionAll}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response10 = await request(getUrl(vendor)) + .get(`/items/${localCollectionM2O}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response11 = await request(getUrl(vendor)) + .get(`/items/${localCollectionM2O2}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(403); + expect(response2.statusCode).toEqual(403); + expect(response3.statusCode).toEqual(403); + expect(response4.statusCode).toEqual(403); + expect(response5.statusCode).toEqual(403); + expect(response6.statusCode).toEqual(403); + expect(response7.statusCode).toEqual(403); + expect(response8.statusCode).toEqual(403); + expect(response9.statusCode).toEqual(403); + expect(response10.statusCode).toEqual(403); + expect(response11.statusCode).toEqual(403); + } +} diff --git a/tests-blackbox/setup/sequentialTests.js b/tests-blackbox/setup/sequentialTests.js index f7b6a0b95c..abcd433f23 100644 --- a/tests-blackbox/setup/sequentialTests.js +++ b/tests-blackbox/setup/sequentialTests.js @@ -3,6 +3,7 @@ exports.list = { before: [ { testFilePath: '/common/seed-database.test.ts' }, { testFilePath: '/common/common.test.ts' }, + { testFilePath: '/routes/schema/schema.test.ts' }, { testFilePath: '/routes/collections/crud.test.ts' }, { testFilePath: '/routes/fields/change-fields.test.ts' }, ],