diff --git a/.gitignore b/.gitignore index 832008f29e..2c27a2ccd4 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,8 @@ dist .e2e-containers.json coverage TODO -schema.yaml -schema.json +/schema.yaml +/schema.json .*.swp debug debug.ts diff --git a/api/package.json b/api/package.json index d37c4e27e2..2ae7cf6aec 100644 --- a/api/package.json +++ b/api/package.json @@ -87,6 +87,7 @@ "@directus/storage-driver-gcs": "workspace:*", "@directus/storage-driver-local": "workspace:*", "@directus/storage-driver-s3": "workspace:*", + "@directus/utils": "workspace:*", "@godaddy/terminus": "4.11.2", "@rollup/plugin-alias": "4.0.2", "@rollup/plugin-virtual": "3.0.1", @@ -129,7 +130,7 @@ "jsonwebtoken": "9.0.0", "keyv": "4.5.2", "knex": "2.4.1", - "knex-schema-inspector": "3.0.0", + "knex-schema-inspector": "3.0.1", "ldapjs": "2.3.3", "liquidjs": "10.3.3", "lodash": "4.17.21", diff --git a/api/src/app.ts b/api/src/app.ts index 4f33a954cb..a090758368 100644 --- a/api/src/app.ts +++ b/api/src/app.ts @@ -26,6 +26,7 @@ import presetsRouter from './controllers/presets'; import relationsRouter from './controllers/relations'; import revisionsRouter from './controllers/revisions'; import rolesRouter from './controllers/roles'; +import schemaRouter from './controllers/schema'; import serverRouter from './controllers/server'; import settingsRouter from './controllers/settings'; import usersRouter from './controllers/users'; @@ -253,6 +254,7 @@ export default async function createApp(): Promise { app.use('/relations', relationsRouter); app.use('/revisions', revisionsRouter); app.use('/roles', rolesRouter); + app.use('/schema', schemaRouter); app.use('/server', serverRouter); app.use('/settings', settingsRouter); app.use('/shares', sharesRouter); diff --git a/api/src/cli/commands/schema/apply.ts b/api/src/cli/commands/schema/apply.ts index e5e281dde5..9976b12463 100644 --- a/api/src/cli/commands/schema/apply.ts +++ b/api/src/cli/commands/schema/apply.ts @@ -6,8 +6,9 @@ import { load as loadYaml } from 'js-yaml'; import path from 'path'; import getDatabase, { isInstalled, validateDatabaseConnection } from '../../../database'; import logger from '../../../logger'; -import { Snapshot } from '../../../types'; -import { applySnapshot, isNestedMetaUpdate } from '../../../utils/apply-snapshot'; +import { DiffKind, Snapshot } from '../../../types'; +import { isNestedMetaUpdate } from '../../../utils/apply-diff'; +import { applySnapshot } from '../../../utils/apply-snapshot'; import { getSnapshot } from '../../../utils/get-snapshot'; import { getSnapshotDiff } from '../../../utils/get-snapshot-diff'; @@ -57,20 +58,20 @@ export async function apply(snapshotPath: string, options?: { yes: boolean; dryR message += chalk.black.underline.bold('Collections:'); for (const { collection, diff } of snapshotDiff.collections) { - if (diff[0]?.kind === 'E') { + if (diff[0]?.kind === DiffKind.EDIT) { message += `\n - ${chalk.blue('Update')} ${collection}`; for (const change of diff) { - if (change.kind === 'E') { + if (change.kind === DiffKind.EDIT) { const path = change.path!.slice(1).join('.'); message += `\n - Set ${path} to ${change.rhs}`; } } - } else if (diff[0]?.kind === 'D') { + } else if (diff[0]?.kind === DiffKind.DELETE) { message += `\n - ${chalk.red('Delete')} ${collection}`; - } else if (diff[0]?.kind === 'N') { + } else if (diff[0]?.kind === DiffKind.NEW) { message += `\n - ${chalk.green('Create')} ${collection}`; - } else if (diff[0]?.kind === 'A') { + } else if (diff[0]?.kind === DiffKind.ARRAY) { message += `\n - ${chalk.blue('Update')} ${collection}`; } } @@ -80,24 +81,24 @@ export async function apply(snapshotPath: string, options?: { yes: boolean; dryR message += '\n\n' + chalk.black.underline.bold('Fields:'); for (const { collection, field, diff } of snapshotDiff.fields) { - if (diff[0]?.kind === 'E' || isNestedMetaUpdate(diff[0])) { + if (diff[0]?.kind === DiffKind.EDIT || isNestedMetaUpdate(diff[0])) { message += `\n - ${chalk.blue('Update')} ${collection}.${field}`; for (const change of diff) { const path = change.path!.slice(1).join('.'); - if (change.kind === 'E') { + if (change.kind === DiffKind.EDIT) { message += `\n - Set ${path} to ${change.rhs}`; - } else if (change.kind === 'D') { + } else if (change.kind === DiffKind.DELETE) { message += `\n - Remove ${path}`; - } else if (change.kind === 'N') { + } else if (change.kind === DiffKind.NEW) { message += `\n - Add ${path} and set it to ${change.rhs}`; } } - } else if (diff[0]?.kind === 'D') { + } else if (diff[0]?.kind === DiffKind.DELETE) { message += `\n - ${chalk.red('Delete')} ${collection}.${field}`; - } else if (diff[0]?.kind === 'N') { + } else if (diff[0]?.kind === DiffKind.NEW) { message += `\n - ${chalk.green('Create')} ${collection}.${field}`; - } else if (diff[0]?.kind === 'A') { + } else if (diff[0]?.kind === DiffKind.ARRAY) { message += `\n - ${chalk.blue('Update')} ${collection}.${field}`; } } @@ -107,20 +108,20 @@ export async function apply(snapshotPath: string, options?: { yes: boolean; dryR message += '\n\n' + chalk.black.underline.bold('Relations:'); for (const { collection, field, related_collection, diff } of snapshotDiff.relations) { - if (diff[0]?.kind === 'E') { + if (diff[0]?.kind === DiffKind.EDIT) { message += `\n - ${chalk.blue('Update')} ${collection}.${field}`; for (const change of diff) { - if (change.kind === 'E') { + if (change.kind === DiffKind.EDIT) { const path = change.path!.slice(1).join('.'); message += `\n - Set ${path} to ${change.rhs}`; } } - } else if (diff[0]?.kind === 'D') { + } else if (diff[0]?.kind === DiffKind.DELETE) { message += `\n - ${chalk.red('Delete')} ${collection}.${field}`; - } else if (diff[0]?.kind === 'N') { + } else if (diff[0]?.kind === DiffKind.NEW) { message += `\n - ${chalk.green('Create')} ${collection}.${field}`; - } else if (diff[0]?.kind === 'A') { + } else if (diff[0]?.kind === DiffKind.ARRAY) { message += `\n - ${chalk.blue('Update')} ${collection}.${field}`; } else { continue; diff --git a/api/src/cli/utils/create-db-connection.ts b/api/src/cli/utils/create-db-connection.ts index e03e414b07..b97d51e146 100644 --- a/api/src/cli/utils/create-db-connection.ts +++ b/api/src/cli/utils/create-db-connection.ts @@ -1,6 +1,7 @@ import { knex, Knex } from 'knex'; import path from 'path'; import { promisify } from 'util'; +import { Driver } from '../../types'; export type Credentials = { filename?: string; @@ -12,10 +13,7 @@ export type Credentials = { ssl?: boolean; options__encrypt?: boolean; }; -export default function createDBConnection( - client: 'sqlite3' | 'mysql' | 'pg' | 'oracledb' | 'mssql' | 'cockroachdb', - credentials: Credentials -): Knex { +export default function createDBConnection(client: Driver, credentials: Credentials): Knex { let connection: Knex.Config['connection'] = {}; if (client === 'sqlite3') { diff --git a/api/src/cli/utils/drivers.ts b/api/src/cli/utils/drivers.ts index c5d03258cb..271c57102c 100644 --- a/api/src/cli/utils/drivers.ts +++ b/api/src/cli/utils/drivers.ts @@ -1,4 +1,6 @@ -export const drivers = { +import { Driver } from '../../types'; + +export const drivers: Record = { pg: 'PostgreSQL / Redshift', cockroachdb: 'CockroachDB (Beta)', mysql: 'MySQL / MariaDB / Aurora', @@ -7,9 +9,9 @@ export const drivers = { oracledb: 'Oracle Database', }; -export function getDriverForClient(client: string): keyof typeof drivers | null { +export function getDriverForClient(client: string): Driver | null { for (const [key, value] of Object.entries(drivers)) { - if (value === client) return key as keyof typeof drivers; + if (value === client) return key as Driver; } return null; diff --git a/api/src/controllers/files.test.ts b/api/src/controllers/files.test.ts index a9540a81df..fb48b951a9 100644 --- a/api/src/controllers/files.test.ts +++ b/api/src/controllers/files.test.ts @@ -7,6 +7,14 @@ import { Request, Response } from 'express'; vi.mock('../../src/database'); +vi.mock('../services', () => { + const FilesService = vi.fn(); + FilesService.prototype.uploadOne = vi.fn(); + const MetaService = vi.fn(); + MetaService.prototype.getMetaForQuery = vi.fn().mockResolvedValue({}); + return { FilesService }; +}); + describe('multipartHandler', () => { it(`Errors out if request doesn't contain any files to upload`, () => { const fakeForm = new FormData(); diff --git a/api/src/controllers/files.ts b/api/src/controllers/files.ts index 21bd39473e..ebd38ff897 100644 --- a/api/src/controllers/files.ts +++ b/api/src/controllers/files.ts @@ -110,7 +110,7 @@ export const multipartHandler: RequestHandler = (req, res, next) => { function tryDone() { if (savedFiles.length === fileCount) { if (fileCount === 0) { - return next(new InvalidPayloadException(`No files where included in the body`)); + return next(new InvalidPayloadException(`No files were included in the body`)); } res.locals.savedFiles = savedFiles; diff --git a/api/src/controllers/schema.ts b/api/src/controllers/schema.ts new file mode 100644 index 0000000000..e651150f1e --- /dev/null +++ b/api/src/controllers/schema.ts @@ -0,0 +1,120 @@ +import { parseJSON } from '@directus/shared/utils'; +import Busboy from 'busboy'; +import express, { RequestHandler } from 'express'; +import { load as loadYaml } from 'js-yaml'; +import { InvalidPayloadException, UnsupportedMediaTypeException } from '../exceptions'; +import logger from '../logger'; +import { respond } from '../middleware/respond'; +import { SchemaService } from '../services/schema'; +import { Snapshot } from '../types'; +import asyncHandler from '../utils/async-handler'; +import { getVersionedHash } from '../utils/get-versioned-hash'; + +const router = express.Router(); + +router.get( + '/snapshot', + asyncHandler(async (req, res, next) => { + const service = new SchemaService({ accountability: req.accountability }); + const currentSnapshot = await service.snapshot(); + res.locals.payload = { data: currentSnapshot }; + return next(); + }), + respond +); + +router.post( + '/apply', + asyncHandler(async (req, _res, next) => { + const service = new SchemaService({ accountability: req.accountability }); + await service.apply(req.body); + return next(); + }), + respond +); + +const schemaMultipartHandler: RequestHandler = (req, res, next) => { + if (req.is('application/json')) { + if (Object.keys(req.body).length === 0) throw new InvalidPayloadException(`No data was included in the body`); + res.locals.uploadedSnapshot = req.body; + return next(); + } + + if (!req.is('multipart/form-data')) throw new UnsupportedMediaTypeException(`Unsupported Content-Type header`); + + const headers = req.headers['content-type'] + ? req.headers + : { + ...req.headers, + 'content-type': 'application/octet-stream', + }; + + const busboy = Busboy({ headers }); + + let isFileIncluded = false; + let uploadedSnapshot: Snapshot | null = null; + + busboy.on('file', async (_, fileStream, { mimeType }) => { + if (isFileIncluded) return next(new InvalidPayloadException(`More than one file was included in the body`)); + + isFileIncluded = true; + + const { readableStreamToString } = await import('@directus/utils/node'); + + try { + const uploadedString = await readableStreamToString(fileStream); + + if (mimeType === 'application/json') { + try { + uploadedSnapshot = parseJSON(uploadedString); + } catch (err: any) { + logger.warn(err); + throw new InvalidPayloadException('Invalid JSON schema snapshot'); + } + } else { + try { + uploadedSnapshot = (await loadYaml(uploadedString)) as Snapshot; + } catch (err: any) { + logger.warn(err); + throw new InvalidPayloadException('Invalid YAML schema snapshot'); + } + } + + if (!uploadedSnapshot) throw new InvalidPayloadException(`No file was included in the body`); + + res.locals.uploadedSnapshot = uploadedSnapshot; + + return next(); + } catch (error: any) { + busboy.emit('error', error); + } + }); + + busboy.on('error', (error: Error) => next(error)); + + busboy.on('close', () => { + if (!isFileIncluded) return next(new InvalidPayloadException(`No file was included in the body`)); + }); + + req.pipe(busboy); +}; + +router.post( + '/diff', + asyncHandler(schemaMultipartHandler), + asyncHandler(async (req, res, next) => { + const service = new SchemaService({ accountability: req.accountability }); + const snapshot: Snapshot = res.locals.uploadedSnapshot; + + const currentSnapshot = await service.snapshot(); + const snapshotDiff = await service.diff(snapshot, { currentSnapshot, force: 'force' in req.query }); + if (!snapshotDiff) return next(); + + const currentSnapshotHash = getVersionedHash(currentSnapshot); + res.locals.payload = { data: { hash: currentSnapshotHash, diff: snapshotDiff } }; + return next(); + }), + respond +); + +export default router; diff --git a/api/src/database/helpers/schema/dialects/oracle.ts b/api/src/database/helpers/schema/dialects/oracle.ts index 7d66e1ce39..70741385e4 100644 --- a/api/src/database/helpers/schema/dialects/oracle.ts +++ b/api/src/database/helpers/schema/dialects/oracle.ts @@ -1,4 +1,5 @@ import { KNEX_TYPES } from '@directus/shared/constants'; +import { Field, Relation, Type } from '@directus/shared/types'; import { Options, SchemaHelper } from '../types'; export class SchemaHelperOracle extends SchemaHelper { @@ -14,4 +15,28 @@ export class SchemaHelperOracle extends SchemaHelper { castA2oPrimaryKey(): string { return 'CAST(?? AS VARCHAR2(255))'; } + + preRelationChange(relation: Partial): void { + if (relation.collection === relation.related_collection) { + // Constraints are not allowed on self referencing relationships + // Setting NO ACTION throws - ORA-00905: missing keyword + if (relation.schema?.on_delete) { + relation.schema.on_delete = null; + } + } + } + + processFieldType(field: Field): Type { + if (field.type === 'integer') { + if (field.schema?.numeric_precision === 20) { + return 'bigInteger'; + } else if (field.schema?.numeric_precision === 1) { + return 'boolean'; + } else if (field.schema?.numeric_precision || field.schema?.numeric_scale) { + return 'decimal'; + } + } + + return field.type; + } } diff --git a/api/src/database/helpers/schema/types.ts b/api/src/database/helpers/schema/types.ts index 4113fc81f9..bd1e7ef5e1 100644 --- a/api/src/database/helpers/schema/types.ts +++ b/api/src/database/helpers/schema/types.ts @@ -1,14 +1,14 @@ +import { KNEX_TYPES } from '@directus/shared/constants'; +import { Field, Relation, Type } from '@directus/shared/types'; +import { Knex } from 'knex'; +import { DatabaseClient } from '../../../types'; import { getDatabaseClient } from '../../index'; import { DatabaseHelper } from '../types'; -import { KNEX_TYPES } from '@directus/shared/constants'; -import { Knex } from 'knex'; - -type Clients = 'mysql' | 'postgres' | 'cockroachdb' | 'sqlite' | 'oracle' | 'mssql' | 'redshift'; export type Options = { nullable?: boolean; default?: any; length?: number }; export abstract class SchemaHelper extends DatabaseHelper { - isOneOfClients(clients: Clients[]): boolean { + isOneOfClients(clients: DatabaseClient[]): boolean { return clients.includes(getDatabaseClient(this.knex)); } @@ -92,6 +92,14 @@ export abstract class SchemaHelper extends DatabaseHelper { return; } + preRelationChange(_relation: Partial): void { + return; + } + + processFieldType(field: Field): Type { + return field.type; + } + constraintName(existingName: string): string { // most vendors allow for dropping/creating constraints with the same name // reference issue #14873 diff --git a/api/src/database/index.ts b/api/src/database/index.ts index 8e96b66bf3..47c900c345 100644 --- a/api/src/database/index.ts +++ b/api/src/database/index.ts @@ -10,6 +10,7 @@ import path from 'path'; import { merge } from 'lodash'; import { promisify } from 'util'; import { getHelpers } from './helpers'; +import { DatabaseClient } from '../types'; let database: Knex | null = null; let inspector: ReturnType | null = null; @@ -201,9 +202,7 @@ export async function validateDatabaseConnection(database?: Knex): Promise } } -export function getDatabaseClient( - database?: Knex -): 'mysql' | 'postgres' | 'cockroachdb' | 'sqlite' | 'oracle' | 'mssql' | 'redshift' { +export function getDatabaseClient(database?: Knex): DatabaseClient { database = database ?? getDatabase(); switch (database.client.constructor.name) { diff --git a/api/src/middleware/respond.ts b/api/src/middleware/respond.ts index 90f46abad2..c5c06ed592 100644 --- a/api/src/middleware/respond.ts +++ b/api/src/middleware/respond.ts @@ -77,6 +77,12 @@ export const respond: RequestHandler = asyncHandler(async (req, res) => { res.set('Content-Type', 'text/csv'); return res.status(200).send(exportService.transform(res.locals.payload?.data, 'csv')); } + + if (req.sanitizedQuery.export === 'yaml') { + res.attachment(`${filename}.yaml`); + res.set('Content-Type', 'text/yaml'); + return res.status(200).send(exportService.transform(res.locals.payload?.data, 'yaml')); + } } if (Buffer.isBuffer(res.locals.payload)) { diff --git a/api/src/services/fields.ts b/api/src/services/fields.ts index dbd5552d8e..1a6b367f90 100644 --- a/api/src/services/fields.ts +++ b/api/src/services/fields.ts @@ -178,6 +178,8 @@ export class FieldsService { } else if (field.meta?.special?.includes('cast-datetime')) { field.type = 'dateTime'; } + + field.type = this.helpers.schema.processFieldType(field); } return result; diff --git a/api/src/services/import-export.test.ts b/api/src/services/import-export.test.ts index 5e5d9952b0..dd0a4761d1 100644 --- a/api/src/services/import-export.test.ts +++ b/api/src/services/import-export.test.ts @@ -1,10 +1,12 @@ -import knex, { Knex } from 'knex'; -import { MockClient, Tracker, getTracker } from 'knex-mock-client'; -import { ImportService } from '.'; -import { describe, beforeAll, afterEach, it, expect, vi, beforeEach, MockedFunction } from 'vitest'; -import { Readable } from 'stream'; -import emitter from '../emitter'; import { parse } from 'json2csv'; +import knex, { Knex } from 'knex'; +import { getTracker, MockClient, Tracker } from 'knex-mock-client'; +import { EOL } from 'node:os'; +import { Readable } from 'stream'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, MockedFunction, vi } from 'vitest'; +import { ExportService, ImportService } from '.'; +import { ServiceUnavailableException } from '..'; +import emitter from '../emitter'; vi.mock('../../src/database/index', () => ({ default: vi.fn(), @@ -116,4 +118,58 @@ describe('Integration Tests', () => { }); }); }); + + describe('Services / ExportService', () => { + describe('transform', () => { + it('should return json string with header and footer', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(service.transform(input, 'json')).toBe(`[\n\t{\n\t\t"key": "value"\n\t}\n]`); + }); + + it('should return xml string with header and footer', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(service.transform(input, 'xml')).toBe( + `\n\n \n value\n \n` + ); + }); + + it('should return csv string with header', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(service.transform(input, 'csv')).toBe(`"key"${EOL}"value"`); + }); + + it('should return csv string without header', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(service.transform(input, 'csv', { includeHeader: false })).toBe('\n"value"'); + }); + + it('should return yaml string', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(service.transform(input, 'yaml')).toBe('- key: value\n'); + }); + + it('should throw ServiceUnavailableException error when using a non-existent export type', () => { + const input = [{ key: 'value' }]; + + const service = new ExportService({ knex: db, schema: { collections: {}, relations: [] } }); + + expect(() => service.transform(input, 'invalid-format' as any)).toThrowError(ServiceUnavailableException); + }); + }); + }); }); diff --git a/api/src/services/import-export.ts b/api/src/services/import-export.ts index 9be002480a..ec3b02f05d 100644 --- a/api/src/services/import-export.ts +++ b/api/src/services/import-export.ts @@ -4,6 +4,7 @@ import { queue } from 'async'; import csv from 'csv-parser'; import destroyStream from 'destroy'; import { appendFile, createReadStream } from 'fs-extra'; +import { dump as toYAML } from 'js-yaml'; import { parse as toXML } from 'js2xmlparser'; import { Parser as CSVParser, transforms as CSVTransforms } from 'json2csv'; import { Knex } from 'knex'; @@ -28,6 +29,8 @@ import { NotificationsService } from './notifications'; import emitter from '../emitter'; import type { Readable } from 'node:stream'; +type ExportFormat = 'csv' | 'json' | 'xml' | 'yaml'; + export class ImportService { knex: Knex; accountability: Accountability | null; @@ -190,16 +193,17 @@ export class ExportService { async exportToFile( collection: string, query: Partial, - format: 'xml' | 'csv' | 'json', + format: ExportFormat, options?: { file?: Partial; } ) { try { const mimeTypes = { - xml: 'text/xml', csv: 'text/csv', json: 'application/json', + xml: 'text/xml', + yaml: 'text/yaml', }; const database = getDatabase(); @@ -316,7 +320,7 @@ export class ExportService { */ transform( input: Record[], - format: 'xml' | 'csv' | 'json', + format: ExportFormat, options?: { includeHeader?: boolean; includeFooter?: boolean; @@ -366,6 +370,10 @@ export class ExportService { return string; } + if (format === 'yaml') { + return toYAML(input); + } + throw new ServiceUnavailableException(`Illegal export type used: "${format}"`, { service: 'export' }); } } diff --git a/api/src/services/index.ts b/api/src/services/index.ts index 29c82c9980..1bb8115dfe 100644 --- a/api/src/services/index.ts +++ b/api/src/services/index.ts @@ -23,6 +23,7 @@ export * from './presets'; export * from './relations'; export * from './revisions'; export * from './roles'; +export * from './schema'; export * from './server'; export * from './settings'; export * from './specifications'; diff --git a/api/src/services/relations.ts b/api/src/services/relations.ts index dca529be2e..e699656182 100644 --- a/api/src/services/relations.ts +++ b/api/src/services/relations.ts @@ -171,6 +171,8 @@ export class RelationsService { } const runPostColumnChange = await this.helpers.schema.preColumnChange(); + this.helpers.schema.preRelationChange(relation); + const nestedActionEvents: ActionEventParams[] = []; try { @@ -264,6 +266,8 @@ export class RelationsService { } const runPostColumnChange = await this.helpers.schema.preColumnChange(); + this.helpers.schema.preRelationChange(relation); + const nestedActionEvents: ActionEventParams[] = []; try { diff --git a/api/src/services/schema.test.ts b/api/src/services/schema.test.ts new file mode 100644 index 0000000000..754aeb9b1c --- /dev/null +++ b/api/src/services/schema.test.ts @@ -0,0 +1,188 @@ +import { Diff } from 'deep-diff'; +import knex, { Knex } from 'knex'; +import { getTracker, MockClient, Tracker } from 'knex-mock-client'; +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import { SchemaService } from '.'; +import { ForbiddenException } from '..'; +import { Collection } from '../types/collection'; +import { Snapshot, SnapshotDiffWithHash } from '../types/snapshot'; +import { applyDiff } from '../utils/apply-diff'; +import { getSnapshot } from '../utils/get-snapshot'; + +vi.mock('../../package.json', () => ({ version: '0.0.0' })); + +vi.mock('../../src/database/index', () => { + return { __esModule: true, default: vi.fn(), getDatabaseClient: vi.fn().mockReturnValue('postgres') }; +}); + +vi.mock('../utils/get-snapshot', () => ({ + getSnapshot: vi.fn(), +})); + +vi.mock('../utils/apply-diff', () => ({ + applyDiff: vi.fn(), +})); + +class Client_PG extends MockClient {} + +let db: Knex; +let tracker: Tracker; + +const testSnapshot = { + directus: '0.0.0', + version: 1, + vendor: 'postgres', + collections: [], + fields: [], + relations: [], +} satisfies Snapshot; + +const testCollectionDiff = { + collection: 'test', + diff: [ + { + kind: 'N', + rhs: { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { name: 'test' }, + }, + }, + ] satisfies Diff[], +}; + +beforeAll(() => { + db = knex({ client: Client_PG }); + tracker = getTracker(); +}); + +afterEach(() => { + tracker.reset(); + vi.clearAllMocks(); +}); + +describe('Services / Schema', () => { + describe('snapshot', () => { + it('should throw ForbiddenException for non-admin user', async () => { + vi.mocked(getSnapshot).mockResolvedValueOnce(testSnapshot); + + const service = new SchemaService({ knex: db, accountability: { role: 'test', admin: false } }); + + expect(service.snapshot()).rejects.toThrowError(ForbiddenException); + }); + + it('should return snapshot for admin user', async () => { + vi.mocked(getSnapshot).mockResolvedValueOnce(testSnapshot); + + const service = new SchemaService({ knex: db, accountability: { role: 'admin', admin: true } }); + + expect(service.snapshot()).resolves.toEqual(testSnapshot); + }); + }); + + describe('apply', () => { + const snapshotDiffWithHash = { + hash: '813b3cdf7013310fafde7813b7d5e6bd4eb1e73f', + diff: { + collections: [testCollectionDiff], + fields: [], + relations: [], + }, + } satisfies SnapshotDiffWithHash; + + it('should throw ForbiddenException for non-admin user', async () => { + vi.mocked(getSnapshot).mockResolvedValueOnce(testSnapshot); + + const service = new SchemaService({ knex: db, accountability: { role: 'test', admin: false } }); + + expect(service.apply(snapshotDiffWithHash)).rejects.toThrowError(ForbiddenException); + expect(vi.mocked(applyDiff)).not.toHaveBeenCalledOnce(); + }); + + it('should apply for admin user', async () => { + vi.mocked(getSnapshot).mockResolvedValueOnce(testSnapshot); + + const service = new SchemaService({ knex: db, accountability: { role: 'admin', admin: true } }); + + await service.apply(snapshotDiffWithHash); + + expect(vi.mocked(applyDiff)).toHaveBeenCalledOnce(); + }); + }); + + describe('diff', () => { + const snapshotToApply = { + directus: '0.0.0', + version: 1, + vendor: 'postgres', + collections: [ + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { + name: 'test', + }, + }, + ], + fields: [], + relations: [], + } satisfies Snapshot; + + it('should throw ForbiddenException for non-admin user', async () => { + const service = new SchemaService({ knex: db, accountability: { role: 'test', admin: false } }); + + expect(service.diff(snapshotToApply, { currentSnapshot: testSnapshot, force: true })).rejects.toThrowError( + ForbiddenException + ); + }); + + it('should return diff for admin user', async () => { + const service = new SchemaService({ knex: db, accountability: { role: 'admin', admin: true } }); + + expect(service.diff(snapshotToApply, { currentSnapshot: testSnapshot, force: true })).resolves.toEqual({ + collections: [testCollectionDiff], + fields: [], + relations: [], + }); + }); + + it('should return null for empty diff', async () => { + const service = new SchemaService({ knex: db, accountability: { role: 'admin', admin: true } }); + + expect(service.diff(testSnapshot, { currentSnapshot: testSnapshot, force: true })).resolves.toBeNull(); + }); + }); + + describe('getHashedSnapshot', () => { + it('should return snapshot for admin user', async () => { + const service = new SchemaService({ knex: db, accountability: { role: 'admin', admin: true } }); + + expect(service.getHashedSnapshot(testSnapshot)).toEqual( + expect.objectContaining({ + ...testSnapshot, + hash: expect.any(String), + }) + ); + }); + }); +}); diff --git a/api/src/services/schema.ts b/api/src/services/schema.ts new file mode 100644 index 0000000000..0935e89ab7 --- /dev/null +++ b/api/src/services/schema.ts @@ -0,0 +1,67 @@ +import { Accountability } from '@directus/shared/types'; +import { Knex } from 'knex'; +import getDatabase from '../database'; +import { ForbiddenException } from '../exceptions'; +import { AbstractServiceOptions, Snapshot, SnapshotDiff, SnapshotDiffWithHash, SnapshotWithHash } from '../types'; +import { applyDiff } from '../utils/apply-diff'; +import { getSnapshot } from '../utils/get-snapshot'; +import { getSnapshotDiff } from '../utils/get-snapshot-diff'; +import { getVersionedHash } from '../utils/get-versioned-hash'; +import { validateApplyDiff } from '../utils/validate-diff'; +import { validateSnapshot } from '../utils/validate-snapshot'; + +export class SchemaService { + knex: Knex; + accountability: Accountability | null; + + constructor(options: Omit) { + this.knex = options.knex ?? getDatabase(); + this.accountability = options.accountability ?? null; + } + + async snapshot(): Promise { + if (this.accountability?.admin !== true) throw new ForbiddenException(); + + const currentSnapshot = await getSnapshot({ database: this.knex }); + + return currentSnapshot; + } + + async apply(payload: SnapshotDiffWithHash): Promise { + if (this.accountability?.admin !== true) throw new ForbiddenException(); + + const currentSnapshot = await this.snapshot(); + const snapshotWithHash = this.getHashedSnapshot(currentSnapshot); + + if (!validateApplyDiff(payload, snapshotWithHash)) return; + + await applyDiff(currentSnapshot, payload.diff, { database: this.knex }); + } + + async diff( + snapshot: Snapshot, + options?: { currentSnapshot?: Snapshot; force?: boolean } + ): Promise { + if (this.accountability?.admin !== true) throw new ForbiddenException(); + + validateSnapshot(snapshot, options?.force); + + const currentSnapshot = options?.currentSnapshot ?? (await getSnapshot({ database: this.knex })); + const diff = getSnapshotDiff(currentSnapshot, snapshot); + + if (diff.collections.length === 0 && diff.fields.length === 0 && diff.relations.length === 0) { + return null; + } + + return diff; + } + + getHashedSnapshot(snapshot: Snapshot): SnapshotWithHash { + const snapshotHash = getVersionedHash(snapshot); + + return { + ...snapshot, + hash: snapshotHash, + }; + } +} diff --git a/api/src/types/database.ts b/api/src/types/database.ts new file mode 100644 index 0000000000..a3410db489 --- /dev/null +++ b/api/src/types/database.ts @@ -0,0 +1,4 @@ +export type Driver = 'mysql' | 'pg' | 'cockroachdb' | 'sqlite3' | 'oracledb' | 'mssql'; + +export const DatabaseClients = ['mysql', 'postgres', 'cockroachdb', 'sqlite', 'oracle', 'mssql', 'redshift'] as const; +export type DatabaseClient = typeof DatabaseClients[number]; diff --git a/api/src/types/index.ts b/api/src/types/index.ts index 0660a3610c..7dc87dc525 100644 --- a/api/src/types/index.ts +++ b/api/src/types/index.ts @@ -2,6 +2,7 @@ export * from './assets'; export * from './ast'; export * from './auth'; export * from './collection'; +export * from './database'; export * from './events'; export * from './files'; export * from './graphql'; diff --git a/api/src/types/snapshot.ts b/api/src/types/snapshot.ts index 47e86b2914..b92df5dba0 100644 --- a/api/src/types/snapshot.ts +++ b/api/src/types/snapshot.ts @@ -1,10 +1,12 @@ import { Collection } from './collection'; import { Relation, RelationMeta, Field, FieldMeta } from '@directus/shared/types'; import { Diff } from 'deep-diff'; +import { DatabaseClient } from './database'; export type Snapshot = { version: number; directus: string; + vendor?: DatabaseClient; collections: Collection[]; fields: SnapshotField[]; relations: SnapshotRelation[]; @@ -13,6 +15,8 @@ export type Snapshot = { export type SnapshotField = Field & { meta: Omit }; export type SnapshotRelation = Relation & { meta: Omit }; +export type SnapshotWithHash = Snapshot & { hash: string }; + export type SnapshotDiff = { collections: { collection: string; @@ -30,3 +34,19 @@ export type SnapshotDiff = { diff: Diff[]; }[]; }; + +export type SnapshotDiffWithHash = { hash: string; diff: SnapshotDiff }; + +/** + * Indicates the kind of change based on comparisons by deep-diff package + */ +export const DiffKind = { + /** indicates a newly added property/element */ + NEW: 'N', + /** indicates a property/element was deleted */ + DELETE: 'D', + /** indicates a property/element was edited */ + EDIT: 'E', + /** indicates a change occurred within an array */ + ARRAY: 'A', +} as const; diff --git a/api/src/utils/apply-diff.test.ts b/api/src/utils/apply-diff.test.ts new file mode 100644 index 0000000000..54f53c35b2 --- /dev/null +++ b/api/src/utils/apply-diff.test.ts @@ -0,0 +1,28 @@ +import { Diff } from 'deep-diff'; +import { describe, expect, it } from 'vitest'; +import { SnapshotField } from '../types/snapshot'; + +import { isNestedMetaUpdate } from './apply-diff'; + +describe('isNestedMetaUpdate', () => { + it.each([ + { kind: 'E', path: ['meta', 'options', 'option_a'], rhs: {} }, + { kind: 'A', path: ['meta', 'options', 'option_a'], rhs: [] }, + ] as Diff[])('Returns false when diff is kind $kind', (diff) => { + expect(isNestedMetaUpdate(diff)).toBe(false); + }); + + it.each([ + { kind: 'N', path: ['schema', 'default_value'], rhs: {} }, + { kind: 'D', path: ['schema'], lhs: {} }, + ] as Diff[])('Returns false when diff path is not nested in meta', (diff) => { + expect(isNestedMetaUpdate(diff)).toBe(false); + }); + + it.each([ + { kind: 'N', path: ['meta', 'options', 'option_a'], rhs: { test: 'value' } }, + { kind: 'D', path: ['meta', 'options', 'option_b'], lhs: {} }, + ] as Diff[])('Returns true when diff path is nested in meta', (diff) => { + expect(isNestedMetaUpdate(diff)).toBe(true); + }); +}); diff --git a/api/src/utils/apply-diff.ts b/api/src/utils/apply-diff.ts new file mode 100644 index 0000000000..5afe1354ab --- /dev/null +++ b/api/src/utils/apply-diff.ts @@ -0,0 +1,317 @@ +import { Field, Relation, SchemaOverview } from '@directus/shared/types'; +import { Knex } from 'knex'; +import { CollectionsService, FieldsService, RelationsService } from '../services'; +import { + ActionEventParams, + Collection, + DiffKind, + MutationOptions, + Snapshot, + SnapshotDiff, + SnapshotField, +} from '../types'; +import { getSchema } from './get-schema'; +import getDatabase from '../database'; +import { applyChange, Diff, DiffDeleted, DiffNew } from 'deep-diff'; +import { cloneDeep, merge, set } from 'lodash'; +import logger from '../logger'; +import emitter from '../emitter'; +import { clearSystemCache } from '../cache'; + +type CollectionDelta = { + collection: string; + diff: Diff[]; +}; + +export async function applyDiff( + currentSnapshot: Snapshot, + snapshotDiff: SnapshotDiff, + options?: { database?: Knex; schema?: SchemaOverview } +): Promise { + const database = options?.database ?? getDatabase(); + const schema = options?.schema ?? (await getSchema({ database, bypassCache: true })); + + const nestedActionEvents: ActionEventParams[] = []; + const mutationOptions: MutationOptions = { + autoPurgeSystemCache: false, + bypassEmitAction: (params) => nestedActionEvents.push(params), + }; + + await database.transaction(async (trx) => { + const collectionsService = new CollectionsService({ knex: trx, schema }); + + const getNestedCollectionsToCreate = (currentLevelCollection: string) => + snapshotDiff.collections.filter( + ({ diff }) => (diff[0] as DiffNew).rhs?.meta?.group === currentLevelCollection + ) as CollectionDelta[]; + + const getNestedCollectionsToDelete = (currentLevelCollection: string) => + snapshotDiff.collections.filter( + ({ diff }) => (diff[0] as DiffDeleted).lhs?.meta?.group === currentLevelCollection + ) as CollectionDelta[]; + + const createCollections = async (collections: CollectionDelta[]) => { + for (const { collection, diff } of collections) { + if (diff?.[0].kind === DiffKind.NEW && diff[0].rhs) { + // We'll nest the to-be-created fields in the same collection creation, to prevent + // creating a collection without a primary key + const fields = snapshotDiff.fields + .filter((fieldDiff) => fieldDiff.collection === collection) + .map((fieldDiff) => (fieldDiff.diff[0] as DiffNew).rhs) + .map((fieldDiff) => { + // Casts field type to UUID when applying non-PostgreSQL schema onto PostgreSQL database. + // This is needed because they snapshots UUID fields as char with length 36. + if ( + String(fieldDiff.schema?.data_type).toLowerCase() === 'char' && + fieldDiff.schema?.max_length === 36 && + (fieldDiff.schema?.is_primary_key || + (fieldDiff.schema?.foreign_key_table && fieldDiff.schema?.foreign_key_column)) + ) { + return merge(fieldDiff, { type: 'uuid', schema: { data_type: 'uuid', max_length: null } }); + } else { + return fieldDiff; + } + }); + + try { + await collectionsService.createOne( + { + ...diff[0].rhs, + fields, + }, + mutationOptions + ); + } catch (err: any) { + logger.error(`Failed to create collection "${collection}"`); + throw err; + } + + // Now that the fields are in for this collection, we can strip them from the field edits + snapshotDiff.fields = snapshotDiff.fields.filter((fieldDiff) => fieldDiff.collection !== collection); + + await createCollections(getNestedCollectionsToCreate(collection)); + } + } + }; + + const deleteCollections = async (collections: CollectionDelta[]) => { + for (const { collection, diff } of collections) { + if (diff?.[0].kind === DiffKind.DELETE) { + const relations = schema.relations.filter( + (r) => r.related_collection === collection || r.collection === collection + ); + + if (relations.length > 0) { + const relationsService = new RelationsService({ knex: trx, schema }); + + for (const relation of relations) { + try { + await relationsService.deleteOne(relation.collection, relation.field, mutationOptions); + } catch (err) { + logger.error( + `Failed to delete collection "${collection}" due to relation "${relation.collection}.${relation.field}"` + ); + throw err; + } + } + + // clean up deleted relations from existing schema + schema.relations = schema.relations.filter( + (r) => r.related_collection !== collection && r.collection !== collection + ); + } + + await deleteCollections(getNestedCollectionsToDelete(collection)); + + try { + await collectionsService.deleteOne(collection, mutationOptions); + } catch (err) { + logger.error(`Failed to delete collection "${collection}"`); + throw err; + } + } + } + }; + + // Finds all collections that need to be created + const filterCollectionsForCreation = ({ diff }: { collection: string; diff: Diff[] }) => { + // Check new collections only + const isNewCollection = diff[0].kind === DiffKind.NEW; + if (!isNewCollection) return false; + + // Create now if no group + const groupName = (diff[0] as DiffNew).rhs.meta?.group; + if (!groupName) return true; + + // Check if parent collection already exists in schema + const parentExists = currentSnapshot.collections.find((c) => c.collection === groupName) !== undefined; + // If this is a new collection and the parent collection doesn't exist in current schema -> + // Check if the parent collection will be created as part of applying this snapshot -> + // If yes -> this collection will be created recursively + // If not -> create now + // (ex.) + // TopLevelCollection - I exist in current schema + // NestedCollection - I exist in snapshotDiff as a new collection + // TheCurrentCollectionInIteration - I exist in snapshotDiff as a new collection but will be created as part of NestedCollection + const parentWillBeCreatedInThisApply = + snapshotDiff.collections.filter( + ({ collection, diff }) => diff[0].kind === DiffKind.NEW && collection === groupName + ).length > 0; + // Has group, but parent is not new, parent is also not being created in this snapshot apply + if (parentExists && !parentWillBeCreatedInThisApply) return true; + + return false; + }; + + // Create top level collections (no group, or highest level in existing group) first, + // then continue with nested collections recursively + await createCollections(snapshotDiff.collections.filter(filterCollectionsForCreation)); + + // delete top level collections (no group) first, then continue with nested collections recursively + await deleteCollections( + snapshotDiff.collections.filter( + ({ diff }) => diff[0].kind === DiffKind.DELETE && (diff[0] as DiffDeleted).lhs.meta?.group === null + ) + ); + + for (const { collection, diff } of snapshotDiff.collections) { + if (diff?.[0].kind === DiffKind.EDIT || diff?.[0].kind === DiffKind.ARRAY) { + const currentCollection = currentSnapshot.collections.find((field) => { + return field.collection === collection; + }); + + if (currentCollection) { + try { + const newValues = diff.reduce((acc, currentDiff) => { + applyChange(acc, undefined, currentDiff); + return acc; + }, cloneDeep(currentCollection)); + + await collectionsService.updateOne(collection, newValues, mutationOptions); + } catch (err) { + logger.error(`Failed to update collection "${collection}"`); + throw err; + } + } + } + } + + const fieldsService = new FieldsService({ + knex: trx, + schema: await getSchema({ database: trx, bypassCache: true }), + }); + + for (const { collection, field, diff } of snapshotDiff.fields) { + if (diff?.[0].kind === DiffKind.NEW && !isNestedMetaUpdate(diff?.[0])) { + try { + await fieldsService.createField(collection, (diff[0] as DiffNew).rhs, undefined, mutationOptions); + } catch (err) { + logger.error(`Failed to create field "${collection}.${field}"`); + throw err; + } + } + + if (diff?.[0].kind === DiffKind.EDIT || diff?.[0].kind === DiffKind.ARRAY || isNestedMetaUpdate(diff?.[0])) { + const currentField = currentSnapshot.fields.find((snapshotField) => { + return snapshotField.collection === collection && snapshotField.field === field; + }); + + if (currentField) { + try { + const newValues = diff.reduce((acc, currentDiff) => { + applyChange(acc, undefined, currentDiff); + return acc; + }, cloneDeep(currentField)); + await fieldsService.updateField(collection, newValues, mutationOptions); + } catch (err) { + logger.error(`Failed to update field "${collection}.${field}"`); + throw err; + } + } + } + + if (diff?.[0].kind === DiffKind.DELETE && !isNestedMetaUpdate(diff?.[0])) { + try { + await fieldsService.deleteField(collection, field, mutationOptions); + } catch (err) { + logger.error(`Failed to delete field "${collection}.${field}"`); + throw err; + } + + // Field deletion also cleans up the relationship. We should ignore any relationship + // changes attached to this now non-existing field + snapshotDiff.relations = snapshotDiff.relations.filter( + (relation) => (relation.collection === collection && relation.field === field) === false + ); + } + } + + const relationsService = new RelationsService({ + knex: trx, + schema: await getSchema({ database: trx, bypassCache: true }), + }); + + for (const { collection, field, diff } of snapshotDiff.relations) { + const structure = {}; + + for (const diffEdit of diff) { + set(structure, diffEdit.path!, undefined); + } + + if (diff?.[0].kind === DiffKind.NEW) { + try { + await relationsService.createOne((diff[0] as DiffNew).rhs, mutationOptions); + } catch (err) { + logger.error(`Failed to create relation "${collection}.${field}"`); + throw err; + } + } + + if (diff?.[0].kind === DiffKind.EDIT || diff?.[0].kind === DiffKind.ARRAY) { + const currentRelation = currentSnapshot.relations.find((relation) => { + return relation.collection === collection && relation.field === field; + }); + + if (currentRelation) { + try { + const newValues = diff.reduce((acc, currentDiff) => { + applyChange(acc, undefined, currentDiff); + return acc; + }, cloneDeep(currentRelation)); + await relationsService.updateOne(collection, field, newValues, mutationOptions); + } catch (err) { + logger.error(`Failed to update relation "${collection}.${field}"`); + throw err; + } + } + } + + if (diff?.[0].kind === DiffKind.DELETE) { + try { + await relationsService.deleteOne(collection, field, mutationOptions); + } catch (err) { + logger.error(`Failed to delete relation "${collection}.${field}"`); + throw err; + } + } + } + }); + + await clearSystemCache(); + + if (nestedActionEvents.length > 0) { + const updatedSchema = await getSchema({ database, bypassCache: true }); + + for (const nestedActionEvent of nestedActionEvents) { + nestedActionEvent.context.schema = updatedSchema; + emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context); + } + } +} + +export function isNestedMetaUpdate(diff: Diff): boolean { + if (!diff) return false; + if (diff.kind !== DiffKind.NEW && diff.kind !== DiffKind.DELETE) return false; + if (!diff.path || diff.path.length < 2 || diff.path[0] !== 'meta') return false; + return true; +} diff --git a/api/src/utils/apply-snapshot.test.ts b/api/src/utils/apply-snapshot.test.ts index dec2deca3e..db95f20eb8 100644 --- a/api/src/utils/apply-snapshot.test.ts +++ b/api/src/utils/apply-snapshot.test.ts @@ -50,7 +50,7 @@ describe('applySnapshot', () => { singleton: false, translations: {}, }, - schema: { comment: null, name: 'test_table_2', schema: 'public' }, + schema: { name: 'test_table_2' }, fields: [ { collection: 'test_table_2', @@ -76,11 +76,9 @@ describe('applySnapshot', () => { width: 'full', }, schema: { - comment: null, data_type: 'uuid', default_value: null, foreign_key_column: null, - foreign_key_schema: null, foreign_key_table: null, generation_expression: null, has_auto_increment: false, @@ -92,7 +90,6 @@ describe('applySnapshot', () => { name: 'id', numeric_precision: null, numeric_scale: null, - schema: 'public', table: 'test_table_2', }, type: 'uuid', @@ -134,7 +131,7 @@ describe('applySnapshot', () => { singleton: false, translations: {}, }, - schema: { comment: null, name: 'test_table_2', schema: 'public' }, + schema: { name: 'test_table_2' }, fields: [ { collection: 'test_table_2', @@ -160,11 +157,9 @@ describe('applySnapshot', () => { width: 'full', }, schema: { - comment: null, data_type: 'uuid', default_value: null, foreign_key_column: null, - foreign_key_schema: null, foreign_key_table: null, generation_expression: null, has_auto_increment: false, @@ -176,7 +171,6 @@ describe('applySnapshot', () => { name: 'id', numeric_precision: null, numeric_scale: null, - schema: 'public', table: 'test_table_2', }, type: 'uuid', @@ -211,11 +205,9 @@ describe('applySnapshot', () => { width: 'full', }, schema: { - comment: null, data_type: 'uuid', default_value: null, foreign_key_column: null, - foreign_key_schema: null, foreign_key_table: null, generation_expression: null, has_auto_increment: false, @@ -227,7 +219,6 @@ describe('applySnapshot', () => { name: 'id', numeric_precision: null, numeric_scale: null, - schema: 'public', table: 'test_table_3', }, type: 'uuid', @@ -244,7 +235,7 @@ describe('applySnapshot', () => { singleton: false, translations: {}, }, - schema: { comment: null, name: 'test_table_3', schema: 'public' }, + schema: { name: 'test_table_3' }, }; // Stop call to db later on in apply-snapshot diff --git a/api/src/utils/apply-snapshot.ts b/api/src/utils/apply-snapshot.ts index b6356bc484..3e1be93dd3 100644 --- a/api/src/utils/apply-snapshot.ts +++ b/api/src/utils/apply-snapshot.ts @@ -1,21 +1,12 @@ -import { Field, Relation, SchemaOverview } from '@directus/shared/types'; -import { Diff, DiffDeleted, DiffNew } from 'deep-diff'; +import { SchemaOverview } from '@directus/shared/types'; import { Knex } from 'knex'; -import { merge, set } from 'lodash'; +import { getCache } from '../cache'; import getDatabase from '../database'; -import logger from '../logger'; -import { CollectionsService, FieldsService, RelationsService } from '../services'; -import { ActionEventParams, Collection, MutationOptions, Snapshot, SnapshotDiff, SnapshotField } from '../types'; +import { Snapshot, SnapshotDiff } from '../types'; +import { applyDiff } from './apply-diff'; import { getSchema } from './get-schema'; import { getSnapshot } from './get-snapshot'; import { getSnapshotDiff } from './get-snapshot-diff'; -import { getCache } from '../cache'; -import emitter from '../emitter'; - -type CollectionDelta = { - collection: string; - diff: Diff[]; -}; export async function applySnapshot( snapshot: Snapshot, @@ -28,279 +19,7 @@ export async function applySnapshot( const current = options?.current ?? (await getSnapshot({ database, schema })); const snapshotDiff = options?.diff ?? getSnapshotDiff(current, snapshot); - const nestedActionEvents: ActionEventParams[] = []; - const mutationOptions: MutationOptions = { - autoPurgeSystemCache: false, - bypassEmitAction: (params) => nestedActionEvents.push(params), - }; - - await database.transaction(async (trx) => { - const collectionsService = new CollectionsService({ knex: trx, schema }); - - const getNestedCollectionsToCreate = (currentLevelCollection: string) => - snapshotDiff.collections.filter( - ({ diff }) => (diff[0] as DiffNew).rhs?.meta?.group === currentLevelCollection - ) as CollectionDelta[]; - - const getNestedCollectionsToDelete = (currentLevelCollection: string) => - snapshotDiff.collections.filter( - ({ diff }) => (diff[0] as DiffDeleted).lhs?.meta?.group === currentLevelCollection - ) as CollectionDelta[]; - - const createCollections = async (collections: CollectionDelta[]) => { - for (const { collection, diff } of collections) { - if (diff?.[0].kind === 'N' && diff[0].rhs) { - // We'll nest the to-be-created fields in the same collection creation, to prevent - // creating a collection without a primary key - const fields = snapshotDiff.fields - .filter((fieldDiff) => fieldDiff.collection === collection) - .map((fieldDiff) => (fieldDiff.diff[0] as DiffNew).rhs) - .map((fieldDiff) => { - // Casts field type to UUID when applying non-PostgreSQL schema onto PostgreSQL database. - // This is needed because they snapshots UUID fields as char with length 36. - if ( - fieldDiff.schema?.data_type === 'char' && - fieldDiff.schema?.max_length === 36 && - (fieldDiff.schema?.is_primary_key || - (fieldDiff.schema?.foreign_key_table && fieldDiff.schema?.foreign_key_column)) - ) { - return merge(fieldDiff, { type: 'uuid', schema: { data_type: 'uuid', max_length: null } }); - } else { - return fieldDiff; - } - }); - - try { - await collectionsService.createOne( - { - ...diff[0].rhs, - fields, - }, - mutationOptions - ); - } catch (err: any) { - logger.error(`Failed to create collection "${collection}"`); - throw err; - } - - // Now that the fields are in for this collection, we can strip them from the field edits - snapshotDiff.fields = snapshotDiff.fields.filter((fieldDiff) => fieldDiff.collection !== collection); - - await createCollections(getNestedCollectionsToCreate(collection)); - } - } - }; - - const deleteCollections = async (collections: CollectionDelta[]) => { - for (const { collection, diff } of collections) { - if (diff?.[0].kind === 'D') { - const relations = schema.relations.filter( - (r) => r.related_collection === collection || r.collection === collection - ); - - if (relations.length > 0) { - const relationsService = new RelationsService({ knex: trx, schema }); - - for (const relation of relations) { - try { - await relationsService.deleteOne(relation.collection, relation.field, mutationOptions); - } catch (err) { - logger.error( - `Failed to delete collection "${collection}" due to relation "${relation.collection}.${relation.field}"` - ); - throw err; - } - } - - // clean up deleted relations from existing schema - schema.relations = schema.relations.filter( - (r) => r.related_collection !== collection && r.collection !== collection - ); - } - - await deleteCollections(getNestedCollectionsToDelete(collection)); - - try { - await collectionsService.deleteOne(collection, mutationOptions); - } catch (err) { - logger.error(`Failed to delete collection "${collection}"`); - throw err; - } - } - } - }; - - // Finds all collections that need to be created - const filterCollectionsForCreation = ({ diff }: { collection: string; diff: Diff[] }) => { - // Check new collections only - const isNewCollection = diff[0].kind === 'N'; - if (!isNewCollection) return false; - - // Create now if no group - const groupName = (diff[0] as DiffNew).rhs.meta?.group; - if (!groupName) return true; - - // Check if parent collection already exists in schema - const parentExists = current.collections.find((c) => c.collection === groupName) !== undefined; - // If this is a new collection and the parent collection doesn't exist in current schema -> - // Check if the parent collection will be created as part of applying this snapshot -> - // If yes -> this collection will be created recursively - // If not -> create now - // (ex.) - // TopLevelCollection - I exist in current schema - // NestedCollection - I exist in snapshotDiff as a new collection - // TheCurrentCollectionInIteration - I exist in snapshotDiff as a new collection but will be created as part of NestedCollection - const parentWillBeCreatedInThisApply = - snapshotDiff.collections.filter(({ collection, diff }) => diff[0].kind === 'N' && collection === groupName) - .length > 0; - // Has group, but parent is not new, parent is also not being created in this snapshot apply - if (parentExists && !parentWillBeCreatedInThisApply) return true; - - return false; - }; - - // Create top level collections (no group, or highest level in existing group) first, - // then continue with nested collections recursively - await createCollections(snapshotDiff.collections.filter(filterCollectionsForCreation)); - - // delete top level collections (no group) first, then continue with nested collections recursively - await deleteCollections( - snapshotDiff.collections.filter( - ({ diff }) => diff[0].kind === 'D' && (diff[0] as DiffDeleted).lhs.meta?.group === null - ) - ); - - for (const { collection, diff } of snapshotDiff.collections) { - if (diff?.[0].kind === 'E' || diff?.[0].kind === 'A') { - const newValues = snapshot.collections.find((field) => { - return field.collection === collection; - }); - - if (newValues) { - try { - await collectionsService.updateOne(collection, newValues, mutationOptions); - } catch (err) { - logger.error(`Failed to update collection "${collection}"`); - throw err; - } - } - } - } - - const fieldsService = new FieldsService({ - knex: trx, - schema: await getSchema({ database: trx, bypassCache: true }), - }); - - for (const { collection, field, diff } of snapshotDiff.fields) { - if (diff?.[0].kind === 'N' && !isNestedMetaUpdate(diff?.[0])) { - try { - await fieldsService.createField(collection, (diff[0] as DiffNew).rhs, undefined, mutationOptions); - } catch (err) { - logger.error(`Failed to create field "${collection}.${field}"`); - throw err; - } - } - - if (diff?.[0].kind === 'E' || diff?.[0].kind === 'A' || isNestedMetaUpdate(diff?.[0])) { - const newValues = snapshot.fields.find((snapshotField) => { - return snapshotField.collection === collection && snapshotField.field === field; - }); - - if (newValues) { - try { - await fieldsService.updateField( - collection, - { - ...newValues, - }, - mutationOptions - ); - } catch (err) { - logger.error(`Failed to update field "${collection}.${field}"`); - throw err; - } - } - } - - if (diff?.[0].kind === 'D' && !isNestedMetaUpdate(diff?.[0])) { - try { - await fieldsService.deleteField(collection, field, mutationOptions); - } catch (err) { - logger.error(`Failed to delete field "${collection}.${field}"`); - throw err; - } - - // Field deletion also cleans up the relationship. We should ignore any relationship - // changes attached to this now non-existing field - snapshotDiff.relations = snapshotDiff.relations.filter( - (relation) => (relation.collection === collection && relation.field === field) === false - ); - } - } - - const relationsService = new RelationsService({ - knex: trx, - schema: await getSchema({ database: trx, bypassCache: true }), - }); - - for (const { collection, field, diff } of snapshotDiff.relations) { - const structure = {}; - - for (const diffEdit of diff) { - set(structure, diffEdit.path!, undefined); - } - - if (diff?.[0].kind === 'N') { - try { - await relationsService.createOne((diff[0] as DiffNew).rhs, mutationOptions); - } catch (err) { - logger.error(`Failed to create relation "${collection}.${field}"`); - throw err; - } - } - - if (diff?.[0].kind === 'E' || diff?.[0].kind === 'A') { - const newValues = snapshot.relations.find((relation) => { - return relation.collection === collection && relation.field === field; - }); - - if (newValues) { - try { - await relationsService.updateOne(collection, field, newValues, mutationOptions); - } catch (err) { - logger.error(`Failed to update relation "${collection}.${field}"`); - throw err; - } - } - } - - if (diff?.[0].kind === 'D') { - try { - await relationsService.deleteOne(collection, field, mutationOptions); - } catch (err) { - logger.error(`Failed to delete relation "${collection}.${field}"`); - throw err; - } - } - } - }); + await applyDiff(current, snapshotDiff, { database, schema }); await systemCache?.clear(); - - if (nestedActionEvents.length > 0) { - const updatedSchema = await getSchema({ database, bypassCache: true }); - - for (const nestedActionEvent of nestedActionEvents) { - nestedActionEvent.context.schema = updatedSchema; - emitter.emitAction(nestedActionEvent.event, nestedActionEvent.meta, nestedActionEvent.context); - } - } -} - -export function isNestedMetaUpdate(diff: Diff): boolean { - if (!diff) return false; - if (diff.kind !== 'N' && diff.kind !== 'D') return false; - if (!diff.path || diff.path.length < 2 || diff.path[0] !== 'meta') return false; - return true; } diff --git a/api/src/utils/get-snapshot-diff.ts b/api/src/utils/get-snapshot-diff.ts index c790e84d3b..6b7ca3d00c 100644 --- a/api/src/utils/get-snapshot-diff.ts +++ b/api/src/utils/get-snapshot-diff.ts @@ -1,6 +1,7 @@ -import { Snapshot, SnapshotDiff } from '../types'; import { diff } from 'deep-diff'; import { orderBy } from 'lodash'; +import { Snapshot, SnapshotDiff, DiffKind } from '../types'; +import { sanitizeCollection, sanitizeField, sanitizeRelation } from './sanitize-schema'; export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDiff { const diffedSnapshot: SnapshotDiff = { @@ -13,7 +14,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif return { collection: currentCollection.collection, - diff: diff(currentCollection, afterCollection), + diff: diff(sanitizeCollection(currentCollection), sanitizeCollection(afterCollection)), }; }), ...after.collections @@ -26,7 +27,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif }) .map((afterCollection) => ({ collection: afterCollection.collection, - diff: diff(undefined, afterCollection), + diff: diff(undefined, sanitizeCollection(afterCollection)), })), ].filter((obj) => Array.isArray(obj.diff)) as SnapshotDiff['collections'], 'collection' @@ -38,10 +39,16 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif (afterField) => afterField.collection === currentField.collection && afterField.field === currentField.field ); + const isAutoIncrementPrimaryKey = + !!currentField.schema?.is_primary_key && !!currentField.schema?.has_auto_increment; + return { collection: currentField.collection, field: currentField.field, - diff: diff(currentField, afterField), + diff: diff( + sanitizeField(currentField, isAutoIncrementPrimaryKey), + sanitizeField(afterField, isAutoIncrementPrimaryKey) + ), }; }), ...after.fields @@ -56,7 +63,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif .map((afterField) => ({ collection: afterField.collection, field: afterField.field, - diff: diff(undefined, afterField), + diff: diff(undefined, sanitizeField(afterField)), })), ].filter((obj) => Array.isArray(obj.diff)) as SnapshotDiff['fields'], ['collection'] @@ -73,7 +80,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif collection: currentRelation.collection, field: currentRelation.field, related_collection: currentRelation.related_collection, - diff: diff(currentRelation, afterRelation), + diff: diff(sanitizeRelation(currentRelation), sanitizeRelation(afterRelation)), }; }), ...after.relations @@ -89,7 +96,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif collection: afterRelation.collection, field: afterRelation.field, related_collection: afterRelation.related_collection, - diff: diff(undefined, afterRelation), + diff: diff(undefined, sanitizeRelation(afterRelation)), })), ].filter((obj) => Array.isArray(obj.diff)) as SnapshotDiff['relations'], ['collection'] @@ -101,7 +108,7 @@ export function getSnapshotDiff(current: Snapshot, after: Snapshot): SnapshotDif */ const deletedCollections = diffedSnapshot.collections - .filter((collection) => collection.diff?.[0].kind === 'D') + .filter((collection) => collection.diff?.[0].kind === DiffKind.DELETE) .map(({ collection }) => collection); diffedSnapshot.fields = diffedSnapshot.fields.filter( diff --git a/api/src/utils/get-snapshot.ts b/api/src/utils/get-snapshot.ts index c54aa83f22..3e4d5e0b19 100644 --- a/api/src/utils/get-snapshot.ts +++ b/api/src/utils/get-snapshot.ts @@ -1,14 +1,16 @@ -import getDatabase from '../database'; +import getDatabase, { getDatabaseClient } from '../database'; import { getSchema } from './get-schema'; import { CollectionsService, FieldsService, RelationsService } from '../services'; import { version } from '../../package.json'; -import { Snapshot, SnapshotField, SnapshotRelation } from '../types'; +import { Collection, Snapshot, SnapshotField, SnapshotRelation } from '../types'; import { Knex } from 'knex'; import { omit, sortBy, toPairs, fromPairs, mapValues, isPlainObject, isArray } from 'lodash'; import { SchemaOverview } from '@directus/shared/types'; +import { sanitizeCollection, sanitizeField, sanitizeRelation } from './sanitize-schema'; export async function getSnapshot(options?: { database?: Knex; schema?: SchemaOverview }): Promise { const database = options?.database ?? getDatabase(); + const vendor = getDatabaseClient(database); const schema = options?.schema ?? (await getSchema({ database, bypassCache: true })); const collectionsService = new CollectionsService({ knex: database, schema }); @@ -32,9 +34,10 @@ export async function getSnapshot(options?: { database?: Knex; schema?: SchemaOv return { version: 1, directus: version, - collections: collectionsSorted, - fields: fieldsSorted, - relations: relationsSorted, + vendor, + collections: collectionsSorted.map((collection) => sanitizeCollection(collection)) as Collection[], + fields: fieldsSorted.map((field) => sanitizeField(field)) as SnapshotField[], + relations: relationsSorted.map((relation) => sanitizeRelation(relation)) as SnapshotRelation[], }; } diff --git a/api/src/utils/get-versioned-hash.test.ts b/api/src/utils/get-versioned-hash.test.ts new file mode 100644 index 0000000000..de8bacd38c --- /dev/null +++ b/api/src/utils/get-versioned-hash.test.ts @@ -0,0 +1,118 @@ +import { test, expect, describe, vi } from 'vitest'; + +import { getVersionedHash } from './get-versioned-hash'; + +vi.mock('../../package.json', () => ({ + version: '10.10.10', +})); + +describe('getVersionedHash', () => { + test.each([ + { + input: { + collection: 'test', + meta: { + accountability: 'all', + archive_app_filter: true, + archive_field: 'status', + archive_value: 'archived', + collapse: 'open', + collection: 'test', + color: null, + display_template: null, + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + sort: null, + sort_field: null, + translations: null, + unarchive_value: 'draft', + }, + schema: { name: 'test' }, + }, + expected: '8acdde88d26ea4142e224fe0b4bfdaab9ac9c923', + }, + { + input: { + collection: 'test', + field: 'id', + meta: { + collection: 'test', + conditions: null, + display: null, + display_options: null, + field: 'id', + group: null, + hidden: true, + interface: 'input', + note: null, + options: null, + readonly: true, + required: false, + sort: null, + special: null, + translations: null, + validation: null, + validation_message: null, + width: 'full', + }, + schema: { + comment: null, + data_type: 'integer', + default_value: "nextval('test_id_seq'::regclass)", + foreign_key_column: null, + foreign_key_schema: null, + foreign_key_table: null, + generation_expression: null, + has_auto_increment: true, + is_generated: false, + is_nullable: false, + is_primary_key: true, + is_unique: true, + max_length: null, + name: 'id', + numeric_precision: 32, + numeric_scale: 0, + schema: 'public', + table: 'test', + }, + type: 'integer', + }, + expected: '60f0d169cfa32799cae884e1bf33e0e49c3ff383', + }, + { + input: { + collection: 'test_example', + field: 'm2m', + related_collection: 'test', + meta: { + id: 1, + junction_field: 'example_id', + many_collection: 'test_example', + many_field: 'test_id', + one_allowed_collections: null, + one_collection: 'test', + one_collection_field: null, + one_deselect_action: 'nullify', + one_field: 'm2m', + sort_field: null, + }, + schema: { + table: 'test_example', + column: 'test_id', + foreign_key_table: 'test', + foreign_key_column: 'id', + constraint_name: 'test_example_test_id_foreign', + on_update: 'NO ACTION', + on_delete: 'SET NULL', + }, + }, + expected: 'fa17767ef6646a72a6cfc211d36886d06896d0fc', + }, + ])('should return $expected', ({ input, expected }) => { + expect(getVersionedHash(input)).toBe(expected); + }); +}); diff --git a/api/src/utils/get-versioned-hash.ts b/api/src/utils/get-versioned-hash.ts new file mode 100644 index 0000000000..8d96b0aa83 --- /dev/null +++ b/api/src/utils/get-versioned-hash.ts @@ -0,0 +1,6 @@ +import hash from 'object-hash'; +import { version } from '../../package.json'; + +export function getVersionedHash(item: Record): string { + return hash({ item, version }); +} diff --git a/api/src/utils/sanitize-schema.test.ts b/api/src/utils/sanitize-schema.test.ts new file mode 100644 index 0000000000..4b48afb40b --- /dev/null +++ b/api/src/utils/sanitize-schema.test.ts @@ -0,0 +1,347 @@ +import { Field, Relation } from '@directus/shared/types'; +import { expect, test, describe } from 'vitest'; +import { Collection } from '../types'; + +import { sanitizeCollection, sanitizeField, sanitizeRelation } from './sanitize-schema'; + +describe('sanitizeCollection', () => { + test.each([ + // Not supported in SQLite + comment in MSSQL + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { comment: null, name: 'test', schema: 'public' }, + }, + // MySQL Only + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { collation: 'latin1_swedish_ci', name: 'test', engine: 'InnoDB' }, + }, + // Postgres Only + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { name: 'test', owner: 'postgres' }, + }, + // SQLite Only + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { name: 'test', sql: 'CREATE TABLE `test` (`id` integer not null primary key autoincrement)' }, + }, + // MSSQL only + { + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { name: 'test', catalog: 'test-db' }, + }, + ] satisfies Collection[])('should only contain name property in collection schema', (testCollection) => { + const result = sanitizeCollection(testCollection); + + expect(result).toEqual({ + collection: 'test', + meta: { + accountability: 'all', + collection: 'test', + group: null, + hidden: false, + icon: null, + item_duplication_fields: null, + note: null, + singleton: false, + translations: {}, + }, + schema: { name: 'test' }, + }); + }); +}); + +describe('sanitizeField', () => { + test('should only contain certain properties in field schema when sanitizeAllSchema is false', () => { + const testField = { + collection: 'test', + field: 'id', + name: 'id', + meta: { + id: 1, + collection: 'test', + conditions: null, + display: null, + display_options: null, + field: 'id', + group: null, + hidden: true, + interface: 'input', + note: null, + options: null, + readonly: true, + required: false, + sort: null, + special: null, + translations: null, + validation: null, + validation_message: null, + width: 'full', + }, + schema: { + comment: null, + data_type: 'integer', + default_value: "nextval('test_id_seq'::regclass)", + foreign_key_column: null, + foreign_key_schema: null, + foreign_key_table: null, + generation_expression: null, + has_auto_increment: true, + is_generated: false, + is_nullable: false, + is_primary_key: true, + is_unique: true, + max_length: null, + name: 'id', + numeric_precision: 32, + numeric_scale: 0, + schema: 'public', + table: 'test', + }, + type: 'integer', + } satisfies Field; + + const result = sanitizeField(testField); + + expect(result).toEqual({ + collection: 'test', + field: 'id', + name: 'id', + meta: { + id: 1, + collection: 'test', + conditions: null, + display: null, + display_options: null, + field: 'id', + group: null, + hidden: true, + interface: 'input', + note: null, + options: null, + readonly: true, + required: false, + sort: null, + special: null, + translations: null, + validation: null, + validation_message: null, + width: 'full', + }, + schema: { + data_type: 'integer', + default_value: "nextval('test_id_seq'::regclass)", + foreign_key_column: null, + foreign_key_table: null, + generation_expression: null, + has_auto_increment: true, + is_generated: false, + is_nullable: false, + is_primary_key: true, + is_unique: true, + max_length: null, + name: 'id', + numeric_precision: 32, + numeric_scale: 0, + + table: 'test', + }, + type: 'integer', + }); + }); + + test('should not contain field schema when sanitizeAllSchema is true', () => { + const testField = { + collection: 'test', + field: 'id', + name: 'id', + meta: { + id: 1, + collection: 'test', + conditions: null, + display: null, + display_options: null, + field: 'id', + group: null, + hidden: true, + interface: 'input', + note: null, + options: null, + readonly: true, + required: false, + sort: null, + special: null, + translations: null, + validation: null, + validation_message: null, + width: 'full', + }, + schema: { + data_type: 'integer', + default_value: "nextval('test_id_seq'::regclass)", + foreign_key_column: null, + foreign_key_table: null, + generation_expression: null, + has_auto_increment: true, + is_generated: false, + is_nullable: false, + is_primary_key: true, + is_unique: true, + max_length: null, + name: 'id', + numeric_precision: 32, + numeric_scale: 0, + table: 'test', + }, + type: 'integer', + } satisfies Field; + + const result = sanitizeField(testField, true); + + expect(result).toEqual({ + collection: 'test', + field: 'id', + name: 'id', + meta: { + id: 1, + collection: 'test', + conditions: null, + display: null, + display_options: null, + field: 'id', + group: null, + hidden: true, + interface: 'input', + note: null, + options: null, + readonly: true, + required: false, + sort: null, + special: null, + translations: null, + validation: null, + validation_message: null, + width: 'full', + }, + type: 'integer', + }); + }); +}); + +describe('sanitizeRelation', () => { + test.each([ + // Postgres + MSSSQL + { + collection: 'test_example', + field: 'm2m', + related_collection: 'test', + meta: { + id: 1, + junction_field: 'example_id', + many_collection: 'test_example', + many_field: 'test_id', + one_allowed_collections: null, + one_collection: 'test', + one_collection_field: null, + one_deselect_action: 'nullify', + one_field: 'm2m', + sort_field: null, + }, + schema: { + table: 'test_example', + column: 'test_id', + foreign_key_table: 'test', + foreign_key_column: 'id', + foreign_key_schema: 'public', + constraint_name: 'test_example_test_id_foreign', + on_update: 'NO ACTION', + on_delete: 'SET NULL', + }, + }, + ] satisfies Relation[])('should only contain certain properties in relation schema', (testRelation) => { + const result = sanitizeRelation(testRelation); + + expect(result).toEqual({ + collection: 'test_example', + field: 'm2m', + related_collection: 'test', + meta: { + id: 1, + junction_field: 'example_id', + many_collection: 'test_example', + many_field: 'test_id', + one_allowed_collections: null, + one_collection: 'test', + one_collection_field: null, + one_deselect_action: 'nullify', + one_field: 'm2m', + sort_field: null, + }, + schema: { + table: 'test_example', + column: 'test_id', + foreign_key_table: 'test', + foreign_key_column: 'id', + constraint_name: 'test_example_test_id_foreign', + on_update: 'NO ACTION', + on_delete: 'SET NULL', + }, + }); + }); +}); diff --git a/api/src/utils/sanitize-schema.ts b/api/src/utils/sanitize-schema.ts new file mode 100644 index 0000000000..14e4e28742 --- /dev/null +++ b/api/src/utils/sanitize-schema.ts @@ -0,0 +1,81 @@ +import { Field, Relation } from '@directus/shared/types'; +import { pick } from 'lodash'; +import { Collection } from '../types'; + +/** + * Pick certain database vendor specific collection properties that should be compared when performing diff + * + * @param collection collection to sanitize + * @returns sanitized collection + * + * @see {@link https://github.com/knex/knex-schema-inspector/blob/master/lib/types/table.ts} + */ + +export function sanitizeCollection(collection: Collection | undefined) { + if (!collection) return collection; + + return pick(collection, ['collection', 'fields', 'meta', 'schema.name']); +} + +/** + * Pick certain database vendor specific field properties that should be compared when performing diff + * + * @param field field to sanitize + * @param sanitizeAllSchema Whether or not the whole field schema should be sanitized. Mainly used to prevent modifying autoincrement fields + * @returns sanitized field + * + * @see {@link https://github.com/knex/knex-schema-inspector/blob/master/lib/types/column.ts} + */ +export function sanitizeField(field: Field | undefined, sanitizeAllSchema = false) { + if (!field) return field; + + const defaultPaths = ['collection', 'field', 'type', 'meta', 'name', 'children']; + const pickedPaths = sanitizeAllSchema + ? defaultPaths + : [ + ...defaultPaths, + 'schema.name', + 'schema.table', + 'schema.data_type', + 'schema.default_value', + 'schema.max_length', + 'schema.numeric_precision', + 'schema.numeric_scale', + 'schema.is_nullable', + 'schema.is_unique', + 'schema.is_primary_key', + 'schema.is_generated', + 'schema.generation_expression', + 'schema.has_auto_increment', + 'schema.foreign_key_table', + 'schema.foreign_key_column', + ]; + + return pick(field, pickedPaths); +} + +/** + * Pick certain database vendor specific relation properties that should be compared when performing diff + * + * @param relation relation to sanitize + * @returns sanitized relation + * + * @see {@link https://github.com/knex/knex-schema-inspector/blob/master/lib/types/foreign-key.ts} + */ +export function sanitizeRelation(relation: Relation | undefined) { + if (!relation) return relation; + + return pick(relation, [ + 'collection', + 'field', + 'related_collection', + 'meta', + 'schema.table', + 'schema.column', + 'schema.foreign_key_table', + 'schema.foreign_key_column', + 'schema.constraint_name', + 'schema.on_update', + 'schema.on_delete', + ]); +} diff --git a/api/src/utils/validate-diff.test.ts b/api/src/utils/validate-diff.test.ts new file mode 100644 index 0000000000..8aa8de142d --- /dev/null +++ b/api/src/utils/validate-diff.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, test } from 'vitest'; +import { Collection } from '../types/collection'; +import { + Snapshot, + SnapshotDiff, + SnapshotDiffWithHash, + SnapshotField, + SnapshotRelation, + SnapshotWithHash, +} from '../types/snapshot'; +import { validateApplyDiff } from './validate-diff'; + +test('should fail on invalid diff schema', () => { + const diff = {} as SnapshotDiffWithHash; + const snapshot = {} as SnapshotWithHash; + + expect(() => validateApplyDiff(diff, snapshot)).toThrowError('"hash" is required'); +}); + +test('should fail on invalid hash', () => { + const diff = { + hash: 'abc', + diff: { collections: [{ collection: 'test', diff: [] }], fields: [], relations: [] }, + } as SnapshotDiffWithHash; + const snapshot = { hash: 'xyz' } as SnapshotWithHash; + + expect(() => validateApplyDiff(diff, snapshot)).toThrowError( + "Provided hash does not match the current instance's schema hash" + ); +}); + +describe('should throw accurate error', () => { + const baseDiff = (partialDiff: Partial): SnapshotDiffWithHash => { + return { + hash: 'abc', + diff: { + fields: [], + collections: [], + relations: [], + ...partialDiff, + }, + }; + }; + const baseSnapshot = (partialSnapshot?: Partial) => { + return { + hash: 'xyz', + collections: [] as Collection[], + fields: [] as SnapshotField[], + relations: [] as SnapshotRelation[], + ...partialSnapshot, + } as SnapshotWithHash; + }; + + test('creating collection which already exists', () => { + const diff = baseDiff({ + collections: [{ collection: 'test', diff: [{ kind: 'N', rhs: {} as Collection }] }], + }); + const snapshot = baseSnapshot({ collections: [{ collection: 'test' } as Collection] }); + + expect(() => validateApplyDiff(diff, snapshot)).toThrowError( + 'Provided diff is trying to create collection "test" but it already exists' + ); + }); + + test('deleting collection which does not exist', () => { + const diff = baseDiff({ + collections: [{ collection: 'test', diff: [{ kind: 'D', lhs: {} as Collection }] }], + }); + + expect(() => validateApplyDiff(diff, baseSnapshot())).toThrowError( + 'Provided diff is trying to delete collection "test" but it does not exist' + ); + }); + + test('creating field which already exists', () => { + const diff = baseDiff({ + fields: [{ collection: 'test', field: 'test', diff: [{ kind: 'N', rhs: {} as SnapshotField }] }], + }); + const snapshot = baseSnapshot({ fields: [{ collection: 'test', field: 'test' } as SnapshotField] }); + + expect(() => validateApplyDiff(diff, snapshot)).toThrowError( + 'Provided diff is trying to create field "test.test" but it already exists' + ); + }); + + test('deleting field which does not exist', () => { + const diff = baseDiff({ + fields: [{ collection: 'test', field: 'test', diff: [{ kind: 'D', lhs: {} as SnapshotField }] }], + }); + + expect(() => validateApplyDiff(diff, baseSnapshot())).toThrowError( + 'Provided diff is trying to delete field "test.test" but it does not exist' + ); + }); + + test('creating relation which already exists', () => { + const diff = baseDiff({ + relations: [ + { + collection: 'test', + field: 'test', + related_collection: 'relation', + diff: [{ kind: 'N', rhs: {} as SnapshotRelation }], + }, + ], + }); + const snapshot = baseSnapshot({ + relations: [{ collection: 'test', field: 'test', related_collection: 'relation' } as SnapshotRelation], + }); + + expect(() => validateApplyDiff(diff, snapshot)).toThrowError( + 'Provided diff is trying to create relation "test.test-> relation" but it already exists' + ); + }); + + test('deleting relation which does not exist', () => { + const diff = baseDiff({ + relations: [ + { + collection: 'test', + field: 'test', + related_collection: 'relation', + diff: [{ kind: 'D', lhs: {} as SnapshotRelation }], + }, + ], + }); + + expect(() => validateApplyDiff(diff, baseSnapshot())).toThrowError( + 'Provided diff is trying to delete relation "test.test-> relation" but it does not exist' + ); + }); +}); + +test('should detect empty diff', () => { + const diff = { + hash: 'abc', + diff: { collections: [], fields: [], relations: [] }, + }; + const snapshot = {} as SnapshotWithHash; + + expect(validateApplyDiff(diff, snapshot)).toBe(false); +}); + +test('should pass on valid diff', () => { + const diff = { + hash: 'abc', + diff: { collections: [{ collection: 'test', diff: [] }], fields: [], relations: [] }, + }; + const snapshot = { hash: 'abc' } as SnapshotWithHash; + + expect(validateApplyDiff(diff, snapshot)).toBe(true); +}); diff --git a/api/src/utils/validate-diff.ts b/api/src/utils/validate-diff.ts new file mode 100644 index 0000000000..1c1f5dad02 --- /dev/null +++ b/api/src/utils/validate-diff.ts @@ -0,0 +1,150 @@ +import Joi from 'joi'; +import { InvalidPayloadException } from '../index'; +import { DiffKind, SnapshotDiffWithHash, SnapshotWithHash } from '../types/snapshot'; + +const deepDiffSchema = Joi.object({ + kind: Joi.string() + .valid(...Object.values(DiffKind)) + .required(), + path: Joi.array().items(Joi.string()), + lhs: Joi.object().when('kind', { is: [DiffKind.DELETE, DiffKind.EDIT], then: Joi.required() }), + rhs: Joi.object().when('kind', { is: [DiffKind.NEW, DiffKind.EDIT], then: Joi.required() }), + index: Joi.number().when('kind', { is: DiffKind.ARRAY, then: Joi.required() }), + item: Joi.link('/').when('kind', { is: DiffKind.ARRAY, then: Joi.required() }), +}); + +const applyJoiSchema = Joi.object({ + hash: Joi.string().required(), + diff: Joi.object({ + collections: Joi.array() + .items( + Joi.object({ + collection: Joi.string().required(), + diff: Joi.array().items(deepDiffSchema).required(), + }) + ) + .required(), + fields: Joi.array() + .items( + Joi.object({ + collection: Joi.string().required(), + field: Joi.string().required(), + diff: Joi.array().items(deepDiffSchema).required(), + }) + ) + .required(), + relations: Joi.array() + .items( + Joi.object({ + collection: Joi.string().required(), + field: Joi.string().required(), + related_collection: Joi.string(), + diff: Joi.array().items(deepDiffSchema).required(), + }) + ) + .required(), + }).required(), +}); + +/** + * Validates the diff against the current schema snapshot. + * + * @returns True if the diff can be applied (valid & not empty). + */ +export function validateApplyDiff(applyDiff: SnapshotDiffWithHash, currentSnapshotWithHash: SnapshotWithHash) { + const { error } = applyJoiSchema.validate(applyDiff); + if (error) throw new InvalidPayloadException(error.message); + + // No changes to apply + if ( + applyDiff.diff.collections.length === 0 && + applyDiff.diff.fields.length === 0 && + applyDiff.diff.relations.length === 0 + ) { + return false; + } + + // Diff can be applied due to matching hash + if (applyDiff.hash === currentSnapshotWithHash.hash) return true; + + for (const diffCollection of applyDiff.diff.collections) { + const collection = diffCollection.collection; + + if (diffCollection.diff[0]?.kind === DiffKind.NEW) { + const existingCollection = currentSnapshotWithHash.collections.find( + (c) => c.collection === diffCollection.collection + ); + + if (existingCollection) { + throw new InvalidPayloadException( + `Provided diff is trying to create collection "${collection}" but it already exists. Please generate a new diff and try again.` + ); + } + } else if (diffCollection.diff[0]?.kind === DiffKind.DELETE) { + const existingCollection = currentSnapshotWithHash.collections.find( + (c) => c.collection === diffCollection.collection + ); + + if (!existingCollection) { + throw new InvalidPayloadException( + `Provided diff is trying to delete collection "${collection}" but it does not exist. Please generate a new diff and try again.` + ); + } + } + } + for (const diffField of applyDiff.diff.fields) { + const field = `${diffField.collection}.${diffField.field}`; + + if (diffField.diff[0]?.kind === DiffKind.NEW) { + const existingField = currentSnapshotWithHash.fields.find( + (f) => f.collection === diffField.collection && f.field === diffField.field + ); + + if (existingField) { + throw new InvalidPayloadException( + `Provided diff is trying to create field "${field}" but it already exists. Please generate a new diff and try again.` + ); + } + } else if (diffField.diff[0]?.kind === DiffKind.DELETE) { + const existingField = currentSnapshotWithHash.fields.find( + (f) => f.collection === diffField.collection && f.field === diffField.field + ); + + if (!existingField) { + throw new InvalidPayloadException( + `Provided diff is trying to delete field "${field}" but it does not exist. Please generate a new diff and try again.` + ); + } + } + } + for (const diffRelation of applyDiff.diff.relations) { + let relation = `${diffRelation.collection}.${diffRelation.field}`; + if (diffRelation.related_collection) relation += `-> ${diffRelation.related_collection}`; + + if (diffRelation.diff[0]?.kind === DiffKind.NEW) { + const existingRelation = currentSnapshotWithHash.relations.find( + (r) => r.collection === diffRelation.collection && r.field === diffRelation.field + ); + + if (existingRelation) { + throw new InvalidPayloadException( + `Provided diff is trying to create relation "${relation}" but it already exists. Please generate a new diff and try again.` + ); + } + } else if (diffRelation.diff[0]?.kind === DiffKind.DELETE) { + const existingRelation = currentSnapshotWithHash.relations.find( + (r) => r.collection === diffRelation.collection && r.field === diffRelation.field + ); + + if (!existingRelation) { + throw new InvalidPayloadException( + `Provided diff is trying to delete relation "${relation}" but it does not exist. Please generate a new diff and try again.` + ); + } + } + } + + throw new InvalidPayloadException( + `Provided hash does not match the current instance's schema hash, indicating the schema has changed after this diff was generated. Please generate a new diff and try again.` + ); +} diff --git a/api/src/utils/validate-query.test.ts b/api/src/utils/validate-query.test.ts new file mode 100644 index 0000000000..d5924ff82a --- /dev/null +++ b/api/src/utils/validate-query.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, test, vi } from 'vitest'; +import { validateQuery } from './validate-query'; + +vi.mock('../env', async () => { + const actual = (await vi.importActual('../env')) as { default: Record }; + const MOCK_ENV = { + ...actual.default, + MAX_QUERY_LIMIT: 100, + }; + return { + default: MOCK_ENV, + getEnv: () => MOCK_ENV, + }; +}); + +describe('export', () => { + test.each(['csv', 'json', 'xml', 'yaml'])('should accept format %i', (format) => { + expect(() => validateQuery({ export: format } as any)).not.toThrowError(); + }); + + test('should error with invalid-format', () => { + expect(() => validateQuery({ export: 'invalid-format' } as any)).toThrowError('"export" must be one of'); + }); +}); diff --git a/api/src/utils/validate-query.ts b/api/src/utils/validate-query.ts index f4900e2b97..5b03158444 100644 --- a/api/src/utils/validate-query.ts +++ b/api/src/utils/validate-query.ts @@ -16,7 +16,7 @@ const querySchema = Joi.object({ page: Joi.number().integer().min(0), meta: Joi.array().items(Joi.string().valid('total_count', 'filter_count')), search: Joi.string(), - export: Joi.string().valid('json', 'csv', 'xml'), + export: Joi.string().valid('csv', 'json', 'xml', 'yaml'), aggregate: Joi.object(), deep: Joi.object(), alias: Joi.object(), diff --git a/api/src/utils/validate-snapshot.test.ts b/api/src/utils/validate-snapshot.test.ts new file mode 100644 index 0000000000..07be6b6d16 --- /dev/null +++ b/api/src/utils/validate-snapshot.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, test, vi } from 'vitest'; +import { Snapshot } from '../types/snapshot'; +import { validateSnapshot } from './validate-snapshot'; + +vi.mock('../../package.json', () => ({ + version: '9.22.4', +})); + +vi.mock('../database', () => ({ + getDatabaseClient: () => 'sqlite', +})); + +describe('should fail on invalid snapshot schema', () => { + test('empty snapshot', () => { + const snapshot = {} as Snapshot; + + expect(() => validateSnapshot(snapshot)).toThrowError('"version" is required'); + }); + + test('invalid version', () => { + const snapshot = { version: 0 } as Snapshot; + + expect(() => validateSnapshot(snapshot)).toThrowError('"version" must be [1]'); + }); + + test('invalid schema', () => { + const snapshot = { version: 1, directus: '9.22.4', collections: {} } as Snapshot; + + expect(() => validateSnapshot(snapshot)).toThrowError('"collections" must be an array'); + }); +}); + +describe('should require force option on version / vendor mismatch', () => { + test('directus version mismatch', () => { + const snapshot = { version: 1, directus: '9.22.3' } as Snapshot; + + expect(() => validateSnapshot(snapshot)).toThrowError( + "Provided snapshot's directus version 9.22.3 does not match the current instance's version 9.22.4" + ); + }); + + test('db vendor mismatch', () => { + const snapshot = { version: 1, directus: '9.22.4', vendor: 'postgres' } as Snapshot; + + expect(() => validateSnapshot(snapshot)).toThrowError( + "Provided snapshot's vendor postgres does not match the current instance's vendor sqlite." + ); + }); +}); + +test('should allow bypass on version / vendor mismatch via force option ', () => { + const snapshot = { version: 1, directus: '9.22.3', vendor: 'postgres' } as Snapshot; + + expect(validateSnapshot(snapshot, true)).toBeUndefined(); +}); diff --git a/api/src/utils/validate-snapshot.ts b/api/src/utils/validate-snapshot.ts new file mode 100644 index 0000000000..2a41440850 --- /dev/null +++ b/api/src/utils/validate-snapshot.ts @@ -0,0 +1,80 @@ +import { version as currentDirectusVersion } from '../../package.json'; +import { InvalidPayloadException } from '../exceptions'; +import { getDatabaseClient } from '../database'; +import Joi from 'joi'; +import { TYPES } from '@directus/shared/constants'; +import { ALIAS_TYPES } from '../constants'; +import { DatabaseClients, Snapshot } from '../types'; + +const snapshotJoiSchema = Joi.object({ + version: Joi.number().valid(1).required(), + directus: Joi.string().required(), + vendor: Joi.string() + .valid(...DatabaseClients) + .optional(), + collections: Joi.array().items( + Joi.object({ + collection: Joi.string(), + meta: Joi.any(), + schema: Joi.object({ + name: Joi.string(), + }), + }) + ), + fields: Joi.array().items( + Joi.object({ + collection: Joi.string(), + field: Joi.string(), + meta: Joi.any(), + schema: Joi.object({ + default_value: Joi.any(), + max_length: [Joi.number(), Joi.string(), Joi.valid(null)], + is_nullable: Joi.bool(), + }) + .unknown() + .allow(null), + type: Joi.string() + .valid(...TYPES, ...ALIAS_TYPES) + .allow(null), + }) + ), + relations: Joi.array().items( + Joi.object({ + collection: Joi.string(), + field: Joi.string(), + meta: Joi.any(), + related_collection: Joi.any(), + schema: Joi.any(), + }) + ), +}); + +/** + * Validates the snapshot against the current instance. + **/ +export function validateSnapshot(snapshot: Snapshot, force = false) { + const { error } = snapshotJoiSchema.validate(snapshot); + if (error) throw new InvalidPayloadException(error.message); + + // Bypass checks when "force" option is enabled + if (force) return; + + if (snapshot.directus !== currentDirectusVersion) { + throw new InvalidPayloadException( + `Provided snapshot's directus version ${snapshot.directus} does not match the current instance's version ${currentDirectusVersion}. You can bypass this check by passing the "force" query parameter.` + ); + } + + if (!snapshot.vendor) { + throw new InvalidPayloadException( + 'Provided snapshot does not contain the "vendor" property. You can bypass this check by passing the "force" query parameter.' + ); + } + + const currentVendor = getDatabaseClient(); + if (snapshot.vendor !== currentVendor) { + throw new InvalidPayloadException( + `Provided snapshot's vendor ${snapshot.vendor} does not match the current instance's vendor ${currentVendor}. You can bypass this check by passing the "force" query parameter.` + ); + } +} diff --git a/app/src/lang/translations/en-US.yaml b/app/src/lang/translations/en-US.yaml index 51743b3982..633ba88a1d 100644 --- a/app/src/lang/translations/en-US.yaml +++ b/app/src/lang/translations/en-US.yaml @@ -250,6 +250,7 @@ float: Float integer: Integer json: JSON xml: XML +yaml: YAML string: String text: Text time: Time diff --git a/app/src/views/private/components/export-sidebar-detail.vue b/app/src/views/private/components/export-sidebar-detail.vue index b1df838470..c5ef5b38f6 100644 --- a/app/src/views/private/components/export-sidebar-detail.vue +++ b/app/src/views/private/components/export-sidebar-detail.vue @@ -104,6 +104,10 @@ text: t('xml'), value: 'xml', }, + { + text: t('yaml'), + value: 'yaml', + }, ]" /> diff --git a/package.json b/package.json index b18546d8c3..df6b9f69ce 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "devDependencies": { "@directus/shared": "workspace:*", "@types/jest": "29.2.4", + "@types/js-yaml": "4.0.5", "@types/lodash": "4.14.191", "@types/seedrandom": "3.0.2", "@types/supertest": "2.0.12", @@ -42,6 +43,7 @@ "globby": "11.1.0", "jest": "29.3.1", "jest-environment-node": "29.3.1", + "js-yaml": "4.1.0", "json-to-graphql-query": "2.2.4", "knex": "2.4.1", "lint-staged": "13.1.0", diff --git a/packages/schema/package.json b/packages/schema/package.json index 6980d0ebd6..5ab61e7eac 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -43,7 +43,7 @@ }, "dependencies": { "knex": "2.4.1", - "knex-schema-inspector": "3.0.0" + "knex-schema-inspector": "3.0.1" }, "devDependencies": { "typescript": "4.9.4" diff --git a/packages/shared/package.json b/packages/shared/package.json index 6301d2e902..1b35a755c1 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -71,7 +71,7 @@ "geojson": "0.5.0", "joi": "17.7.0", "knex": "2.4.1", - "knex-schema-inspector": "3.0.0", + "knex-schema-inspector": "3.0.1", "lodash": "4.17.21", "micromustache": "8.0.3", "nanoid": "4.0.0", diff --git a/packages/specs/src/components/diff.yaml b/packages/specs/src/components/diff.yaml new file mode 100644 index 0000000000..a8cd9d7ef5 --- /dev/null +++ b/packages/specs/src/components/diff.yaml @@ -0,0 +1,46 @@ +type: object +properties: + hash: + type: string + diff: + type: object + properties: + collections: + type: array + items: + type: object + properties: + collection: + type: string + diff: + type: array + items: + type: object + fields: + type: array + items: + type: object + properties: + collection: + type: string + field: + type: string + diff: + type: array + items: + type: object + relations: + type: array + items: + type: object + properties: + collection: + type: string + field: + type: string + related_collection: + type: string + diff: + type: array + items: + type: object diff --git a/packages/specs/src/components/schema.yaml b/packages/specs/src/components/schema.yaml new file mode 100644 index 0000000000..9be3a47b7c --- /dev/null +++ b/packages/specs/src/components/schema.yaml @@ -0,0 +1,21 @@ +type: object +properties: + version: + type: integer + example: 1 + directus: + type: string + vendor: + type: string + collections: + type: array + items: + $ref: '../openapi.yaml#/components/schemas/Collections' + fields: + type: array + items: + $ref: '../openapi.yaml#/components/schemas/Fields' + relations: + type: array + items: + $ref: '../openapi.yaml#/components/schemas/Relations' diff --git a/packages/specs/src/openapi.yaml b/packages/specs/src/openapi.yaml index 99671d5b60..446409203a 100644 --- a/packages/specs/src/openapi.yaml +++ b/packages/specs/src/openapi.yaml @@ -73,6 +73,8 @@ tags: - name: Roles description: Roles are groups of users that share permissions. x-collection: directus_roles + - name: Schema + description: Retrieve and update the schema of an instance. - name: Server description: Access to where Directus runs. Allows you to make sure your server has everything needed to run the platform, and @@ -205,6 +207,14 @@ paths: /roles/{id}: $ref: './paths/roles/role.yaml' + # Schema + /schema/snapshot: + $ref: './paths/schema/snapshot.yaml' + /schema/apply: + $ref: './paths/schema/apply.yaml' + /schema/diff: + $ref: './paths/schema/diff.yaml' + # Server /server/info: $ref: './paths/server/info.yaml' @@ -263,6 +273,8 @@ components: $ref: './components/preset.yaml' Collections: $ref: './components/collection.yaml' + Diff: + $ref: './components/diff.yaml' Fields: $ref: './components/field.yaml' Files: @@ -283,6 +295,8 @@ components: $ref: './components/revision.yaml' Roles: $ref: './components/role.yaml' + Schema: + $ref: './components/schema.yaml' Settings: $ref: './components/setting.yaml' Users: @@ -330,6 +344,8 @@ components: $ref: './parameters/fields.yaml' Mode: $ref: './parameters/mode.yaml' + Export: + $ref: './parameters/export.yaml' responses: NotFoundError: $ref: './responses/notFoundError.yaml' diff --git a/packages/specs/src/parameters/export.yaml b/packages/specs/src/parameters/export.yaml new file mode 100644 index 0000000000..6d26116281 --- /dev/null +++ b/packages/specs/src/parameters/export.yaml @@ -0,0 +1,11 @@ +name: export +description: Saves the API response to a file. Accepts one of "csv", "json", "xml", "yaml". +in: query +required: false +schema: + type: string + enum: + - csv + - json + - xml + - yaml diff --git a/packages/specs/src/paths/items/items.yaml b/packages/specs/src/paths/items/items.yaml index 30a9802708..5e44ba8c6b 100644 --- a/packages/specs/src/paths/items/items.yaml +++ b/packages/specs/src/paths/items/items.yaml @@ -43,6 +43,7 @@ post: requestBody: content: application/json: + # Intentionally empty, see https://github.com/directus/directus/pull/16294 schema: responses: '200': diff --git a/packages/specs/src/paths/schema/apply.yaml b/packages/specs/src/paths/schema/apply.yaml new file mode 100644 index 0000000000..5c44f062f6 --- /dev/null +++ b/packages/specs/src/paths/schema/apply.yaml @@ -0,0 +1,22 @@ +post: + summary: Apply Schema Difference + description: + Update the instance's schema by passing the diff previously retrieved via `/schema/diff` endpoint in the request + body. This endpoint is only available to admin users. + operationId: schemaApply + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + data: + $ref: '../../openapi.yaml#/components/schemas/Diff' + responses: + '204': + description: Successful request + '403': + $ref: '../../openapi.yaml#/components/responses/UnauthorizedError' + tags: + - Schema diff --git a/packages/specs/src/paths/schema/diff.yaml b/packages/specs/src/paths/schema/diff.yaml new file mode 100644 index 0000000000..e85ebd3a3d --- /dev/null +++ b/packages/specs/src/paths/schema/diff.yaml @@ -0,0 +1,45 @@ +post: + summary: Retrieve Schema Difference + description: + Compare the current instance's schema against the schema snapshot in JSON request body or a JSON/YAML file and + retrieve the difference. This endpoint is only available to admin users. + operationId: schemaDiff + parameters: + - name: force + description: Bypass version and database vendor restrictions. + in: query + required: false + schema: + type: boolean + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + data: + $ref: '../../openapi.yaml#/components/schemas/Schema' + multipart/form-data: + schema: + type: object + properties: + file: + type: string + format: binary + responses: + '200': + description: Successful request + content: + application/json: + schema: + type: object + properties: + data: + $ref: '../../openapi.yaml#/components/schemas/Diff' + '204': + description: No schema difference. + '403': + $ref: '../../openapi.yaml#/components/responses/UnauthorizedError' + tags: + - Schema diff --git a/packages/specs/src/paths/schema/snapshot.yaml b/packages/specs/src/paths/schema/snapshot.yaml new file mode 100644 index 0000000000..e7fbc7d19f --- /dev/null +++ b/packages/specs/src/paths/schema/snapshot.yaml @@ -0,0 +1,24 @@ +get: + summary: Retrieve Schema Snapshot + description: Retrieve the current schema. This endpoint is only available to admin users. + operationId: schemaSnapshot + parameters: + - $ref: '../../openapi.yaml#/components/parameters/Export' + responses: + '200': + description: Successful request + content: + application/json: + schema: + type: object + properties: + data: + $ref: '../../openapi.yaml#/components/schemas/Schema' + text/yaml: + schema: + type: string + format: binary + '403': + $ref: '../../openapi.yaml#/components/responses/UnauthorizedError' + tags: + - Schema diff --git a/packages/utils/src/node/index.ts b/packages/utils/src/node/index.ts index 13eb4d4af8..71648c45a1 100644 --- a/packages/utils/src/node/index.ts +++ b/packages/utils/src/node/index.ts @@ -1 +1,2 @@ +export * from './readable-stream-to-string.js'; export * from './is-readable-stream.js'; diff --git a/packages/utils/src/node/readable-stream-to-string.test.ts b/packages/utils/src/node/readable-stream-to-string.test.ts new file mode 100644 index 0000000000..2987cb1eaa --- /dev/null +++ b/packages/utils/src/node/readable-stream-to-string.test.ts @@ -0,0 +1,11 @@ +import { test, expect } from 'vitest'; +import { Readable } from 'node:stream'; + +import { readableStreamToString } from './readable-stream-to-string.js'; + +test.each([Readable.from('test', { encoding: 'utf8' }), Readable.from(Buffer.from([0x74, 0x65, 0x73, 0x74]))])( + 'Returns readable stream as string', + async (readableStream) => { + expect(readableStreamToString(readableStream)).resolves.toBe('test'); + } +); diff --git a/packages/utils/src/node/readable-stream-to-string.ts b/packages/utils/src/node/readable-stream-to-string.ts new file mode 100644 index 0000000000..71dcc40d7e --- /dev/null +++ b/packages/utils/src/node/readable-stream-to-string.ts @@ -0,0 +1,11 @@ +import type { Readable } from 'node:stream'; + +export const readableStreamToString = async (stream: Readable): Promise => { + const chunks = []; + + for await (const chunk of stream) { + chunks.push(Buffer.from(chunk)); + } + + return Buffer.concat(chunks).toString('utf8'); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 975166529d..42c2ba2850 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,6 +6,7 @@ importers: specifiers: '@directus/shared': workspace:* '@types/jest': 29.2.4 + '@types/js-yaml': 4.0.5 '@types/lodash': 4.14.191 '@types/seedrandom': 3.0.2 '@types/supertest': 2.0.12 @@ -21,6 +22,7 @@ importers: globby: 11.1.0 jest: 29.3.1 jest-environment-node: 29.3.1 + js-yaml: 4.1.0 json-to-graphql-query: 2.2.4 knex: 2.4.1 lint-staged: 13.1.0 @@ -36,6 +38,7 @@ importers: devDependencies: '@directus/shared': link:packages/shared '@types/jest': 29.2.4 + '@types/js-yaml': 4.0.5 '@types/lodash': 4.14.191 '@types/seedrandom': 3.0.2 '@types/supertest': 2.0.12 @@ -51,6 +54,7 @@ importers: globby: 11.1.0 jest: 29.3.1 jest-environment-node: 29.3.1 + js-yaml: 4.1.0 json-to-graphql-query: 2.2.4 knex: 2.4.1 lint-staged: 13.1.0 @@ -80,6 +84,7 @@ importers: '@directus/storage-driver-gcs': workspace:* '@directus/storage-driver-local': workspace:* '@directus/storage-driver-s3': workspace:* + '@directus/utils': workspace:* '@godaddy/terminus': 4.11.2 '@keyv/redis': 2.5.4 '@ngneat/falso': 6.3.2 @@ -164,7 +169,7 @@ importers: keyv-memcache: 1.3.3 knex: 2.4.1 knex-mock-client: 1.11.0 - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 ldapjs: 2.3.3 liquidjs: 10.3.3 lodash: 4.17.21 @@ -226,6 +231,7 @@ importers: '@directus/storage-driver-gcs': link:../packages/storage-driver-gcs '@directus/storage-driver-local': link:../packages/storage-driver-local '@directus/storage-driver-s3': link:../packages/storage-driver-s3 + '@directus/utils': link:../packages/utils '@godaddy/terminus': 4.11.2 '@rollup/plugin-alias': 4.0.2_rollup@3.7.5 '@rollup/plugin-virtual': 3.0.1_rollup@3.7.5 @@ -268,7 +274,7 @@ importers: jsonwebtoken: 9.0.0 keyv: 4.5.2 knex: 2.4.1_i7frrkiwsbjh7pywr5yuup4dni - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 ldapjs: 2.3.3 liquidjs: 10.3.3 lodash: 4.17.21 @@ -648,11 +654,11 @@ importers: packages/schema: specifiers: knex: 2.4.1 - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 typescript: 4.9.4 dependencies: knex: 2.4.1 - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 devDependencies: typescript: 4.9.4 @@ -672,7 +678,7 @@ importers: geojson: 0.5.0 joi: 17.7.0 knex: 2.4.1 - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 lodash: 4.17.21 micromustache: 8.0.3 nanoid: 4.0.0 @@ -692,7 +698,7 @@ importers: geojson: 0.5.0 joi: 17.7.0 knex: 2.4.1 - knex-schema-inspector: 3.0.0 + knex-schema-inspector: 3.0.1 lodash: 4.17.21 micromustache: 8.0.3 nanoid: 4.0.0 @@ -9040,7 +9046,7 @@ packages: mississippi: 3.0.0 mkdirp: 0.5.6 move-concurrently: 1.0.1 - promise-inflight: 1.0.1 + promise-inflight: 1.0.1_bluebird@3.7.2 rimraf: 2.7.1 ssri: 6.0.2 unique-filename: 1.1.1 @@ -9064,7 +9070,7 @@ packages: minipass-pipeline: 1.2.4 mkdirp: 1.0.4 p-map: 4.0.0 - promise-inflight: 1.0.1 + promise-inflight: 1.0.1_bluebird@3.7.2 rimraf: 3.0.2 ssri: 8.0.1 tar: 6.1.13 @@ -14284,8 +14290,8 @@ packages: lodash.clonedeep: 4.5.0 dev: true - /knex-schema-inspector/3.0.0: - resolution: {integrity: sha512-J3Aeh4kIZD2sYUjnpak+sdVtegGVkkhCF5/VSziocSZpseW4MwTcnxHbttsaT3tA85INuPxxUBxFgyCm9tMZDQ==} + /knex-schema-inspector/3.0.1: + resolution: {integrity: sha512-ofglN/HoHYhNxQJbuuKEj0vtVsBpEVlbvfoFd9lPn1ABzJHdwZmKLquCYbVLTXZIN0O6y6DDEuN4EqBKMhH+xQ==} dependencies: lodash.flatten: 4.4.0 lodash.isnil: 4.0.0 @@ -17120,13 +17126,15 @@ packages: engines: {node: '>=0.4.0'} dev: true - /promise-inflight/1.0.1: + /promise-inflight/1.0.1_bluebird@3.7.2: resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} peerDependencies: bluebird: '*' peerDependenciesMeta: bluebird: optional: true + dependencies: + bluebird: 3.7.2 /promise-retry/2.0.1: resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==} diff --git a/tests-blackbox/common/config.ts b/tests-blackbox/common/config.ts index 16acab0b09..73fd110001 100644 --- a/tests-blackbox/common/config.ts +++ b/tests-blackbox/common/config.ts @@ -56,6 +56,7 @@ const directusConfig = { LOG_LEVEL: logLevel, SERVE_APP: 'false', DB_EXCLUDE_TABLES: 'knex_migrations,knex_migrations_lock,spatial_ref_sys,sysdiagrams', + MAX_PAYLOAD_SIZE: '10mb', EXTENSIONS_PATH: './tests-blackbox/extensions', ...directusAuthConfig, }; diff --git a/tests-blackbox/common/functions.ts b/tests-blackbox/common/functions.ts index d51242ea60..15dbb499df 100644 --- a/tests-blackbox/common/functions.ts +++ b/tests-blackbox/common/functions.ts @@ -442,6 +442,8 @@ export async function CreateFieldM2M(vendor: string, options: OptionsCreateField schema: options.fieldSchema, }; + const isSelfReferencing = options.collection === options.otherCollection; + if (!fieldOptions.meta.special) { fieldOptions.meta.special = ['m2m']; } else if (!fieldOptions.meta.special.includes('m2m')) { @@ -483,7 +485,7 @@ export async function CreateFieldM2M(vendor: string, options: OptionsCreateField const junctionField = await CreateField(vendor, junctionFieldOptions); - const otherJunctionFieldName = `${options.otherCollection}_id`; + const otherJunctionFieldName = `${options.otherCollection}_id${isSelfReferencing ? '2' : ''}`; const otherJunctionFieldOptions: OptionsCreateField = { collection: options.junctionCollection, field: otherJunctionFieldName, diff --git a/tests-blackbox/common/seed-functions.ts b/tests-blackbox/common/seed-functions.ts index 0ec039205a..819bbb69dc 100644 --- a/tests-blackbox/common/seed-functions.ts +++ b/tests-blackbox/common/seed-functions.ts @@ -7,7 +7,9 @@ const FIVE_YEARS_IN_MILLISECONDS = 5 * 365 * 24 * 60 * 60 * 1000; type OptionsSeedGenerateBase = { quantity: number; - seed?: string | undefined; + seed?: string; + vendor?: string; + isDefaultValue?: boolean; }; export type OptionsSeedGeneratePrimaryKeys = OptionsSeedGenerateBase & OptionsSeedGenerateInteger; @@ -262,6 +264,18 @@ function generateDate(options: OptionsSeedGenerateDate) { } } + if (options.isDefaultValue && options.vendor === 'oracle') { + for (let i = 0; i < values.length; i++) { + values[i] = new Date(values[i]) + .toLocaleDateString('en-GB', { + day: 'numeric', + month: 'short', + year: 'numeric', + }) + .replace(/ /g, '-'); + } + } + return values; } @@ -272,6 +286,12 @@ function generateDateTime(options: OptionsSeedGenerateDateTime) { values[i] = values[i].slice(0, -5); } + if (options.isDefaultValue && options.vendor === 'oracle') { + for (let index = 0; index < values.length; index++) { + values[index] = 'CURRENT_TIMESTAMP'; + } + } + return values; } @@ -315,6 +335,12 @@ function generateTime(options: OptionsSeedGenerateTime) { } } + if (options.isDefaultValue && options.vendor === 'oracle') { + for (let index = 0; index < values.length; index++) { + values[index] = 'CURRENT_TIMESTAMP'; + } + } + return values; } @@ -354,5 +380,17 @@ function generateTimestamp(options: OptionsSeedGenerateTimestamp) { values[index] = values[index].slice(0, 20) + '000Z'; } + if (options.isDefaultValue && options.vendor) { + if (['mysql', 'mysql5', 'maria'].includes(options.vendor)) { + for (let index = 0; index < values.length; index++) { + values[index] = new Date(values[index]).toISOString().replace(/([^T]+)T([^.]+).*/g, '$1 $2'); + } + } else if (options.vendor === 'oracle') { + for (let index = 0; index < values.length; index++) { + values[index] = 'CURRENT_TIMESTAMP'; + } + } + } + return values; } diff --git a/tests-blackbox/routes/items/seed-all-field-types.ts b/tests-blackbox/routes/items/seed-all-field-types.ts index b34f0d4231..ff276b8b49 100644 --- a/tests-blackbox/routes/items/seed-all-field-types.ts +++ b/tests-blackbox/routes/items/seed-all-field-types.ts @@ -21,16 +21,38 @@ export function getTestsAllTypesSchema(): TestsFieldSchema { return fieldSchema; } -export const seedAllFieldTypesStructure = async (vendor: string, collection: string) => { +export const seedAllFieldTypesStructure = async (vendor: string, collection: string, setDefaultValues = false) => { try { const fieldSchema = getTestsAllTypesSchema(); // Create fields for (const key of Object.keys(fieldSchema)) { + let meta = {}; + let schema = {}; + + const fieldType = fieldSchema[key].type; + + if (fieldType === 'uuid') { + meta = { special: ['uuid'] }; + } + + if (setDefaultValues && SeedFunctions.generateValues[fieldType as keyof typeof SeedFunctions.generateValues]) { + schema = { + default_value: SeedFunctions.generateValues[fieldType as keyof typeof SeedFunctions.generateValues]({ + quantity: 1, + seed: `${collection}_${fieldType}`, + vendor, + isDefaultValue: true, + })[0], + }; + } + await CreateField(vendor, { collection: collection, field: fieldSchema[key].field.toLowerCase(), - type: fieldSchema[key].type, + type: fieldType, + meta, + schema, }); } diff --git a/tests-blackbox/routes/schema/schema.seed.ts b/tests-blackbox/routes/schema/schema.seed.ts new file mode 100644 index 0000000000..6389d89ece --- /dev/null +++ b/tests-blackbox/routes/schema/schema.seed.ts @@ -0,0 +1,250 @@ +import vendors from '@common/get-dbs-to-test'; +import { + CreateCollection, + CreateField, + DeleteCollection, + PRIMARY_KEY_TYPES, + CreateFieldM2M, + CreateFieldM2O, + CreateFieldO2M, + DeleteField, + PrimaryKeyType, +} from '@common/index'; +import { seedAllFieldTypesStructure } from '../items/seed-all-field-types'; + +export const collectionAll = 'test_schema_all'; +export const collectionM2O = 'test_schema_m2o'; +export const collectionM2O2 = 'test_schema_m2o2'; +export const collectionO2M = 'test_schema_o2m'; +export const collectionO2M2 = 'test_schema_o2m2'; +export const collectionM2M = 'test_schema_m2m'; +export const collectionM2M2 = 'test_schema_m2m2'; +export const junctionM2M = 'test_schema_jm2m'; +export const junctionM2M2 = 'test_schema_jm2m2'; +export const collectionSelf = 'test_schema_self'; +export const junctionSelfM2M = 'test_schema_jm2m_self'; +export const tempTestCollection = 'temp_test_collection'; + +export type SampleItem = { + id?: number | string; + name: string; +}; + +export type Ingredient = { + id?: number | string; + name: string; +}; + +export type Supplier = { + id?: number | string; + name: string; +}; + +export const deleteAllCollections = async (vendor: string, pkType: PrimaryKeyType, setDefaultValues: boolean) => { + const suffix = setDefaultValues ? '2' : ''; + + // Setup + const localCollectionAll = `${collectionAll}_${pkType}${suffix}`; + const localCollectionM2M = `${collectionM2M}_${pkType}${suffix}`; + const localCollectionM2M2 = `${collectionM2M2}_${pkType}${suffix}`; + const localJunctionAllM2M = `${junctionM2M}_${pkType}${suffix}`; + const localJunctionM2MM2M2 = `${junctionM2M2}_${pkType}${suffix}`; + const localCollectionM2O = `${collectionM2O}_${pkType}${suffix}`; + const localCollectionM2O2 = `${collectionM2O2}_${pkType}${suffix}`; + const localCollectionO2M = `${collectionO2M}_${pkType}${suffix}`; + const localCollectionO2M2 = `${collectionO2M2}_${pkType}${suffix}`; + const localCollectionSelf = `${collectionSelf}_${pkType}${suffix}`; + const localJunctionSelfM2M = `${junctionSelfM2M}_${pkType}${suffix}`; + + // Delete existing collections + await DeleteField(vendor, { collection: localCollectionSelf, field: 'self_id' }); + await DeleteField(vendor, { collection: localCollectionSelf, field: 'self_id' }); + await DeleteCollection(vendor, { collection: localJunctionSelfM2M }); + await DeleteCollection(vendor, { collection: localCollectionSelf }); + await DeleteCollection(vendor, { collection: localCollectionO2M2 }); + await DeleteCollection(vendor, { collection: localCollectionO2M }); + await DeleteCollection(vendor, { collection: localJunctionM2MM2M2 }); + await DeleteCollection(vendor, { collection: localJunctionAllM2M }); + await DeleteCollection(vendor, { collection: localCollectionM2M2 }); + await DeleteCollection(vendor, { collection: localCollectionM2M }); + await DeleteCollection(vendor, { collection: localCollectionAll }); + await DeleteCollection(vendor, { collection: localCollectionM2O }); + await DeleteCollection(vendor, { collection: localCollectionM2O2 }); +}; + +export const seedDBStructure = () => { + it.each(vendors)( + '%s', + async (vendor) => { + for (const setDefaultValues of [false, true]) { + const suffix = setDefaultValues ? '2' : ''; + + for (const pkType of PRIMARY_KEY_TYPES) { + try { + const localCollectionAll = `${collectionAll}_${pkType}${suffix}`; + const localCollectionM2M = `${collectionM2M}_${pkType}${suffix}`; + const localCollectionM2M2 = `${collectionM2M2}_${pkType}${suffix}`; + const localJunctionAllM2M = `${junctionM2M}_${pkType}${suffix}`; + const localJunctionM2MM2M2 = `${junctionM2M2}_${pkType}${suffix}`; + const localCollectionM2O = `${collectionM2O}_${pkType}${suffix}`; + const localCollectionM2O2 = `${collectionM2O2}_${pkType}${suffix}`; + const localCollectionO2M = `${collectionO2M}_${pkType}${suffix}`; + const localCollectionO2M2 = `${collectionO2M2}_${pkType}${suffix}`; + const localCollectionSelf = `${collectionSelf}_${pkType}${suffix}`; + const localJunctionSelfM2M = `${junctionSelfM2M}_${pkType}${suffix}`; + + // Delete existing collections + await deleteAllCollections(vendor, pkType, setDefaultValues); + + // Delete the temp collection created in previous test run + await DeleteCollection(vendor, { collection: tempTestCollection }); + + // Create All collection + await CreateCollection(vendor, { + collection: localCollectionAll, + primaryKeyType: pkType, + }); + + await CreateField(vendor, { + collection: localCollectionAll, + field: 'name', + type: 'string', + }); + + // Create M2M collection + await CreateCollection(vendor, { + collection: localCollectionM2M, + primaryKeyType: pkType, + }); + + // Create nested M2M collection + await CreateCollection(vendor, { + collection: localCollectionM2M2, + primaryKeyType: pkType, + }); + + // Create M2M relationships + await CreateFieldM2M(vendor, { + collection: localCollectionAll, + field: 'all_m2m', + otherCollection: localCollectionM2M, + otherField: 'm2m_all', + junctionCollection: localJunctionAllM2M, + primaryKeyType: pkType, + }); + + await CreateFieldM2M(vendor, { + collection: localCollectionM2M, + field: 'm2m_m2m2', + otherCollection: localCollectionM2M2, + otherField: 'm2m2_m2m', + junctionCollection: localJunctionM2MM2M2, + primaryKeyType: pkType, + }); + + // Create M2O collection + await CreateCollection(vendor, { + collection: localCollectionM2O, + primaryKeyType: pkType, + }); + + // Create nested M2O collection + await CreateCollection(vendor, { + collection: localCollectionM2O2, + primaryKeyType: pkType, + }); + + // Create M2O relationships + await CreateFieldM2O(vendor, { + collection: localCollectionAll, + field: 'all_id', + primaryKeyType: pkType, + otherCollection: localCollectionM2O, + }); + + await CreateFieldM2O(vendor, { + collection: localCollectionM2O, + field: 'm2o_id', + primaryKeyType: pkType, + otherCollection: localCollectionM2O2, + }); + + // Create O2M collection + await CreateCollection(vendor, { + collection: localCollectionO2M, + primaryKeyType: pkType, + }); + + // Create nested O2M collection + await CreateCollection(vendor, { + collection: localCollectionO2M2, + primaryKeyType: pkType, + }); + + // Create O2M relationships + await CreateFieldO2M(vendor, { + collection: localCollectionAll, + field: 'o2m', + otherCollection: localCollectionO2M, + otherField: 'all_id', + primaryKeyType: pkType, + }); + + await CreateFieldO2M(vendor, { + collection: localCollectionO2M, + field: 'o2m2', + otherCollection: localCollectionO2M2, + otherField: 'o2m_id', + primaryKeyType: pkType, + }); + + // Create Self collection + await CreateCollection(vendor, { + collection: localCollectionSelf, + primaryKeyType: pkType, + }); + + // Create self M2M relationship + await CreateFieldM2M(vendor, { + collection: localCollectionSelf, + field: 'm2m', + otherCollection: localCollectionSelf, + otherField: 'm2m2', + junctionCollection: localJunctionSelfM2M, + primaryKeyType: pkType, + relationSchema: { + on_delete: 'SET NULL', + }, + otherRelationSchema: + vendor === 'mssql' + ? { on_delete: 'NO ACTION' } + : { + on_delete: 'SET NULL', + }, + }); + + // Create self O2M relationship + await CreateFieldO2M(vendor, { + collection: localCollectionSelf, + field: 'o2m', + otherCollection: localCollectionSelf, + otherField: 'self_id', + primaryKeyType: pkType, + relationSchema: { + on_delete: 'NO ACTION', + }, + }); + + await seedAllFieldTypesStructure(vendor, localCollectionAll, setDefaultValues); + await seedAllFieldTypesStructure(vendor, localCollectionSelf, setDefaultValues); + + expect(true).toBeTruthy(); + } catch (error) { + expect(error).toBeFalsy(); + } + } + } + }, + 1800000 + ); +}; diff --git a/tests-blackbox/routes/schema/schema.test.ts b/tests-blackbox/routes/schema/schema.test.ts new file mode 100644 index 0000000000..dfe24c0b5c --- /dev/null +++ b/tests-blackbox/routes/schema/schema.test.ts @@ -0,0 +1,559 @@ +import request from 'supertest'; +import { getUrl } from '@common/config'; +import vendors from '@common/get-dbs-to-test'; +import * as common from '@common/index'; +import { + collectionAll, + collectionM2M, + collectionM2M2, + collectionM2O, + collectionM2O2, + collectionO2M, + collectionO2M2, + collectionSelf, + deleteAllCollections, + junctionM2M, + junctionM2M2, + junctionSelfM2M, + tempTestCollection, +} from './schema.seed'; +import { cloneDeep } from 'lodash'; +import { PrimaryKeyType, PRIMARY_KEY_TYPES } from '@common/index'; +import { load as loadYaml } from 'js-yaml'; +import { version as currentDirectusVersion } from '../../../api/package.json'; + +describe('Schema Snapshots', () => { + const snapshotsCacheOriginal: { + [vendor: string]: any; + } = {}; + + const snapshotsCacheOriginalYaml: { + [vendor: string]: any; + } = {}; + + const snapshotsCacheEmpty: { + [vendor: string]: any; + } = {}; + + describe('GET /schema/snapshot', () => { + common.DisableTestCachingSetup(); + + describe('denies non-admin users', () => { + it.each(vendors)('%s', async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.APP_ACCESS.TOKEN}`); + const response2 = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.API_ONLY.TOKEN}`); + const response3 = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.NO_ROLE.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(403); + expect(response2.statusCode).toEqual(403); + expect(response3.statusCode).toEqual(403); + }); + }); + + describe('retrieves a snapshot (JSON)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(200); + + snapshotsCacheOriginal[vendor] = response.body.data; + }, + 300000 + ); + }); + + describe('retrieves a snapshot (YAML)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .query({ export: 'yaml' }) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(200); + + snapshotsCacheOriginalYaml[vendor] = response.text; + }, + 300000 + ); + }); + + describe('remove tables', () => { + describe.each(common.PRIMARY_KEY_TYPES)('%s primary keys', (pkType) => { + it.each(vendors)( + '%s', + async (vendor) => { + for (const setDefaultValues of [false, true]) { + // Delete existing collections + await deleteAllCollections(vendor, pkType, setDefaultValues); + } + + await assertCollectionsDeleted(vendor, pkType); + }, + 300000 + ); + }); + }); + }); + + common.ClearCaches(); + + describe('retrieves empty snapshot', () => { + it.each(vendors)( + '%s', + async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(200); + expect(snapshotsCacheEmpty[vendor]).not.toEqual(snapshotsCacheOriginal[vendor]); + + snapshotsCacheEmpty[vendor] = response.body.data; + }, + 300000 + ); + }); + + describe('POST /schema/diff', () => { + describe('denies non-admin users', () => { + it.each(vendors)('%s', async (vendor) => { + // Action + const currentVendor = vendor.replace(/[0-9]/g, ''); + const response = await request(getUrl(vendor)) + .post('/schema/diff') + .send({ + version: 1, + directus: currentDirectusVersion, + vendor: currentVendor, + collections: [], + fields: [], + relations: [], + }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.APP_ACCESS.TOKEN}`); + const response2 = await request(getUrl(vendor)) + .post('/schema/diff') + .send({ + version: 1, + directus: currentDirectusVersion, + vendor: currentVendor, + collections: [], + fields: [], + relations: [], + }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.API_ONLY.TOKEN}`); + const response3 = await request(getUrl(vendor)) + .post('/schema/diff') + .send({ + version: 1, + directus: currentDirectusVersion, + vendor: currentVendor, + collections: [], + fields: [], + relations: [], + }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.NO_ROLE.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(403); + expect(response2.statusCode).toEqual(403); + expect(response3.statusCode).toEqual(403); + }); + }); + + describe('returns diffs with empty snapshot', () => { + it.each(vendors)( + '%s', + async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheEmpty[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + expect(response.statusCode).toEqual(204); + }, + 300000 + ); + }); + + describe('returns diffs with original snapshot', () => { + it.each(vendors)( + '%s', + async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheOriginal[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(200); + expect(response.body.data?.diff?.collections?.length).toBe(66); + expect(response.body.data?.diff?.fields?.length).toBe(342); + expect(response.body.data?.diff?.relations?.length).toBe(66); + }, + 300000 + ); + }); + }); + + describe('POST /schema/apply', () => { + describe('denies non-admin users', () => { + it.each(vendors)('%s', async (vendor) => { + // Action + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send({ data: true }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.APP_ACCESS.TOKEN}`); + const response2 = await request(getUrl(vendor)) + .post('/schema/apply') + .send({ data: true }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.API_ONLY.TOKEN}`); + const response3 = await request(getUrl(vendor)) + .post('/schema/apply') + .send({ data: true }) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.NO_ROLE.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(403); + expect(response2.statusCode).toEqual(403); + expect(response3.statusCode).toEqual(403); + }); + }); + + describe('applies a snapshot (JSON)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheOriginal[vendor]).toBeDefined(); + + // Action + const responseDiff = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheOriginal[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send(responseDiff.body.data) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(204); + }, + 1200000 + ); + }); + + describe('retrieves the same snapshot after applying a snapshot (JSON)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheOriginal[vendor]).toBeDefined(); + + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const curSnapshot = cloneDeep(response.body.data); + const oldSnapshot = cloneDeep(snapshotsCacheOriginal[vendor]); + + parseSnapshot(vendor, curSnapshot); + parseSnapshot(vendor, oldSnapshot); + + // Assert + expect(response.statusCode).toEqual(200); + expect(curSnapshot).toStrictEqual(oldSnapshot); + }, + 300000 + ); + }); + + describe('applies empty snapshot', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheEmpty[vendor]).toBeDefined(); + + // Action + const responseDiff = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheEmpty[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send(responseDiff.body.data) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(204); + }, + 1200000 + ); + }); + + describe('ensure that tables are removed', () => { + describe.each(common.PRIMARY_KEY_TYPES)('%s primary keys', (pkType) => { + it.each(vendors)( + '%s', + async (vendor) => { + await assertCollectionsDeleted(vendor, pkType); + }, + 600000 + ); + }); + }); + + describe('applies a snapshot (YAML)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheOriginalYaml[vendor]).toBeDefined(); + + // Action + const responseDiff = await request(getUrl(vendor)) + .post('/schema/diff') + .attach('file', Buffer.from(snapshotsCacheOriginalYaml[vendor])) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send(responseDiff.body.data) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(204); + }, + 1200000 + ); + }); + + describe('retrieves the same snapshot after applying a snapshot (YAML)', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheOriginalYaml[vendor]).toBeDefined(); + + // Action + const response = await request(getUrl(vendor)) + .get('/schema/snapshot') + .query({ export: 'yaml' }) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const curSnapshot = await loadYaml(response.text); + const oldSnapshot = cloneDeep(snapshotsCacheOriginal[vendor]); + + parseSnapshot(vendor, curSnapshot); + parseSnapshot(vendor, oldSnapshot); + + // Assert + expect(response.statusCode).toEqual(200); + expect(curSnapshot).toStrictEqual(oldSnapshot); + }, + 300000 + ); + }); + }); + + common.ClearCaches(); + + describe('Hash Tests', () => { + describe('with deleted fields', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheEmpty[vendor]).toBeDefined(); + + // Action + const responseDiff = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheEmpty[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + for (const pkType of PRIMARY_KEY_TYPES) { + await request(getUrl(vendor)) + .delete(`/fields/${collectionSelf}_${pkType}/self_id`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + } + + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send(responseDiff.body.data) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(400); + expect(response.text).toContain('Please generate a new diff and try again.'); + }, + 1200000 + ); + }); + + describe('with new collection', () => { + it.each(vendors)( + '%s', + async (vendor) => { + expect(snapshotsCacheEmpty[vendor]).toBeDefined(); + + // Action + const responseDiff = await request(getUrl(vendor)) + .post('/schema/diff') + .send(snapshotsCacheEmpty[vendor]) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + await request(getUrl(vendor)) + .post(`/collections`) + .send({ + collection: tempTestCollection, + fields: [ + { + field: 'id', + type: 'integer', + meta: { hidden: true, interface: 'input', readonly: true }, + schema: { is_primary_key: true, has_auto_increment: true }, + }, + ], + schema: {}, + meta: { singleton: false }, + }) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + const response = await request(getUrl(vendor)) + .post('/schema/apply') + .send(responseDiff.body.data) + .set('Content-type', 'application/json') + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(400); + expect(response.text).toContain('Please generate a new diff and try again.'); + }, + 1200000 + ); + }); + }); +}); + +function parseSnapshot(vendor: string, snapshot: any) { + if (vendor === 'cockroachdb') { + if (snapshot.fields) { + for (const field of snapshot.fields) { + if ( + field.schema?.default_value && + ['integer', 'bigInteger'].includes(field.type) && + typeof field.schema.default_value === 'string' && + field.schema.default_value.startsWith('nextval(') + ) { + field.schema.default_value = field.schema.default_value.replace( + /(_seq)\d*('::STRING::REGCLASS\))/, + `_seq'::STRING::REGCLASS)` + ); + } + } + } + } +} + +async function assertCollectionsDeleted(vendor: string, pkType: PrimaryKeyType) { + for (const setDefaultValues of [false, true]) { + const suffix = setDefaultValues ? '2' : ''; + + // Setup + const localCollectionAll = `${collectionAll}_${pkType}${suffix}`; + const localCollectionM2M = `${collectionM2M}_${pkType}${suffix}`; + const localCollectionM2M2 = `${collectionM2M2}_${pkType}${suffix}`; + const localJunctionAllM2M = `${junctionM2M}_${pkType}${suffix}`; + const localJunctionM2MM2M2 = `${junctionM2M2}_${pkType}${suffix}`; + const localCollectionM2O = `${collectionM2O}_${pkType}${suffix}`; + const localCollectionM2O2 = `${collectionM2O2}_${pkType}${suffix}`; + const localCollectionO2M = `${collectionO2M}_${pkType}${suffix}`; + const localCollectionO2M2 = `${collectionO2M2}_${pkType}${suffix}`; + const localCollectionSelf = `${collectionSelf}_${pkType}${suffix}`; + const localJunctionSelfM2M = `${junctionSelfM2M}_${pkType}${suffix}`; + + const response = await request(getUrl(vendor)) + .get(`/items/${localJunctionSelfM2M}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response2 = await request(getUrl(vendor)) + .get(`/items/${localCollectionSelf}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response3 = await request(getUrl(vendor)) + .get(`/items/${localCollectionO2M2}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response4 = await request(getUrl(vendor)) + .get(`/items/${localCollectionO2M}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response5 = await request(getUrl(vendor)) + .get(`/items/${localJunctionM2MM2M2}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response6 = await request(getUrl(vendor)) + .get(`/items/${localJunctionAllM2M}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response7 = await request(getUrl(vendor)) + .get(`/items/${localCollectionM2M2}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response8 = await request(getUrl(vendor)) + .get(`/items/${localCollectionM2M}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response9 = await request(getUrl(vendor)) + .get(`/items/${localCollectionAll}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response10 = await request(getUrl(vendor)) + .get(`/items/${localCollectionM2O}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + const response11 = await request(getUrl(vendor)) + .get(`/items/${localCollectionM2O2}`) + .set('Authorization', `Bearer ${common.USER.ADMIN.TOKEN}`); + + // Assert + expect(response.statusCode).toEqual(403); + expect(response2.statusCode).toEqual(403); + expect(response3.statusCode).toEqual(403); + expect(response4.statusCode).toEqual(403); + expect(response5.statusCode).toEqual(403); + expect(response6.statusCode).toEqual(403); + expect(response7.statusCode).toEqual(403); + expect(response8.statusCode).toEqual(403); + expect(response9.statusCode).toEqual(403); + expect(response10.statusCode).toEqual(403); + expect(response11.statusCode).toEqual(403); + } +} diff --git a/tests-blackbox/setup/sequentialTests.js b/tests-blackbox/setup/sequentialTests.js index f7b6a0b95c..abcd433f23 100644 --- a/tests-blackbox/setup/sequentialTests.js +++ b/tests-blackbox/setup/sequentialTests.js @@ -3,6 +3,7 @@ exports.list = { before: [ { testFilePath: '/common/seed-database.test.ts' }, { testFilePath: '/common/common.test.ts' }, + { testFilePath: '/routes/schema/schema.test.ts' }, { testFilePath: '/routes/collections/crud.test.ts' }, { testFilePath: '/routes/fields/change-fields.test.ts' }, ],