diff --git a/.changeset/great-experts-clap.md b/.changeset/great-experts-clap.md new file mode 100644 index 0000000000..b94bd0f9aa --- /dev/null +++ b/.changeset/great-experts-clap.md @@ -0,0 +1,20 @@ +--- +"@directus/api": major +"@directus/types": patch +--- + +Fixed `Content Versioning` to correctly merge relational data and support all query parameter functionality + +::: notice + +The following changes should be kept in mind when updating: +1. Relational versioned data now requires explicit field expansion to be included in the response. +2. Invalid data (e.g. Fails validation rules) will error on query +3. Filter conditions now apply to the versioned data instead of the main record +4. +For more information, please read the [breaking change docs](https://directus.io/docs/releases/breaking-changes/version-11#version-11110) for a full list of changes. + +Additionally there will be further breaking changes to `USER_CREATED`, `USER_UPDATED`, `DATE_CREATED`, `DATE_UPDATED` default values in a followup PR as the current behavior is not fully desired. +Check in with https://github.com/directus/directus/pull/25744 to see more info about the breaking changes. + +::: diff --git a/api/src/constants.ts b/api/src/constants.ts index ac8dd852f8..8d081afe3a 100644 --- a/api/src/constants.ts +++ b/api/src/constants.ts @@ -61,7 +61,15 @@ export const DEFAULT_AUTH_PROVIDER = 'default'; export const COLUMN_TRANSFORMS = ['year', 'month', 'day', 'weekday', 'hour', 'minute', 'second']; -export const GENERATE_SPECIAL = ['uuid', 'date-created', 'role-created', 'user-created'] as const; +export const GENERATE_SPECIAL = [ + 'uuid', + 'date-created', + 'date-updated', + 'role-created', + 'role-updated', + 'user-created', + 'user-updated', +] as const; export const UUID_REGEX = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'; diff --git a/api/src/controllers/items.ts b/api/src/controllers/items.ts index 0083d01f07..9d97ae63d4 100644 --- a/api/src/controllers/items.ts +++ b/api/src/controllers/items.ts @@ -3,7 +3,6 @@ import { isSystemCollection } from '@directus/system-data'; import type { PrimaryKey } from '@directus/types'; import express from 'express'; import collectionExists from '../middleware/collection-exists.js'; -import { mergeContentVersions } from '../middleware/merge-content-versions.js'; import { respond } from '../middleware/respond.js'; import { validateBatch } from '../middleware/validate-batch.js'; import { ItemsService } from '../services/items.js'; @@ -92,8 +91,8 @@ const readHandler = asyncHandler(async (req, res, next) => { return next(); }); -router.search('/:collection', collectionExists, validateBatch('read'), readHandler, mergeContentVersions, respond); -router.get('/:collection', collectionExists, readHandler, mergeContentVersions, respond); +router.search('/:collection', collectionExists, validateBatch('read'), readHandler, respond); +router.get('/:collection', collectionExists, readHandler, respond); router.get( '/:collection/:pk', @@ -114,7 +113,6 @@ router.get( return next(); }), - mergeContentVersions, respond, ); diff --git a/api/src/middleware/merge-content-versions.ts b/api/src/middleware/merge-content-versions.ts deleted file mode 100644 index b85a62e37f..0000000000 --- a/api/src/middleware/merge-content-versions.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { isObject } from '@directus/utils'; -import type { RequestHandler } from 'express'; -import { VersionsService } from '../services/versions.js'; -import asyncHandler from '../utils/async-handler.js'; -import { mergeVersionsRaw, mergeVersionsRecursive } from '../utils/merge-version-data.js'; - -export const mergeContentVersions: RequestHandler = asyncHandler(async (req, res, next) => { - if ( - req.sanitizedQuery.version && - req.collection && - (req.singleton || req.params['pk']) && - 'data' in res.locals['payload'] - ) { - const originalData = res.locals['payload'].data as unknown; - - // only act on single item requests - if (!isObject(originalData)) return next(); - - const versionsService = new VersionsService({ accountability: req.accountability ?? null, schema: req.schema }); - - const versionData = await versionsService.getVersionSaves( - req.sanitizedQuery.version, - req.collection, - req.params['pk'], - ); - - if (!versionData || versionData.length === 0) return next(); - - if (req.sanitizedQuery.versionRaw) { - res.locals['payload'].data = mergeVersionsRaw(originalData, versionData); - } else { - res.locals['payload'].data = mergeVersionsRecursive(originalData, versionData, req.collection, req.schema); - } - } - - return next(); -}); diff --git a/api/src/services/graphql/index.ts b/api/src/services/graphql/index.ts index 4fc16e5920..2d8d90de8b 100644 --- a/api/src/services/graphql/index.ts +++ b/api/src/services/graphql/index.ts @@ -7,6 +7,7 @@ import type { Item, Query, SchemaOverview, + PrimaryKey, } from '@directus/types'; import type { ExecutionResult, FormattedExecutionResult, GraphQLSchema } from 'graphql'; import { NoSchemaIntrospectionCustomRule, execute, specifiedRules, validate } from 'graphql'; @@ -99,18 +100,19 @@ export class GraphQLService { /** * Execute the read action on the correct service. Checks for singleton as well. */ - async read(collection: string, query: Query): Promise> { + async read(collection: string, query: Query, id?: PrimaryKey): Promise> { const service = getService(collection, { knex: this.knex, accountability: this.accountability, schema: this.schema, }); - const result = this.schema.collections[collection]!.singleton - ? await service.readSingleton(query, { stripNonRequested: false }) - : await service.readByQuery(query, { stripNonRequested: false }); + if (this.schema.collections[collection]!.singleton) + return await service.readSingleton(query, { stripNonRequested: false }); - return result; + if (id) return await service.readOne(id, query, { stripNonRequested: false }); + + return await service.readByQuery(query, { stripNonRequested: false }); } /** diff --git a/api/src/services/graphql/resolvers/query.ts b/api/src/services/graphql/resolvers/query.ts index d0f86aa87b..2b226f1150 100644 --- a/api/src/services/graphql/resolvers/query.ts +++ b/api/src/services/graphql/resolvers/query.ts @@ -2,8 +2,6 @@ import type { Item, Query } from '@directus/types'; import { parseFilterFunctionPath } from '@directus/utils'; import type { GraphQLResolveInfo } from 'graphql'; import { omit } from 'lodash-es'; -import { mergeVersionsRaw, mergeVersionsRecursive } from '../../../utils/merge-version-data.js'; -import { VersionsService } from '../../versions.js'; import type { GraphQLService } from '../index.js'; import { parseArgs } from '../schema/parse-args.js'; import { getQuery } from '../schema/parse-query.js'; @@ -23,7 +21,6 @@ export async function resolveQuery(gql: GraphQLService, info: GraphQLResolveInfo const args: Record = parseArgs(info.fieldNodes[0]!.arguments || [], info.variableValues); let query: Query; - let versionRaw = false; const isAggregate = collection.endsWith('_aggregated') && collection in gql.schema.collections === false; @@ -39,25 +36,10 @@ export async function resolveQuery(gql: GraphQLService, info: GraphQLResolveInfo if (collection.endsWith('_by_version') && collection in gql.schema.collections === false) { collection = collection.slice(0, -11); - versionRaw = true; + query.versionRaw = true; } } - if (args['id']) { - query.filter = { - _and: [ - query.filter || {}, - { - [gql.schema.collections[collection]!.primary]: { - _eq: args['id'], - }, - }, - ], - }; - - query.limit = 1; - } - // Transform count(a.b.c) into a.b.count(c) if (query.fields?.length) { for (let fieldIndex = 0; fieldIndex < query.fields.length; fieldIndex++) { @@ -65,31 +47,9 @@ export async function resolveQuery(gql: GraphQLService, info: GraphQLResolveInfo } } - const result = await gql.read(collection, query); + const result = await gql.read(collection, query, args['id']); - if (args['version']) { - const versionsService = new VersionsService({ accountability: gql.accountability, schema: gql.schema }); - - const saves = await versionsService.getVersionSaves(args['version'], collection, args['id']); - - if (saves) { - if (gql.schema.collections[collection]!.singleton) { - return versionRaw - ? mergeVersionsRaw(result, saves) - : mergeVersionsRecursive(result, saves, collection, gql.schema); - } else { - if (result?.[0] === undefined) return null; - - return versionRaw - ? mergeVersionsRaw(result[0], saves) - : mergeVersionsRecursive(result[0], saves, collection, gql.schema); - } - } - } - - if (args['id']) { - return result?.[0] || null; - } + if (args['id']) return result; if (query.group) { // for every entry in result add a group field based on query.group; diff --git a/api/src/services/items.ts b/api/src/services/items.ts index db80f5faf2..de7eadf63d 100644 --- a/api/src/services/items.ts +++ b/api/src/services/items.ts @@ -33,6 +33,7 @@ import { shouldClearCache } from '../utils/should-clear-cache.js'; import { transaction } from '../utils/transaction.js'; import { validateKeys } from '../utils/validate-keys.js'; import { validateUserCountIntegrity } from '../utils/validate-user-count-integrity.js'; +import { handleVersion } from '../utils/versioning/handle-version.js'; import { PayloadService } from './payload.js'; const env = useEnv(); @@ -129,9 +130,6 @@ export class ItemsService opts?.bypassEmitAction ? opts.bypassEmitAction(params) : nestedActionEvents.push(params), emitEvents: opts?.emitEvents, + autoPurgeCache: opts?.autoPurgeCache, + autoPurgeSystemCache: opts?.autoPurgeSystemCache, + skipTracking: opts?.skipTracking, + onItemCreate: opts?.onItemCreate, mutationTracker: opts?.mutationTracker, }); } @@ -546,6 +550,10 @@ export class PayloadService { bypassEmitAction: (params) => opts?.bypassEmitAction ? opts.bypassEmitAction(params) : nestedActionEvents.push(params), emitEvents: opts?.emitEvents, + autoPurgeCache: opts?.autoPurgeCache, + autoPurgeSystemCache: opts?.autoPurgeSystemCache, + skipTracking: opts?.skipTracking, + onItemCreate: opts?.onItemCreate, mutationTracker: opts?.mutationTracker, }); } @@ -626,6 +634,10 @@ export class PayloadService { bypassEmitAction: (params) => opts?.bypassEmitAction ? opts.bypassEmitAction(params) : nestedActionEvents.push(params), emitEvents: opts?.emitEvents, + autoPurgeCache: opts?.autoPurgeCache, + autoPurgeSystemCache: opts?.autoPurgeSystemCache, + skipTracking: opts?.skipTracking, + onItemCreate: opts?.onItemCreate, mutationTracker: opts?.mutationTracker, }); } @@ -636,6 +648,10 @@ export class PayloadService { bypassEmitAction: (params) => opts?.bypassEmitAction ? opts.bypassEmitAction(params) : nestedActionEvents.push(params), emitEvents: opts?.emitEvents, + autoPurgeCache: opts?.autoPurgeCache, + autoPurgeSystemCache: opts?.autoPurgeSystemCache, + skipTracking: opts?.skipTracking, + onItemCreate: opts?.onItemCreate, mutationTracker: opts?.mutationTracker, }); } @@ -759,6 +775,10 @@ export class PayloadService { bypassEmitAction: (params) => opts?.bypassEmitAction ? opts.bypassEmitAction(params) : nestedActionEvents.push(params), emitEvents: opts?.emitEvents, + autoPurgeCache: opts?.autoPurgeCache, + autoPurgeSystemCache: opts?.autoPurgeSystemCache, + skipTracking: opts?.skipTracking, + onItemCreate: opts?.onItemCreate, mutationTracker: opts?.mutationTracker, })), ); @@ -789,6 +809,10 @@ export class PayloadService { bypassEmitAction: (params) => opts?.bypassEmitAction ? opts.bypassEmitAction(params) : nestedActionEvents.push(params), emitEvents: opts?.emitEvents, + autoPurgeCache: opts?.autoPurgeCache, + autoPurgeSystemCache: opts?.autoPurgeSystemCache, + skipTracking: opts?.skipTracking, + onItemCreate: opts?.onItemCreate, mutationTracker: opts?.mutationTracker, }); } else { @@ -801,6 +825,10 @@ export class PayloadService { bypassEmitAction: (params) => opts?.bypassEmitAction ? opts.bypassEmitAction(params) : nestedActionEvents.push(params), emitEvents: opts?.emitEvents, + autoPurgeCache: opts?.autoPurgeCache, + autoPurgeSystemCache: opts?.autoPurgeSystemCache, + skipTracking: opts?.skipTracking, + onItemCreate: opts?.onItemCreate, mutationTracker: opts?.mutationTracker, }, ); @@ -851,6 +879,10 @@ export class PayloadService { bypassEmitAction: (params) => opts?.bypassEmitAction ? opts.bypassEmitAction(params) : nestedActionEvents.push(params), emitEvents: opts?.emitEvents, + autoPurgeCache: opts?.autoPurgeCache, + autoPurgeSystemCache: opts?.autoPurgeSystemCache, + skipTracking: opts?.skipTracking, + onItemCreate: opts?.onItemCreate, mutationTracker: opts?.mutationTracker, }); } @@ -875,6 +907,10 @@ export class PayloadService { bypassEmitAction: (params) => opts?.bypassEmitAction ? opts.bypassEmitAction(params) : nestedActionEvents.push(params), emitEvents: opts?.emitEvents, + autoPurgeCache: opts?.autoPurgeCache, + autoPurgeSystemCache: opts?.autoPurgeSystemCache, + skipTracking: opts?.skipTracking, + onItemCreate: opts?.onItemCreate, mutationTracker: opts?.mutationTracker, }); } @@ -905,6 +941,10 @@ export class PayloadService { bypassEmitAction: (params) => opts?.bypassEmitAction ? opts.bypassEmitAction(params) : nestedActionEvents.push(params), emitEvents: opts?.emitEvents, + autoPurgeCache: opts?.autoPurgeCache, + autoPurgeSystemCache: opts?.autoPurgeSystemCache, + skipTracking: opts?.skipTracking, + onItemCreate: opts?.onItemCreate, mutationTracker: opts?.mutationTracker, }); } else { @@ -917,6 +957,10 @@ export class PayloadService { bypassEmitAction: (params) => opts?.bypassEmitAction ? opts.bypassEmitAction(params) : nestedActionEvents.push(params), emitEvents: opts?.emitEvents, + autoPurgeCache: opts?.autoPurgeCache, + autoPurgeSystemCache: opts?.autoPurgeSystemCache, + skipTracking: opts?.skipTracking, + onItemCreate: opts?.onItemCreate, mutationTracker: opts?.mutationTracker, }, ); diff --git a/api/src/utils/deep-map-response.test.ts b/api/src/utils/deep-map-response.test.ts new file mode 100644 index 0000000000..57acdf6a70 --- /dev/null +++ b/api/src/utils/deep-map-response.test.ts @@ -0,0 +1,505 @@ +import { expect, test } from 'vitest'; +import { deepMapResponse } from './deep-map-response.js'; +import { SchemaBuilder } from '@directus/schema-builder'; +import { getRelation } from '@directus/utils'; + +const schema = new SchemaBuilder() + .collection('articles', (c) => { + c.field('id').id(); + c.field('title').string(); + c.field('date').date(); + c.field('author').m2o('users'); + c.field('tags').m2m('tags'); + c.field('links').o2m('links', 'article_id'); + c.field('sections').m2a(['sec_num', 'sec_text']); + }) + .collection('users', (c) => { + c.field('id').id(); + c.field('name').string(); + }) + .collection('tags', (c) => { + c.field('id').id(); + c.field('tag').string(); + }) + .collection('links', (c) => { + c.field('id').id(); + c.field('name').string(); + }) + .collection('sec_num', (c) => { + c.field('id').id(); + c.field('num').integer(); + }) + .collection('sec_text', (c) => { + c.field('id').id(); + c.field('text').text(); + }) + .build(); + +test('map flat object', () => { + const result = deepMapResponse( + { + id: 1, + title: 2, + author: 3, + tags: [1, 2, 3], + }, + ([key, value], context) => { + return [key, { value, context }]; + }, + { schema: schema, collection: 'articles' }, + ); + + expect(result).toEqual({ + id: { + value: 1, + context: { + collection: schema.collections['articles'], + field: schema.collections['articles']!.fields['id'], + relation: null, + leaf: true, + relationType: null, + }, + }, + title: { + value: 2, + context: { + collection: schema.collections['articles'], + field: schema.collections['articles']!.fields['title'], + relation: null, + leaf: true, + relationType: null, + }, + }, + author: { + value: 3, + context: { + collection: schema.collections['articles'], + field: schema.collections['articles']!.fields['author'], + relation: getRelation(schema.relations, 'articles', 'author'), + leaf: true, + relationType: 'm2o', + }, + }, + tags: { + value: [1, 2, 3], + context: { + collection: schema.collections['articles'], + field: schema.collections['articles']!.fields['tags'], + relation: getRelation(schema.relations, 'articles', 'tags'), + leaf: true, + relationType: 'o2m', + }, + }, + }); +}); + +test('map m2o object', () => { + const result = deepMapResponse( + { + author: { + id: 1, + name: 'hello', + }, + }, + ([key, value], context) => { + return [key, { value, context }]; + }, + { schema: schema, collection: 'articles' }, + ); + + expect(result).toEqual({ + author: { + value: { + id: { + value: 1, + context: { + collection: schema.collections['users'], + field: schema.collections['users']!.fields['id'], + relation: null, + leaf: true, + relationType: null, + }, + }, + name: { + value: 'hello', + context: { + collection: schema.collections['users'], + field: schema.collections['users']!.fields['name'], + relation: null, + leaf: true, + relationType: null, + }, + }, + }, + context: { + collection: schema.collections['articles'], + field: schema.collections['articles']!.fields['author'], + relation: getRelation(schema.relations, 'articles', 'author'), + leaf: false, + relationType: 'm2o', + }, + }, + }); +}); + +test('map o2m object', () => { + const result = deepMapResponse( + { + links: [ + { + id: 1, + }, + { + name: 'hello', + }, + ], + }, + ([key, value], context) => { + return [key, { value, context }]; + }, + { schema: schema, collection: 'articles' }, + ); + + expect(result).toEqual({ + links: { + value: [ + { + id: { + value: 1, + context: { + collection: schema.collections['links'], + field: schema.collections['links']!.fields['id'], + relation: null, + leaf: true, + relationType: null, + }, + }, + }, + { + name: { + value: 'hello', + context: { + collection: schema.collections['links'], + field: schema.collections['links']!.fields['name'], + relation: null, + leaf: true, + relationType: null, + }, + }, + }, + ], + context: { + collection: schema.collections['articles'], + field: schema.collections['articles']!.fields['links'], + relation: getRelation(schema.relations, 'articles', 'links'), + leaf: false, + relationType: 'o2m', + }, + }, + }); +}); + +test('map m2m object', () => { + const result = deepMapResponse( + { + tags: [ + { + id: 1, + articles_id: 2, + tags_id: { + tag: 'myTag', + }, + }, + ], + }, + ([key, value], context) => { + return [key, { value, context }]; + }, + { schema: schema, collection: 'articles' }, + ); + + expect(result).toEqual({ + tags: { + value: [ + { + id: { + value: 1, + context: { + collection: schema.collections['articles_tags_junction'], + field: schema.collections['articles_tags_junction']!.fields['id'], + relation: null, + leaf: true, + relationType: null, + }, + }, + articles_id: { + value: 2, + context: { + collection: schema.collections['articles_tags_junction'], + field: schema.collections['articles_tags_junction']!.fields['articles_id'], + relation: getRelation(schema.relations, 'articles_tags_junction', 'articles_id'), + leaf: true, + relationType: 'm2o', + }, + }, + tags_id: { + value: { + tag: { + value: 'myTag', + context: { + collection: schema.collections['tags'], + field: schema.collections['tags']!.fields['tag'], + relation: null, + leaf: true, + relationType: null, + }, + }, + }, + context: { + collection: schema.collections['articles_tags_junction'], + field: schema.collections['articles_tags_junction']!.fields['tags_id'], + relation: getRelation(schema.relations, 'articles_tags_junction', 'tags_id'), + leaf: false, + relationType: 'm2o', + }, + }, + }, + ], + context: { + collection: schema.collections['articles'], + field: schema.collections['articles']!.fields['tags'], + relation: getRelation(schema.relations, 'articles', 'tags'), + leaf: false, + relationType: 'o2m', + }, + }, + }); +}); + +test('map m2a object', () => { + const result = deepMapResponse( + { + sections: [ + { + collection: 'sec_num', + item: { + num: 123, + }, + }, + { + collection: 'sec_text', + item: { + text: 'abc', + }, + }, + ], + }, + ([key, value], context) => { + return [key, { value, context }]; + }, + { schema: schema, collection: 'articles' }, + ); + + expect(result).toEqual({ + sections: { + value: [ + { + collection: { + value: 'sec_num', + context: { + collection: schema.collections['articles_builder'], + field: schema.collections['articles_builder']!.fields['collection'], + relation: null, + leaf: true, + relationType: null, + }, + }, + item: { + value: { + num: { + value: 123, + context: { + collection: schema.collections['sec_num'], + field: schema.collections['sec_num']!.fields['num'], + relation: null, + leaf: true, + relationType: null, + }, + }, + }, + context: { + collection: schema.collections['articles_builder'], + field: schema.collections['articles_builder']!.fields['item'], + relation: getRelation(schema.relations, 'articles_builder', 'item'), + leaf: false, + relationType: 'a2o', + }, + }, + }, + { + collection: { + value: 'sec_text', + context: { + collection: schema.collections['articles_builder'], + field: schema.collections['articles_builder']!.fields['collection'], + relation: null, + leaf: true, + relationType: null, + }, + }, + item: { + value: { + text: { + value: 'abc', + context: { + collection: schema.collections['sec_text'], + field: schema.collections['sec_text']!.fields['text'], + relation: null, + leaf: true, + relationType: null, + }, + }, + }, + context: { + collection: schema.collections['articles_builder'], + field: schema.collections['articles_builder']!.fields['item'], + relation: getRelation(schema.relations, 'articles_builder', 'item'), + leaf: false, + relationType: 'a2o', + }, + }, + }, + ], + context: { + collection: schema.collections['articles'], + field: schema.collections['articles']!.fields['sections'], + relation: getRelation(schema.relations, 'articles', 'sections'), + leaf: false, + relationType: 'o2m', + }, + }, + }); +}); + +test('map flat invalid field', () => { + const result = deepMapResponse( + { + invalid: 1, + }, + ([key, value], context) => { + return [key, { value, context }]; + }, + { schema: schema, collection: 'articles' }, + ); + + expect(result).toEqual({ + invalid: 1, + }); +}); + +test('map with invalid object', () => { + expect(() => { + deepMapResponse( + new Date(), + ([key, value], context) => { + return [key, { value, context }]; + }, + { schema: schema, collection: 'articles' }, + ); + }).toThrowError(); +}); + +test('map flat date value', () => { + const date = new Date(); + + const result = deepMapResponse( + { date }, + ([key, value]) => { + return [key, value]; + }, + { schema: schema, collection: 'articles' }, + ); + + expect(result).toEqual({ date }); +}); + +test('map flat invalid deep field', () => { + const result = deepMapResponse( + { + author: { + invalid: 1, + }, + }, + ([key, value], context) => { + return [key, { value, context }]; + }, + { schema: schema, collection: 'articles' }, + ); + + expect(result).toEqual({ + author: { + value: { + invalid: 1, + }, + context: { + collection: schema.collections['articles'], + field: schema.collections['articles']!.fields['author'], + relation: getRelation(schema.relations, 'articles', 'author'), + leaf: false, + relationType: 'm2o', + }, + }, + }); +}); + +test('map flat invalid deep field', () => { + const result = deepMapResponse( + { + author: { + invalid: 1, + }, + }, + ([key, value], context) => { + return [key, { value, context }]; + }, + { schema: schema, collection: 'articles' }, + ); + + expect(result).toEqual({ + author: { + value: { + invalid: 1, + }, + context: { + collection: schema.collections['articles'], + field: schema.collections['articles']!.fields['author'], + relation: getRelation(schema.relations, 'articles', 'author'), + leaf: false, + relationType: 'm2o', + }, + }, + }); +}); + +test('map m2a relation without collection field', () => { + const callback = () => + deepMapResponse( + { + sections: [ + { + item: { + num: 123, + }, + }, + ], + }, + ([key, value], context) => { + return [key, { value, context }]; + }, + { schema: schema, collection: 'articles' }, + ); + + expect(callback).toThrowError( + "When selecting 'articles_builder.item', the field 'articles_builder.collection' has to be selected when using versioning and m2a relations", + ); +}); diff --git a/api/src/utils/deep-map-response.ts b/api/src/utils/deep-map-response.ts new file mode 100644 index 0000000000..c98c9cf1be --- /dev/null +++ b/api/src/utils/deep-map-response.ts @@ -0,0 +1,98 @@ +import type { CollectionOverview, FieldOverview, Relation, SchemaOverview } from '@directus/types'; +import { isPlainObject } from 'lodash-es'; +import assert from 'node:assert'; +import { getRelationInfo, type RelationInfo } from './get-relation-info.js'; +import { InvalidQueryError } from '@directus/errors'; + +/** + * Allows to deep map the response from the ItemsService with collection, field and relation context for each entry. + * Bottom to Top depth first mapping of values. + */ +export function deepMapResponse( + object: Record, + callback: ( + entry: [key: string | number, value: unknown], + context: { + collection: CollectionOverview; + field: FieldOverview; + relation: Relation | null; + leaf: boolean; + relationType: RelationInfo['relationType'] | null; + }, + ) => [key: string | number, value: unknown], + context: { + schema: SchemaOverview; + collection: string; + relationInfo?: RelationInfo; + }, +): any { + const collection = context.schema.collections[context.collection]; + + assert( + isPlainObject(object) && typeof object === 'object' && object !== null, + `DeepMapResponse only works on objects, received ${JSON.stringify(object)}`, + ); + + return Object.fromEntries( + Object.entries(object).map(([key, value]) => { + const field = collection?.fields[key]; + + if (!field) return [key, value]; + + const relationInfo = getRelationInfo(context.schema.relations, collection.collection, field.field); + let leaf = true; + + if (relationInfo.relation && typeof value === 'object' && value !== null && isPlainObject(object)) { + switch (relationInfo.relationType) { + case 'm2o': + value = deepMapResponse(value, callback, { + schema: context.schema, + collection: relationInfo.relation.related_collection!, + relationInfo, + }); + + leaf = false; + break; + case 'o2m': + value = (value as any[]).map((childValue) => { + if (isPlainObject(childValue) && typeof childValue === 'object' && childValue !== null) { + leaf = false; + return deepMapResponse(childValue, callback, { + schema: context.schema, + collection: relationInfo!.relation!.collection, + relationInfo, + }); + } else return childValue; + }); + + break; + + case 'a2o': { + const related_collection = object[relationInfo.relation.meta!.one_collection_field!]; + + if (!related_collection) { + throw new InvalidQueryError({ + reason: `When selecting '${collection.collection}.${field.field}', the field '${ + collection.collection + }.${ + relationInfo.relation.meta!.one_collection_field + }' has to be selected when using versioning and m2a relations `, + }); + } + + value = deepMapResponse(value, callback, { + schema: context.schema, + collection: related_collection, + relationInfo, + }); + + leaf = false; + break; + } + } + } + + return callback([key, value], { collection, field, ...relationInfo, leaf }); + }), + ); +} diff --git a/api/src/utils/get-relation-info.ts b/api/src/utils/get-relation-info.ts index 793758305f..92b6d24e34 100644 --- a/api/src/utils/get-relation-info.ts +++ b/api/src/utils/get-relation-info.ts @@ -2,7 +2,7 @@ import type { Relation, RelationMeta } from '@directus/types'; import { getRelation } from '@directus/utils'; import { getRelationType } from './get-relation-type.js'; -type RelationInfo = { +export type RelationInfo = { relation: Relation | null; relationType: 'o2m' | 'm2o' | 'a2o' | 'o2a' | null; }; diff --git a/api/src/utils/merge-version-data.test.ts b/api/src/utils/merge-version-data.test.ts deleted file mode 100644 index 38db116679..0000000000 --- a/api/src/utils/merge-version-data.test.ts +++ /dev/null @@ -1,917 +0,0 @@ -import { SchemaBuilder } from '@directus/schema-builder'; -import { describe, expect, test } from 'vitest'; -import { mergeVersionsRaw, mergeVersionsRecursive } from './merge-version-data.js'; - -describe('content versioning mergeVersionsRaw', () => { - test('No versions available', () => { - const result = mergeVersionsRaw({ test_field: 'value' }, []); - - expect(result).toMatchObject({ test_field: 'value' }); - }); - - test('Basic field versions', () => { - const result = mergeVersionsRaw({ test_field: 'value', edited_field: 'original' }, [ - { edited_field: 'updated' }, - { test_field: null }, - ]); - - expect(result).toMatchObject({ - test_field: null, - edited_field: 'updated', - }); - }); - - test('Relational field versions', () => { - const result = mergeVersionsRaw({ test_field: 'value', relation: null }, [ - { relation: { create: [{ test: 'value ' }], update: [], delete: [] } }, - ]); - - expect(result).toMatchObject({ - test_field: 'value', - relation: { - create: [{ test: 'value ' }], - update: [], - delete: [], - }, - }); - }); -}); - -describe('content versioning mergeVersionsRecursive', () => { - const schema = new SchemaBuilder() - .collection('collection_a', (c) => { - c.field('id').id(); - - c.field('status').string().options({ - defaultValue: 'draft', - }); - - c.field('m2o').m2o('collection_b', 'o2m'); - c.field('m2o_c').m2o('collection_c'); - c.field('m2m').m2m('collection_c'); - c.field('m2a').m2a(['collection_b', 'collection_c']); - }) - .collection('collection_b', (c) => { - c.field('id').id(); - - c.field('status').string().options({ - defaultValue: 'draft', - }); - }) - .collection('collection_c', (c) => { - c.field('id').id(); - - c.field('status').string().options({ - defaultValue: 'draft', - }); - - c.field('translations').translations(); - }) - .collection('collection_c_translations', (c) => { - c.field('id').id(); - c.field('text').string(); - }) - .build(); - - test('No versions available', () => { - const result = mergeVersionsRecursive({ status: 'draft' }, [], 'collection_a', schema); - - expect(result).toMatchObject({ status: 'draft' }); - }); - - describe('m2o field', () => { - test('Setting m2o value', () => { - const result = mergeVersionsRecursive( - { id: 1, status: 'draft', m2o: null }, - [{ status: 'published' }, { m2o: 1 }], - 'collection_a', - schema, - ); - - expect(result).toMatchObject({ id: 1, status: 'published', m2o: 1 }); - }); - - test('Unsetting m2o value', () => { - const result = mergeVersionsRecursive( - { id: 1, status: 'draft', m2o: { id: 1, status: 'draft' } }, - [{ status: 'published', m2o: null }], - 'collection_a', - schema, - ); - - expect(result).toMatchObject({ id: 1, status: 'published', m2o: null }); - }); - - test('Updating m2o value', () => { - const result = mergeVersionsRecursive( - { id: 1, status: 'draft', m2o: { id: 1, test: 'data', status: 'draft' } }, - [{ status: 'published' }, { m2o: { id: 1, status: 'published' } }], - 'collection_a', - schema, - ); - - expect(result).toMatchObject({ id: 1, status: 'published', m2o: { id: 1, test: 'data', status: 'published' } }); - }); - }); - - describe('o2m field', () => { - test('Setting o2m values', () => { - const result = mergeVersionsRecursive( - { id: 2, status: 'draft', o2m: [] }, - [ - { - o2m: { - create: [{ status: 'draft' }], - update: [ - { - m2o: '2', - id: 2, - }, - { - m2o: '2', - id: 3, - }, - ], - delete: [], - }, - }, - ], - 'collection_b', - schema, - ); - - expect(result).toMatchObject({ - id: 2, - status: 'draft', - o2m: [{ m2o: '2', id: 2 }, { m2o: '2', id: 3 }, { status: 'draft' }], - }); - }); - - test('Updating o2m values', () => { - const result = mergeVersionsRecursive( - { id: 1, status: 'draft', o2m: [1, 2, 3, { id: 4, test: 'value' }, { id: 5 }] }, - [ - { - status: 'published', - }, - { - o2m: { - create: [ - { - test: 'new', - }, - ], - update: [ - { - id: 1, - }, - { - id: 4, - }, - ], - delete: [2, 5], - }, - }, - ], - 'collection_b', - schema, - ); - - expect(result).toMatchObject({ - id: 1, - status: 'published', - o2m: [ - { - id: 1, - }, - 3, - { - id: 4, - test: 'value', - }, - { - test: 'new', - }, - ], - }); - }); - }); - - describe('m2m field', () => { - test('Adding related items', () => { - const result = mergeVersionsRecursive( - { - id: 1, - status: 'draft', - m2m: [], - }, - [ - { - status: 'published', - m2m: { - create: [ - { - collection_c_id: { - status: 'published', - }, - }, - { - collection_a_id: '1', - collection_c_id: { - id: 1, - }, - }, - ], - update: [], - delete: [], - }, - }, - ], - 'collection_a', - schema, - ); - - expect(result).toMatchObject({ - id: 1, - status: 'published', - m2m: [ - { - collection_c_id: { - status: 'published', - }, - }, - { - collection_a_id: '1', - collection_c_id: { - id: 1, - }, - }, - ], - }); - }); - - test('Updating m2m values', () => { - const result = mergeVersionsRecursive( - { - id: 1, - status: 'draft', - m2m: [1, 2, 3, { id: 4 }, { id: 5 }], - }, - [ - { - status: 'published', - }, - { - m2m: { - create: [ - { - collection_c_id: { - id: 3, - }, - }, - ], - update: [ - { - id: 1, - collection_c_id: { - id: 1, - }, - }, - { - id: 4, - collection_c_id: { - id: 2, - }, - }, - ], - delete: [2, 5], - }, - }, - ], - 'collection_a', - schema, - ); - - expect(result).toMatchObject({ - id: 1, - status: 'published', - m2m: [ - { - collection_c_id: { - id: 1, - }, - id: 1, - }, - 3, - { - id: 4, - collection_c_id: { - id: 2, - }, - }, - { - collection_c_id: { - id: 3, - }, - }, - ], - }); - }); - }); - - describe('m2a field', () => { - test('Adding related items', () => { - const result = mergeVersionsRecursive( - { - id: 1, - status: 'draft', - m2a: [], - }, - [ - { - m2a: { - create: [ - { - collection_a_id: '1', - collection: 'collection_b', - item: { - id: 2, - }, - }, - { - collection_a_id: '1', - collection: 'collection_c', - item: { - id: 1, - }, - }, - { - collection: 'collection_b', - item: { - status: 'published', - }, - }, - { - collection: 'collection_c', - item: { - status: 'published', - }, - }, - ], - update: [], - delete: [], - }, - }, - ], - 'collection_a', - schema, - ); - - expect(result).toMatchObject({ - id: 1, - status: 'draft', - m2a: [ - { - collection_a_id: '1', - collection: 'collection_b', - item: { - id: 2, - }, - }, - { - collection_a_id: '1', - collection: 'collection_c', - item: { - id: 1, - }, - }, - { - collection: 'collection_b', - item: { - status: 'published', - }, - }, - { - collection: 'collection_c', - item: { - status: 'published', - }, - }, - ], - }); - }); - - test('Updating m2a values', () => { - const result = mergeVersionsRecursive( - { - id: 1, - status: 'draft', - m2a: [ - 1, - { - id: 2, - collection_a_id: 1, - item: '1', - collection: 'collection_c', - }, - 3, - { id: 4 }, - { - id: 5, - collection_a_id: 1, - item: '1', - collection: 'collection_b', - }, - ], - }, - [ - { - status: 'published', - }, - { - m2a: { - create: [ - { - collection: 'collection_c', - item: { - status: 'published', - }, - }, - ], - update: [ - { - collection: 'collection_b', - item: { - status: 'published', - id: 1, - }, - id: 1, - }, - { - collection: 'collection_b', - item: { - id: '2', - }, - id: 5, - }, - ], - delete: [2, 4], - }, - }, - ], - 'collection_a', - schema, - ); - - expect(result).toMatchObject({ - id: 1, - status: 'published', - m2a: [ - { - id: 1, - item: { - status: 'published', - id: 1, - }, - collection: 'collection_b', - }, - 3, - { - id: 5, - collection_a_id: 1, - item: { - id: '2', - }, - collection: 'collection_b', - }, - { - collection: 'collection_c', - item: { - status: 'published', - }, - }, - ], - }); - }); - }); - - describe('nested relations', () => { - test('m2o > translation', () => { - const result = mergeVersionsRecursive( - { - id: 1, - status: 'draft', - m2o_c: { - id: 1, - status: 'draft', - translations: [ - { - id: 1, - collection_c_id: 1, - languages_id: 'ar-SA', - text: 'ar-sa', - }, - { - id: 2, - collection_c_id: 1, - languages_id: 'de-DE', - text: 'de-de', - }, - ], - }, - }, - [ - { - m2o_c: { - translations: { - create: [ - { - text: 'en-us', - languages_id: { - code: 'en-US', - }, - collection_c_id: 1, - }, - ], - update: [ - { - text: 'german', - languages_id: { - code: 'de-DE', - }, - id: 2, - }, - ], - delete: [1], - }, - id: 1, - }, - }, - ], - 'collection_a', - schema, - ); - - expect(result).toMatchObject({ - id: 1, - status: 'draft', - m2o_c: { - id: 1, - status: 'draft', - translations: [ - { - id: 2, - collection_c_id: 1, - languages_id: { - code: 'de-DE', - }, - text: 'german', - }, - { - text: 'en-us', - languages_id: { - code: 'en-US', - }, - collection_c_id: 1, - }, - ], - }, - }); - }); - - test('m2m > translations', () => { - const result = mergeVersionsRecursive( - { - id: 3, - status: 'draft', - m2m: [ - { - id: 2, - collection_a_id: 3, - collection_c_id: { - id: 1, - status: 'draft', - translations: [ - { - id: 1, - collection_c_id: 1, - languages_id: 'ar-SA', - text: 'ar-sa', - }, - { - id: 2, - collection_c_id: 1, - languages_id: 'de-DE', - text: 'de-de', - }, - ], - }, - }, - { - id: 3, - collection_a_id: 3, - collection_c_id: { - id: 2, - status: 'draft', - translations: [], - }, - }, - ], - }, - [ - { - m2m: { - create: [], - update: [ - { - collection_c_id: { - translations: { - create: [ - { - text: 'english', - languages_id: { - code: 'en-US', - }, - collection_c_id: 1, - }, - ], - update: [ - { - text: 'german', - languages_id: { - code: 'de-DE', - }, - id: 2, - }, - ], - delete: [1], - }, - id: 1, - }, - id: 2, - }, - ], - delete: [3], - }, - }, - ], - 'collection_a', - schema, - ); - - expect(result).toMatchObject({ - id: 3, - status: 'draft', - m2m: [ - { - id: 2, - collection_a_id: 3, - collection_c_id: { - translations: [ - { - id: 2, - collection_c_id: 1, - text: 'german', - languages_id: { - code: 'de-DE', - }, - }, - { - text: 'english', - languages_id: { - code: 'en-US', - }, - collection_c_id: 1, - }, - ], - id: 1, - }, - }, - ], - }); - }); - - test('m2a > translations', () => { - const result = mergeVersionsRecursive( - { - id: 4, - status: 'draft', - m2a: [ - { - id: 3, - collection_a_id: 4, - collection: 'collection_b', - item: 2, - }, - { - id: 4, - collection_a_id: 4, - collection: 'collection_c', - item: { - id: 1, - translations: [ - { - id: 1, - collection_c_id: 1, - languages_id: 'ar-SA', - text: 'ar-sa', - }, - { - id: 2, - collection_c_id: 1, - languages_id: 'de-DE', - text: 'de-de', - }, - ], - }, - }, - ], - }, - [ - { - m2a: { - create: [], - update: [ - { - collection: 'collection_c', - item: { - translations: { - create: [ - { - languages_id: { - code: 'en-US', - }, - collection_c_id: 1, - text: 'english', - }, - ], - update: [ - { - text: 'german', - languages_id: { - code: 'de-DE', - }, - id: 2, - }, - ], - delete: [1], - }, - id: 1, - }, - id: 4, - }, - ], - delete: [], - }, - }, - ], - 'collection_a', - schema, - ); - - expect(result).toMatchObject({ - id: 4, - status: 'draft', - m2a: [ - { - id: 3, - collection_a_id: 4, - collection: 'collection_b', - item: 2, - }, - { - id: 4, - collection_a_id: 4, - collection: 'collection_c', - item: { - id: 1, - translations: [ - { - id: 2, - collection_c_id: 1, - languages_id: { - code: 'de-DE', - }, - text: 'german', - }, - { - languages_id: { - code: 'en-US', - }, - collection_c_id: 1, - text: 'english', - }, - ], - }, - }, - ], - }); - }); - - test('creating nested relations', () => { - const result = mergeVersionsRecursive( - { - id: 2, - status: 'draft', - m2m: [], - m2o_c: null, - }, - [ - { - m2m: { - create: [ - { - collection_c_id: { - translations: { - create: [ - { - text: 'german', - languages_id: { - code: 'de-DE', - }, - }, - { - text: 'english', - languages_id: { - code: 'en-US', - }, - }, - ], - update: [], - delete: [], - }, - }, - }, - ], - update: [], - delete: [], - }, - m2o_c: { - translations: { - create: [ - { - text: 'french', - languages_id: { - code: 'fr-FR', - }, - }, - { - text: 'english', - languages_id: { - code: 'en-US', - }, - }, - ], - update: [], - delete: [], - }, - }, - }, - ], - 'collection_a', - schema, - ); - - expect(result).toMatchObject({ - id: 2, - status: 'draft', - m2m: [ - { - collection_c_id: { - translations: [ - { - text: 'german', - languages_id: { - code: 'de-DE', - }, - }, - { - text: 'english', - languages_id: { - code: 'en-US', - }, - }, - ], - }, - }, - ], - m2o_c: { - translations: [ - { - text: 'french', - languages_id: { - code: 'fr-FR', - }, - }, - { - text: 'english', - languages_id: { - code: 'en-US', - }, - }, - ], - }, - }); - }); - }); -}); diff --git a/api/src/utils/merge-version-data.ts b/api/src/utils/merge-version-data.ts deleted file mode 100644 index a37b22de9a..0000000000 --- a/api/src/utils/merge-version-data.ts +++ /dev/null @@ -1,184 +0,0 @@ -import type { Alterations, Item, SchemaOverview } from '@directus/types'; -import { isObject } from '@directus/utils'; -import Joi from 'joi'; -import { cloneDeep } from 'lodash-es'; - -const alterationSchema = Joi.object({ - create: Joi.array().items(Joi.object().unknown()), - update: Joi.array().items(Joi.object().unknown()), - delete: Joi.array().items(Joi.string(), Joi.number()), -}); - -export function mergeVersionsRaw(item: Item, versionData: Partial[]) { - const result = cloneDeep(item); - - for (const versionRecord of versionData) { - for (const key of Object.keys(versionRecord)) { - result[key] = versionRecord[key]; - } - } - - return result; -} - -export function mergeVersionsRecursive( - item: Item, - versionData: Item[], - collection: string, - schema: SchemaOverview, -): Item { - if (versionData.length === 0) return item; - - return recursiveMerging(item, versionData, collection, schema) as Item; -} - -function recursiveMerging(data: Item, versionData: unknown[], collection: string, schema: SchemaOverview): unknown { - const result = cloneDeep(data); - const relations = getRelations(collection, schema); - - for (const versionRecord of versionData) { - if (!isObject(versionRecord)) { - continue; - } - - for (const key of Object.keys(data)) { - if (key in versionRecord === false) { - continue; - } - - const currentValue: unknown = data[key]; - const newValue: unknown = versionRecord[key]; - - if (typeof newValue !== 'object' || newValue === null) { - // primitive type substitution, json and non relational array values are handled in the next check - result[key] = newValue; - continue; - } - - if (key in relations === false) { - // check for m2a exception - if (isManyToAnyCollection(collection, schema) && key === 'item') { - const item = addMissingKeys(isObject(currentValue) ? currentValue : {}, newValue); - result[key] = recursiveMerging(item, [newValue], data['collection'], schema); - } else { - // item is not a relation - result[key] = newValue; - } - - continue; - } - - const { error } = alterationSchema.validate(newValue); - - if (error) { - if (typeof newValue === 'object' && key in relations) { - const newItem = !currentValue || typeof currentValue !== 'object' ? newValue : currentValue; - result[key] = recursiveMerging(newItem, [newValue], relations[key]!, schema); - } - - continue; - } - - const alterations = newValue as Alterations; - const currentPrimaryKeyField = schema.collections[collection]!.primary; - const relatedPrimaryKeyField = schema.collections[relations[key]!]!.primary; - - const mergedRelation: Item[] = []; - - if (Array.isArray(currentValue)) { - if (alterations.delete.length > 0) { - for (const currentItem of currentValue) { - const currentId = typeof currentItem === 'object' ? currentItem[currentPrimaryKeyField] : currentItem; - - if (alterations.delete.includes(currentId) === false) { - mergedRelation.push(currentItem); - } - } - } else { - mergedRelation.push(...currentValue); - } - - if (alterations.update.length > 0) { - for (const updatedItem of alterations.update) { - // find existing item to update - const itemIndex = mergedRelation.findIndex( - (currentItem) => currentItem[relatedPrimaryKeyField] === updatedItem[currentPrimaryKeyField], - ); - - if (itemIndex === -1) { - // check for raw primary keys - const pkIndex = mergedRelation.findIndex( - (currentItem) => currentItem === updatedItem[currentPrimaryKeyField], - ); - - if (pkIndex === -1) { - // nothing to update so add the item as is - mergedRelation.push(updatedItem); - } else { - mergedRelation[pkIndex] = updatedItem; - } - - continue; - } - - const item = addMissingKeys(mergedRelation[itemIndex]!, updatedItem); - - mergedRelation[itemIndex] = recursiveMerging(item, [updatedItem], relations[key]!, schema) as Item; - } - } - } - - if (alterations.create.length > 0) { - for (const createdItem of alterations.create) { - const item = addMissingKeys({}, createdItem); - mergedRelation.push(recursiveMerging(item, [createdItem], relations[key]!, schema) as Item); - } - } - - result[key] = mergedRelation; - } - } - - return result; -} - -function addMissingKeys(item: Item, edits: Item) { - const result: Item = { ...item }; - - for (const key of Object.keys(edits)) { - if (key in item === false) { - result[key] = null; - } - } - - return result; -} - -function isManyToAnyCollection(collection: string, schema: SchemaOverview) { - const relation = schema.relations.find( - (relation) => relation.collection === collection && relation.meta?.many_collection === collection, - ); - - if (!relation || !relation.meta?.one_field || !relation.related_collection) return false; - - return Boolean( - schema.collections[relation.related_collection]?.fields[relation.meta.one_field]?.special.includes('m2a'), - ); -} - -function getRelations(collection: string, schema: SchemaOverview) { - return schema.relations.reduce( - (result, relation) => { - if (relation.related_collection === collection && relation.meta?.one_field) { - result[relation.meta.one_field] = relation.collection; - } - - if (relation.collection === collection && relation.related_collection) { - result[relation.field] = relation.related_collection; - } - - return result; - }, - {} as Record, - ); -} diff --git a/api/src/utils/transaction.ts b/api/src/utils/transaction.ts index 70ae10c00a..79ac7ceefb 100644 --- a/api/src/utils/transaction.ts +++ b/api/src/utils/transaction.ts @@ -11,9 +11,12 @@ import type { DatabaseClient } from '@directus/types'; * Can be used to ensure the handler is run within a transaction, * while preventing nested transactions. */ -export const transaction = async (knex: Knex, handler: (knex: Knex) => Promise): Promise => { +export const transaction = async ( + knex: Knex, + handler: (knex: Knex.Transaction) => Promise, +): Promise => { if (knex.isTransaction) { - return handler(knex); + return handler(knex as Knex.Transaction); } else { try { return await knex.transaction((trx) => handler(trx)); @@ -70,10 +73,27 @@ function shouldRetryTransaction(client: DatabaseClient, error: unknown): boolean * @link https://www.sqlite.org/rescode.html#busy */ const SQLITE_BUSY_ERROR_CODE = 'SQLITE_BUSY'; + // Both mariadb and mysql + const MYSQL_DEADLOCK_CODE = 'ER_LOCK_DEADLOCK'; + const POSTGRES_DEADLOCK_CODE = '40P01'; + const ORACLE_DEADLOCK_CODE = 'ORA-00060'; + const MSSQL_DEADLOCK_CODE = 'EREQUEST'; + const MSSQL_DEADLOCK_NUMBER = '1205'; + + const codes: Record[]> = { + cockroachdb: [{ code: COCKROACH_RETRY_ERROR_CODE }], + sqlite: [{ code: SQLITE_BUSY_ERROR_CODE }], + mysql: [{ code: MYSQL_DEADLOCK_CODE }], + mssql: [{ code: MSSQL_DEADLOCK_CODE, number: MSSQL_DEADLOCK_NUMBER }], + oracle: [{ code: ORACLE_DEADLOCK_CODE }], + postgres: [{ code: POSTGRES_DEADLOCK_CODE }], + redshift: [], + }; return ( isObject(error) && - ((client === 'cockroachdb' && error['code'] === COCKROACH_RETRY_ERROR_CODE) || - (client === 'sqlite' && error['code'] === SQLITE_BUSY_ERROR_CODE)) + codes[client].some((code) => { + return Object.entries(code).every(([key, value]) => String(error[key]) === value); + }) ); } diff --git a/api/src/utils/versioning/handle-version.ts b/api/src/utils/versioning/handle-version.ts new file mode 100644 index 0000000000..2cd2cf1569 --- /dev/null +++ b/api/src/utils/versioning/handle-version.ts @@ -0,0 +1,122 @@ +import { ForbiddenError } from '@directus/errors'; +import type { Accountability, Item, PrimaryKey, Query, QueryOptions } from '@directus/types'; +import type { ItemsService as ItemsServiceType } from '../../services/index.js'; +import { deepMapResponse } from '../deep-map-response.js'; +import { transaction } from '../transaction.js'; +import { mergeVersionsRaw } from './merge-version-data.js'; + +export async function handleVersion(self: ItemsServiceType, key: PrimaryKey, queryWithKey: Query, opts?: QueryOptions) { + const { VersionsService } = await import('../../services/versions.js'); + const { ItemsService } = await import('../../services/items.js'); + + if (queryWithKey.versionRaw) { + const originalData = await self.readByQuery(queryWithKey, opts); + + if (originalData.length === 0) { + throw new ForbiddenError(); + } + + const versionsService = new VersionsService({ + schema: self.schema, + accountability: self.accountability, + knex: self.knex, + }); + + const versionData = await versionsService.getVersionSaves(queryWithKey.version!, self.collection, key as string); + + if (!versionData || versionData.length === 0) return [originalData[0]]; + + return [mergeVersionsRaw(originalData[0]!, versionData)]; + } + + let results: Item[] = []; + + const versionsService = new VersionsService({ + schema: self.schema, + accountability: self.accountability, + knex: self.knex, + }); + + const createdIDs: Record = {}; + const versionData = await versionsService.getVersionSaves(queryWithKey.version!, self.collection, key as string); + + await transaction(self.knex, async (trx) => { + const itemsServiceAdmin = new ItemsService(self.collection, { + schema: self.schema, + accountability: { + admin: true, + } as Accountability, + knex: trx, + }); + + await Promise.all( + (versionData ?? []).map((data) => { + return itemsServiceAdmin.updateOne(key, data as any, { + emitEvents: false, + autoPurgeCache: false, + skipTracking: true, + + onItemCreate: (collection, pk) => { + if (collection in createdIDs === false) createdIDs[collection] = []; + + createdIDs[collection]!.push(pk); + }, + }); + }), + ); + + const itemsServiceUser = new ItemsService(self.collection, { + schema: self.schema, + accountability: self.accountability, + knex: trx, + }); + + results = await itemsServiceUser.readByQuery(queryWithKey, opts); + + await trx.rollback(); + }); + + results = results.map((result) => { + return deepMapResponse( + result, + ([key, value], context) => { + if (context.relationType === 'm2o' || context.relationType === 'a2o') { + const ids = createdIDs[context.relation!.related_collection!]; + const match = ids?.find((id) => String(id) === String(value)); + + if (match) { + return [key, null]; + } + } else if (context.relationType === 'o2m' && Array.isArray(value)) { + const ids = createdIDs[context.relation!.collection]; + return [ + key, + value.map((val) => { + const match = ids?.find((id) => String(id) === String(val)); + + if (match) { + return null; + } + + return val; + }), + ]; + } + + if (context.field.field === context.collection.primary) { + const ids = createdIDs[context.collection.collection]; + const match = ids?.find((id) => String(id) === String(value)); + + if (match) { + return [key, null]; + } + } + + return [key, value]; + }, + { collection: self.collection, schema: self.schema }, + ); + }); + + return results; +} diff --git a/api/src/utils/versioning/merge-version-data.test.ts b/api/src/utils/versioning/merge-version-data.test.ts new file mode 100644 index 0000000000..92157799da --- /dev/null +++ b/api/src/utils/versioning/merge-version-data.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, test } from 'vitest'; +import { mergeVersionsRaw } from './merge-version-data.js'; + +describe('content versioning mergeVersionsRaw', () => { + test('No versions available', () => { + const result = mergeVersionsRaw({ test_field: 'value' }, []); + + expect(result).toMatchObject({ test_field: 'value' }); + }); + + test('Basic field versions', () => { + const result = mergeVersionsRaw({ test_field: 'value', edited_field: 'original' }, [ + { edited_field: 'updated' }, + { test_field: null }, + ]); + + expect(result).toMatchObject({ + test_field: null, + edited_field: 'updated', + }); + }); + + test('Relational field versions', () => { + const result = mergeVersionsRaw({ test_field: 'value', relation: null }, [ + { relation: { create: [{ test: 'value ' }], update: [], delete: [] } }, + ]); + + expect(result).toMatchObject({ + test_field: 'value', + relation: { + create: [{ test: 'value ' }], + update: [], + delete: [], + }, + }); + }); +}); diff --git a/api/src/utils/versioning/merge-version-data.ts b/api/src/utils/versioning/merge-version-data.ts new file mode 100644 index 0000000000..4d612efd0f --- /dev/null +++ b/api/src/utils/versioning/merge-version-data.ts @@ -0,0 +1,14 @@ +import type { Item } from '@directus/types'; +import { cloneDeep } from 'lodash-es'; + +export function mergeVersionsRaw(item: Item, versionData: Partial[]) { + const result = cloneDeep(item); + + for (const versionRecord of versionData) { + for (const key of Object.keys(versionRecord)) { + result[key] = versionRecord[key]; + } + } + + return result; +} diff --git a/packages/types/src/items.ts b/packages/types/src/items.ts index 549b63238b..2e6f2e196f 100644 --- a/packages/types/src/items.ts +++ b/packages/types/src/items.ts @@ -31,6 +31,11 @@ export type QueryOptions = { }; export type MutationOptions = { + /** + * Callback function that's fired whenever a item is made in the mutation + */ + onItemCreate?: ((collection: string, pk: PrimaryKey) => void) | undefined; + /** * Callback function that's fired whenever a revision is made in the mutation */ @@ -62,6 +67,11 @@ export type MutationOptions = { */ bypassLimits?: boolean | undefined; + /** + * Skips the creation of accountability and revision entries + */ + skipTracking?: boolean | undefined; + /** * To keep track of mutation limits */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3cde2995ac..064bb6bce7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6340,6 +6340,9 @@ packages: '@types/node@22.13.14': resolution: {integrity: sha512-Zs/Ollc1SJ8nKUAgc7ivOEdIBM8JAKgrqqUYi2J997JuKO7/tpQC+WCetQ1sypiKCQWHdvdg9wBNpUPEWZae7w==} + '@types/node@22.13.8': + resolution: {integrity: sha512-G3EfaZS+iOGYWLLRCEAXdWK9my08oHNZ+FHluRiggIYJPOXzhOiDgpVCUHaUvyIC5/fj7C/p637jdzC666AOKQ==} + '@types/node@24.2.0': resolution: {integrity: sha512-3xyG3pMCq3oYCNg7/ZP+E1ooTaGB4cG8JWRsqqOYQdbWNY4zbaV0Ennrd7stjiJEFZCaybcIgpTjJWHRfBSIDw==} @@ -18027,7 +18030,7 @@ snapshots: '@types/body-parser@1.19.6': dependencies: '@types/connect': 3.4.38 - '@types/node': 22.13.14 + '@types/node': 22.13.8 '@types/busboy@1.5.4': dependencies: @@ -18039,7 +18042,7 @@ snapshots: dependencies: '@types/http-cache-semantics': 4.0.4 '@types/keyv': 3.1.4 - '@types/node': 24.2.0 + '@types/node': 22.13.8 '@types/responselike': 1.0.3 '@types/caseless@0.12.5': {} @@ -18056,7 +18059,7 @@ snapshots: '@types/connect@3.4.38': dependencies: - '@types/node': 22.13.14 + '@types/node': 22.13.8 '@types/content-disposition@0.5.9': {} @@ -18113,7 +18116,7 @@ snapshots: '@types/fs-extra@9.0.13': dependencies: - '@types/node': 24.2.0 + '@types/node': 22.13.8 '@types/geojson@7946.0.16': {} @@ -18138,7 +18141,7 @@ snapshots: '@types/jsonfile@6.1.4': dependencies: - '@types/node': 22.13.14 + '@types/node': 22.13.8 '@types/jsonwebtoken@9.0.10': dependencies: @@ -18147,7 +18150,7 @@ snapshots: '@types/keyv@3.1.4': dependencies: - '@types/node': 24.2.0 + '@types/node': 22.13.8 '@types/ldapjs@2.2.5': dependencies: @@ -18206,6 +18209,10 @@ snapshots: dependencies: undici-types: 6.20.0 + '@types/node@22.13.8': + dependencies: + undici-types: 6.20.0 + '@types/node@24.2.0': dependencies: undici-types: 7.10.0 @@ -18236,13 +18243,13 @@ snapshots: '@types/readable-stream@4.0.21': dependencies: - '@types/node': 22.13.14 + '@types/node': 22.13.8 optional: true '@types/request@2.48.13': dependencies: '@types/caseless': 0.12.5 - '@types/node': 24.2.0 + '@types/node': 22.13.8 '@types/tough-cookie': 4.0.5 form-data: 2.5.5 @@ -18250,7 +18257,7 @@ snapshots: '@types/responselike@1.0.3': dependencies: - '@types/node': 24.2.0 + '@types/node': 22.13.8 '@types/sanitize-html@2.16.0': dependencies: @@ -18263,21 +18270,21 @@ snapshots: '@types/send@0.17.5': dependencies: '@types/mime': 1.3.5 - '@types/node': 22.13.14 + '@types/node': 22.13.8 '@types/serve-static@1.15.8': dependencies: '@types/http-errors': 2.0.5 - '@types/node': 22.13.14 + '@types/node': 22.13.8 '@types/send': 0.17.5 '@types/ssri@7.1.5': dependencies: - '@types/node': 22.13.14 + '@types/node': 22.13.8 '@types/stream-chain@2.1.0': dependencies: - '@types/node': 22.13.14 + '@types/node': 22.13.8 '@types/stream-json@1.7.8': dependencies: @@ -18288,7 +18295,7 @@ snapshots: dependencies: '@types/cookiejar': 2.1.5 '@types/methods': 1.1.4 - '@types/node': 24.2.0 + '@types/node': 22.13.8 form-data: 4.0.4 '@types/supertest@6.0.2': @@ -18302,7 +18309,7 @@ snapshots: '@types/through@0.0.33': dependencies: - '@types/node': 22.13.14 + '@types/node': 22.13.8 '@types/tmp@0.2.6': {} @@ -20142,7 +20149,7 @@ snapshots: engine.io@6.6.4: dependencies: '@types/cors': 2.8.19 - '@types/node': 24.2.0 + '@types/node': 22.13.8 accepts: 1.3.8 base64id: 2.0.0 cookie: 0.7.2 diff --git a/tests/blackbox/common/functions.ts b/tests/blackbox/common/functions.ts index 6b39fffb2e..fe18d525fc 100644 --- a/tests/blackbox/common/functions.ts +++ b/tests/blackbox/common/functions.ts @@ -1,4 +1,4 @@ -import type { Permission, Query } from '@directus/types'; +import type { Permission, PrimaryKey, Query } from '@directus/types'; import { omit } from 'lodash-es'; import { randomUUID } from 'node:crypto'; import request from 'supertest'; @@ -51,6 +51,36 @@ export type OptionsCreateRole = { name: string; }; +export type OptionsCreateVersion = { + collection: string; + item: PrimaryKey; + key: string; + name: string; +}; + +export async function CreateVersion(vendor: Vendor, options: OptionsCreateVersion) { + const response = await request(getUrl(vendor)) + .post(`/versions`) + .set('Authorization', `Bearer ${USER.TESTS_FLOW.TOKEN}`) + .send(options); + + return response.body.data; +} +export async function SaveVersion( + vendor: Vendor, + options: { + id: string; + delta: any; + }, +) { + const response = await request(getUrl(vendor)) + .post(`/versions/${options.id}/save`) + .set('Authorization', `Bearer ${USER.TESTS_FLOW.TOKEN}`) + .send(options.delta); + + return response.body.data; +} + export async function CreateRole(vendor: Vendor, options: OptionsCreateRole) { // Action const roleResponse = await request(getUrl(vendor)) @@ -668,10 +698,7 @@ export async function CreateItem(vendor: Vendor, options: OptionsCreateItem) { .set('Authorization', `Bearer ${options.token ?? USER.TESTS_FLOW.TOKEN}`) .send(options.item); - if (!response.ok) { - throw new Error('Could not create item', response.body); - } - + expect(response.ok, JSON.stringify(response.body)).toBeTruthy(); return response.body.data; } diff --git a/tests/blackbox/tests/db/routes/items/version.seed.ts b/tests/blackbox/tests/db/routes/items/version.seed.ts new file mode 100644 index 0000000000..76834a944d --- /dev/null +++ b/tests/blackbox/tests/db/routes/items/version.seed.ts @@ -0,0 +1,114 @@ +import { + CreateCollection, + CreateField, + CreateFieldM2A, + CreateFieldM2M, + CreateFieldM2O, + CreateFieldO2M, + DeleteCollection, +} from '@common/functions'; +import vendors from '@common/get-dbs-to-test'; +import { it } from 'vitest'; + +export const c = { + articles_tags: 'test_items_version_articles_tags', + articles_sections: 'test_items_version_articles_sections', + sec_num: 'test_items_version_articles_sec_num', + sec_text: 'test_items_version_articles_sec_text', + tags: 'test_items_version_tags', + links: 'test_items_version_links', + articles: 'test_items_version_article', + users: 'test_items_version_users', +} as const; + +const fields = [ + { + collection: c.articles, + field: 'title', + type: 'string', + }, + { + collection: c.users, + field: 'name', + type: 'string', + }, + { + collection: c.links, + field: 'link', + type: 'string', + }, + { + collection: c.tags, + field: 'tag', + type: 'string', + }, + { + collection: c.sec_num, + field: 'num', + type: 'integer', + }, + { + collection: c.sec_text, + field: 'text', + type: 'string', + }, +]; + +export const seedDBStructure = () => { + it.each(vendors)( + '%s', + async (vendor) => { + // Delete existing collections + for (const collection of Object.values(c)) { + await DeleteCollection(vendor, { collection }); + } + + // // Create all collections + await Promise.all( + Object.values(c).map((collection) => + CreateCollection(vendor, { + collection: collection, + primaryKeyType: 'integer', + meta: { + versioning: true, + }, + }), + ), + ); + + await Promise.all(fields.map((field) => CreateField(vendor, field))); + + // Create M2M relationships + await CreateFieldM2M(vendor, { + collection: c.articles, + field: 'tags', + otherCollection: c.tags, + otherField: 'articles', + junctionCollection: c.articles_tags, + primaryKeyType: 'integer', + }); + + await CreateFieldM2A(vendor, { + collection: c.articles, + field: 'sections', + junctionCollection: c.articles_sections, + relatedCollections: [c.sec_num, c.sec_text], + primaryKeyType: 'integer', + }); + + await CreateFieldM2O(vendor, { + collection: c.articles, + field: 'author', + otherCollection: c.users, + }); + + await CreateFieldO2M(vendor, { + collection: c.articles, + field: 'links', + otherCollection: c.links, + otherField: 'article_id', + }); + }, + 300000, + ); +}; diff --git a/tests/blackbox/tests/db/routes/items/version.test.ts b/tests/blackbox/tests/db/routes/items/version.test.ts new file mode 100644 index 0000000000..b7db069217 --- /dev/null +++ b/tests/blackbox/tests/db/routes/items/version.test.ts @@ -0,0 +1,384 @@ +import { CreateItem, CreateVersion, SaveVersion } from '@common/functions'; +import vendors from '@common/get-dbs-to-test'; +import { describe, expect, it } from 'vitest'; +import { c } from './version.seed'; +import request from 'supertest'; +import { getUrl } from '@common/config'; +import { USER } from '@common/variables'; + +const item = { + title: 'Article 1', + author: { + name: 'Author 1', + }, + links: [ + { + link: 'Link A', + }, + { + link: 'Link B', + }, + ], + tags: [ + { + [`${c.tags}_id`]: { + tag: 'Tag A', + }, + }, + { + [`${c.tags}_id`]: { + tag: 'Tag B', + }, + }, + ], + sections: [ + { + collection: c.sec_text, + item: { + text: 'Text A', + }, + }, + { + collection: c.sec_num, + item: { + num: 2, + }, + }, + ], +} as const; + +describe('version response', () => { + it.each(vendors)('%s', async (vendor) => { + const result = await CreateItem(vendor, { + collection: c.articles, + item, + }); + + const versionResult = await CreateVersion(vendor, { + collection: c.articles, + item: String(result.id), + key: 'test', + name: 'test', + }); + + await SaveVersion(vendor, { + id: versionResult.id, + delta: { + title: 'Changed', + }, + }); + + let response = await request(getUrl(vendor)) + .get(`/items/${c.articles}/${result.id}?version=test`) + .set('Authorization', `Bearer ${USER.ADMIN.TOKEN}`); + + expect(JSON.parse(response.text)).toEqual({ + data: { id: 1, title: 'Changed', author: 1, tags: [1, 2], sections: [1, 2], links: [1, 2] }, + }); + + await SaveVersion(vendor, { + id: versionResult.id, + delta: { + title: 'Changed', + author: { + id: 1, + name: 'Changed', + }, + links: [ + { + id: 1, + link: 'Link A Changed', + }, + ], + tags: [ + { + id: 1, + [`${c.tags}_id`]: { + id: 1, + tag: 'Tag A Changed', + }, + }, + ], + sections: [ + { + id: 1, + collection: c.sec_text, + item: { + id: 1, + text: 'Text A Changed', + }, + }, + ], + }, + }); + + response = await request(getUrl(vendor)) + .get( + `/items/${c.articles}/${result.id}?version=test&fields=id,title,author.id,author.name,tags.id,tags.${c.tags}_id.id,tags.${c.tags}_id.tag,sections.id,sections.collection,sections.item.*,links.id,links.link`, + ) + .set('Authorization', `Bearer ${USER.ADMIN.TOKEN}`); + + expect(JSON.parse(response.text)).toEqual({ + data: { + id: 1, + title: 'Changed', + author: { + id: 1, + name: 'Changed', + }, + tags: [ + { + id: 1, + [`${c.tags}_id`]: { + id: 1, + tag: 'Tag A Changed', + }, + }, + ], + sections: [ + { + id: 1, + item: { id: 1, text: 'Text A Changed' }, + collection: c.sec_text, + }, + ], + links: [ + { + id: 1, + link: 'Link A Changed', + }, + ], + }, + }); + + await SaveVersion(vendor, { + id: versionResult.id, + delta: { + links: { + create: [ + { + link: 'Link C', + }, + ], + update: [ + { + id: 2, + link: 'Link B Changed', + }, + ], + delete: [1], + }, + tags: { + create: [ + { + [`${c.tags}_id`]: { + tag: 'Tag C', + }, + }, + ], + update: [ + { + id: 2, + [`${c.tags}_id`]: { + id: 2, + tag: 'Tag B Changed', + }, + }, + ], + delete: [1], + }, + sections: { + create: [ + { + collection: c.sec_num, + item: { + num: 3, + }, + }, + ], + update: [ + { + id: 1, + collection: c.sec_text, + item: { + id: 1, + text: 'Text B Changed', + }, + }, + ], + delete: [2], + }, + }, + }); + + response = await request(getUrl(vendor)) + .get( + `/items/${c.articles}/${result.id}?version=test&fields=tags.id,tags.${c.tags}_id.id,tags.${c.tags}_id.tag,sections.id,sections.collection,sections.item.*,links.id,links.link`, + ) + .set('Authorization', `Bearer ${USER.ADMIN.TOKEN}`); + + expect(JSON.parse(response.text)).toEqual({ + data: { + links: [ + { + id: 2, + link: 'Link B Changed', + }, + { + id: null, + link: 'Link C', + }, + ], + tags: [ + { + id: 2, + [`${c.tags}_id`]: { + id: 2, + tag: 'Tag B Changed', + }, + }, + { + id: null, + [`${c.tags}_id`]: { + id: null, + tag: 'Tag C', + }, + }, + ], + sections: [ + { + id: 1, + item: { id: 1, text: 'Text B Changed' }, + collection: c.sec_text, + }, + { + id: null, + item: { id: null, num: 3 }, + collection: c.sec_num, + }, + ], + }, + }); + }); +}); + +describe('version deadlocking', () => { + it.each(vendors)('%s', async (vendor) => { + const resultA = await CreateItem(vendor, { + collection: c.articles, + item: { + title: 'Article A', + links: [ + { + link: 'Link A', + }, + { + link: 'Link B', + }, + ], + }, + }); + + const linkIds = resultA['links']; + + const resultB = await CreateItem(vendor, { + collection: c.articles, + item: { + title: 'Article B', + links: linkIds, + }, + }); + + const versionResultA = await CreateVersion(vendor, { + collection: c.articles, + item: String(resultA.id), + key: 'test', + name: 'test', + }); + + const versionResultB = await CreateVersion(vendor, { + collection: c.articles, + item: String(resultB.id), + key: 'test', + name: 'test', + }); + + await SaveVersion(vendor, { + id: versionResultA.id, + delta: { + links: { + create: [], + update: [ + { + id: linkIds[0], + link: 'Link A Changed', + }, + { + id: linkIds[1], + link: 'Link B Changed', + }, + ], + delete: [], + }, + }, + }); + + await SaveVersion(vendor, { + id: versionResultB.id, + delta: { + links: { + create: [], + update: [ + { + id: linkIds[1], + link: 'Link B Changed 2', + }, + { + id: linkIds[0], + link: 'Link A Changed 2', + }, + ], + delete: [], + }, + }, + }); + + const responseA = request(getUrl(vendor)) + .get(`/items/${c.articles}/${resultA.id}?version=test&fields=id,links.id,links.link`) + .set('Authorization', `Bearer ${USER.ADMIN.TOKEN}`); + + const responseB = request(getUrl(vendor)) + .get(`/items/${c.articles}/${resultB.id}?version=test&fields=id,links.id,links.link`) + .set('Authorization', `Bearer ${USER.ADMIN.TOKEN}`); + + const [A, B] = await Promise.all([responseA, responseB]); + + expect(JSON.parse(A.text).data).toEqual({ + id: resultA.id, + links: [ + { + id: linkIds[0], + link: 'Link A Changed', + }, + { + id: linkIds[1], + link: 'Link B Changed', + }, + ], + }); + + expect(JSON.parse(B.text).data).toEqual({ + id: resultB.id, + links: [ + { + id: linkIds[0], + link: 'Link A Changed 2', + }, + { + id: linkIds[1], + link: 'Link B Changed 2', + }, + ], + }); + }); +});