github/directus
1
0
Fork 0
mirror of https://github.com/directus/directus.git synced 2026-02-01 10:34:58 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'aggregation' into insights

Browse Source
This commit is contained in:
rijkvanzanten
2021-06-07 09:33:25 -04:00
parent d61dacbe34 ecce005226
commit c6b154e60d
141 changed files with 9157 additions and 5415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
api/src/database/index.ts
View File

@@ -1,60 +1,98 @@
import SchemaInspector from '@directus/schema';
import dotenv from 'dotenv';
import { knex, Knex } from 'knex';
import path from 'path';
import { performance } from 'perf_hooks';
import env from '../env';
import logger from '../logger';
import { getConfigFromEnv } from '../utils/get-config-from-env';
import { validateEnv } from '../utils/validate-env';
dotenv.config({ path: path.resolve(__dirname, '../../', '.env') });
let database: Knex | null = null;
let inspector: ReturnType<typeof SchemaInspector> | null = null;
const connectionConfig: Record<string, any> = getConfigFromEnv('DB_', [
'DB_CLIENT',
'DB_SEARCH_PATH',
'DB_CONNECTION_STRING',
'DB_POOL',
]);
export default function getDatabase(): Knex {
if (database) {
return database;
}
const poolConfig = getConfigFromEnv('DB_POOL');
const connectionConfig: Record<string, any> = getConfigFromEnv('DB_', [
'DB_CLIENT',
'DB_SEARCH_PATH',
'DB_CONNECTION_STRING',
'DB_POOL',
]);
validateEnv(['DB_CLIENT']);
const poolConfig = getConfigFromEnv('DB_POOL');
const knexConfig: Knex.Config = {
client: env.DB_CLIENT,
searchPath: env.DB_SEARCH_PATH,
connection: env.DB_CONNECTION_STRING || connectionConfig,
log: {
warn: (msg) => logger.warn(msg),
error: (msg) => logger.error(msg),
deprecate: (msg) => logger.info(msg),
debug: (msg) => logger.debug(msg),
},
pool: poolConfig,
};
const requiredEnvVars = ['DB_CLIENT'];
if (env.DB_CLIENT === 'sqlite3') {
knexConfig.useNullAsDefault = true;
poolConfig.afterCreate = (conn: any, cb: any) => {
conn.run('PRAGMA foreign_keys = ON', cb);
if (env.DB_CLIENT && env.DB_CLIENT === 'sqlite3') {
requiredEnvVars.push('DB_FILENAME');
} else if (env.DB_CLIENT && env.DB_CLIENT === 'oracledb') {
requiredEnvVars.push('DB_USER', 'DB_PASSWORD', 'DB_CONNECT_STRING');
} else {
if (env.DB_CLIENT === 'pg') {
if (!env.DB_CONNECTION_STRING) {
requiredEnvVars.push('DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_USER');
}
} else {
requiredEnvVars.push('DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_USER', 'DB_PASSWORD');
}
}
validateEnv(requiredEnvVars);
const knexConfig: Knex.Config = {
client: env.DB_CLIENT,
searchPath: env.DB_SEARCH_PATH,
connection: env.DB_CONNECTION_STRING || connectionConfig,
log: {
warn: (msg) => logger.warn(msg),
error: (msg) => logger.error(msg),
deprecate: (msg) => logger.info(msg),
debug: (msg) => logger.debug(msg),
},
pool: poolConfig,
};
if (env.DB_CLIENT === 'sqlite3') {
knexConfig.useNullAsDefault = true;
poolConfig.afterCreate = (conn: any, cb: any) => {
conn.run('PRAGMA foreign_keys = ON', cb);
};
}
database = knex(knexConfig);
const times: Record<string, number> = {};
database
.on('query', (queryInfo) => {
times[queryInfo.__knexUid] = performance.now();
})
.on('query-response', (response, queryInfo) => {
const delta = performance.now() - times[queryInfo.__knexUid];
logger.trace(`[${delta.toFixed(3)}ms] ${queryInfo.sql} [${queryInfo.bindings.join(', ')}]`);
delete times[queryInfo.__knexUid];
});
return database;
}
const database = knex(knexConfig);
export function getSchemaInspector(): ReturnType<typeof SchemaInspector> {
if (inspector) {
return inspector;
}
const times: Record<string, number> = {};
const database = getDatabase();
database
.on('query', (queryInfo) => {
times[queryInfo.__knexUid] = performance.now();
})
.on('query-response', (response, queryInfo) => {
const delta = performance.now() - times[queryInfo.__knexUid];
logger.trace(`[${delta.toFixed(3)}ms] ${queryInfo.sql} [${queryInfo.bindings.join(', ')}]`);
});
inspector = SchemaInspector(database);
return inspector;
}
export async function hasDatabaseConnection(): Promise<boolean> {
const database = getDatabase();
try {
if (env.DB_CLIENT === 'oracledb') {
await database.raw('select 1 from DUAL');
@@ -77,13 +115,11 @@ export async function validateDBConnection(): Promise<void> {
}
}
export const schemaInspector = SchemaInspector(database);
export async function isInstalled(): Promise<boolean> {
const inspector = getSchemaInspector();
// The existence of a directus_collections table alone isn't a "proper" check to see if everything
// is installed correctly of course, but it's safe enough to assume that this collection only
// exists when using the installer CLI.
return await schemaInspector.hasTable('directus_collections');
return await inspector.hasTable('directus_collections');
}
export default database;

139
api/src/database/migrations/20210518A-add-foreign-key-constraints.ts
View File

@@ -1,14 +1,15 @@
import { Knex } from 'knex';
import SchemaInspector from 'knex-schema-inspector';
import { schemaInspector } from '..';
import logger from '../../logger';
import { RelationMeta } from '../../types';
import { getDefaultIndexName } from '../../utils/get-default-index-name';
export async function up(knex: Knex): Promise<void> {
const inspector = SchemaInspector(knex);
const foreignKeys = await inspector.foreignKeys();
const relations = await knex
.select<RelationMeta[]>('many_collection', 'many_field', 'one_collection')
.select<RelationMeta[]>('id', 'many_collection', 'many_field', 'one_collection')
.from('directus_relations');
const constraintsToAdd = relations.filter((relation) => {
@@ -18,45 +19,82 @@ export async function up(knex: Knex): Promise<void> {
return exists === false;
});
await knex.transaction(async (trx) => {
for (const constraint of constraintsToAdd) {
if (!constraint.one_collection) continue;
const corruptedRelations: number[] = [];
const currentPrimaryKeyField = await schemaInspector.primary(constraint.many_collection);
const relatedPrimaryKeyField = await schemaInspector.primary(constraint.one_collection);
if (!currentPrimaryKeyField || !relatedPrimaryKeyField) continue;
for (const constraint of constraintsToAdd) {
if (!constraint.one_collection) continue;
const rowsWithIllegalFKValues = await trx
.select(`${constraint.many_collection}.${currentPrimaryKeyField}`)
.from(constraint.many_collection)
.leftJoin(
constraint.one_collection,
`${constraint.many_collection}.${constraint.many_field}`,
`${constraint.one_collection}.${relatedPrimaryKeyField}`
)
.whereNull(`${constraint.one_collection}.${relatedPrimaryKeyField}`);
if (
(await inspector.hasTable(constraint.many_collection)) === false ||
(await inspector.hasTable(constraint.one_collection)) === false
) {
logger.warn(
`Ignoring ${constraint.many_collection}.${constraint.many_field}<->${constraint.one_collection}. Tables don't exist.`
);
if (rowsWithIllegalFKValues.length > 0) {
const ids: (string | number)[] = rowsWithIllegalFKValues.map<string | number>(
(row) => row[currentPrimaryKeyField]
);
corruptedRelations.push(constraint.id);
continue;
}
await trx(constraint.many_collection)
const currentPrimaryKeyField = await inspector.primary(constraint.many_collection);
const relatedPrimaryKeyField = await inspector.primary(constraint.one_collection);
if (constraint.many_field === currentPrimaryKeyField) {
logger.warn(
`Illegal relationship ${constraint.many_collection}.${constraint.many_field}<->${constraint.one_collection} encountered. Many field equals collections primary key.`
);
corruptedRelations.push(constraint.id);
continue;
}
if (!currentPrimaryKeyField || !relatedPrimaryKeyField) continue;
const rowsWithIllegalFKValues = await knex
.select(`main.${currentPrimaryKeyField}`)
.from({ main: constraint.many_collection })
.leftJoin(
{ related: constraint.one_collection },
`main.${constraint.many_field}`,
`related.${relatedPrimaryKeyField}`
)
.whereNull(`related.${relatedPrimaryKeyField}`);
if (rowsWithIllegalFKValues.length > 0) {
const ids: (string | number)[] = rowsWithIllegalFKValues.map<string | number>(
(row) => row[currentPrimaryKeyField]
);
try {
await knex(constraint.many_collection)
.update({ [constraint.many_field]: null })
.whereIn(currentPrimaryKeyField, ids);
} catch (err) {
logger.error(
`${constraint.many_collection}.${constraint.many_field} contains illegal foreign keys which couldn't be set to NULL. Please fix these references and rerun this migration to complete the upgrade.`
);
if (ids.length < 25) {
logger.error(`Items with illegal foreign keys: ${ids.join(', ')}`);
} else {
logger.error(`Items with illegal foreign keys: ${ids.slice(0, 25).join(', ')} and ${ids.length} others`);
}
throw 'Migration aborted';
}
}
// Can't reliably have circular cascade
const action = constraint.many_collection === constraint.one_collection ? 'NO ACTION' : 'SET NULL';
// Can't reliably have circular cascade
const action = constraint.many_collection === constraint.one_collection ? 'NO ACTION' : 'SET NULL';
// MySQL doesn't accept FKs from `int` to `int unsigned`. `knex` defaults `.increments()`
// to `unsigned`, but defaults `.integer()` to `int`. This means that created m2o fields
// have the wrong type. This step will force the m2o `int` field into `unsigned`, but only
// if both types are integers, and only if we go from `int` to `int unsigned`.
const columnInfo = await schemaInspector.columnInfo(constraint.many_collection, constraint.many_field);
const relatedColumnInfo = await schemaInspector.columnInfo(constraint.one_collection!, relatedPrimaryKeyField);
// MySQL doesn't accept FKs from `int` to `int unsigned`. `knex` defaults `.increments()`
// to `unsigned`, but defaults `.integer()` to `int`. This means that created m2o fields
// have the wrong type. This step will force the m2o `int` field into `unsigned`, but only
// if both types are integers, and only if we go from `int` to `int unsigned`.
const columnInfo = await inspector.columnInfo(constraint.many_collection, constraint.many_field);
const relatedColumnInfo = await inspector.columnInfo(constraint.one_collection!, relatedPrimaryKeyField);
await trx.schema.alterTable(constraint.many_collection, (table) => {
try {
await knex.schema.alterTable(constraint.many_collection, (table) => {
if (
columnInfo.data_type !== relatedColumnInfo.data_type &&
columnInfo.data_type === 'int' &&
@@ -65,21 +103,48 @@ export async function up(knex: Knex): Promise<void> {
table.specificType(constraint.many_field, 'int unsigned').alter();
}
const indexName = getDefaultIndexName('foreign', constraint.many_collection, constraint.many_field);
table
.foreign(constraint.many_field)
.foreign(constraint.many_field, indexName)
.references(relatedPrimaryKeyField)
.inTable(constraint.one_collection!)
.onDelete(action);
});
} catch (err) {
logger.warn(
`Couldn't add foreign key constraint for ${constraint.many_collection}.${constraint.many_field}<->${constraint.one_collection}`
);
logger.warn(err);
}
});
}
if (corruptedRelations.length > 0) {
logger.warn(
`Encountered one or more corrupted relationships. Please check the following rows in "directus_relations": ${corruptedRelations.join(
', '
)}`
);
}
}
export async function down(knex: Knex): Promise<void> {
const relations = await knex.select<RelationMeta[]>('many_collection', 'many_field').from('directus_relations');
const relations = await knex
.select<RelationMeta[]>('many_collection', 'many_field', 'one_collection')
.from('directus_relations');
for (const relation of relations) {
await knex.schema.alterTable(relation.many_collection, (table) => {
table.dropForeign([relation.many_field]);
});
if (!relation.one_collection) continue;
try {
await knex.schema.alterTable(relation.many_collection, (table) => {
table.dropForeign([relation.many_field]);
});
} catch (err) {
logger.warn(
`Couldn't drop foreign key constraint for ${relation.many_collection}.${relation.many_field}<->${relation.one_collection}`
);
logger.warn(err);
}
}
}

83
api/src/database/migrations/20210519A-add-system-fk-triggers.ts
View File

@@ -1,4 +1,5 @@
import { Knex } from 'knex';
import logger from '../../logger';
/**
* Things to keep in mind:
@@ -80,22 +81,84 @@ const updates = [
export async function up(knex: Knex): Promise<void> {
for (const update of updates) {
await knex.schema.alterTable(update.table, (table) => {
for (const constraint of update.constraints) {
table.dropForeign([constraint.column]);
table.foreign(constraint.column).references(constraint.references).onDelete(constraint.on_delete);
for (const constraint of update.constraints) {
try {
await knex.schema.alterTable(update.table, (table) => {
table.dropForeign([constraint.column]);
});
} catch (err) {
logger.warn(`Couldn't drop foreign key ${update.table}.${constraint.column}->${constraint.references}`);
logger.warn(err);
}
});
/**
* MySQL won't delete the index when you drop the foreign key constraint. Gotta make
* sure to clean those up as well
*/
if (knex.client.constructor.name === 'Client_MySQL') {
try {
await knex.schema.alterTable(update.table, (table) => {
// Knex uses a default convention for index names: `table_column_type`
table.dropIndex([constraint.column], `${update.table}_${constraint.column}_foreign`);
});
} catch (err) {
logger.warn(
`Couldn't clean up index for foreign key ${update.table}.${constraint.column}->${constraint.references}`
);
logger.warn(err);
}
}
try {
await knex.schema.alterTable(update.table, (table) => {
table.foreign(constraint.column).references(constraint.references).onDelete(constraint.on_delete);
});
} catch (err) {
logger.warn(`Couldn't add foreign key to ${update.table}.${constraint.column}->${constraint.references}`);
logger.warn(err);
}
}
}
}
export async function down(knex: Knex): Promise<void> {
for (const update of updates) {
await knex.schema.alterTable(update.table, (table) => {
for (const constraint of update.constraints) {
table.dropForeign([constraint.column]);
table.foreign(constraint.column).references(constraint.references);
for (const constraint of update.constraints) {
try {
await knex.schema.alterTable(update.table, (table) => {
table.dropForeign([constraint.column]);
});
} catch (err) {
logger.warn(`Couldn't drop foreign key ${update.table}.${constraint.column}->${constraint.references}`);
logger.warn(err);
}
});
/**
* MySQL won't delete the index when you drop the foreign key constraint. Gotta make
* sure to clean those up as well
*/
if (knex.client.constructor.name === 'Client_MySQL') {
try {
await knex.schema.alterTable(update.table, (table) => {
// Knex uses a default convention for index names: `table_column_type`
table.dropIndex([constraint.column], `${update.table}_${constraint.column}_foreign`);
});
} catch (err) {
logger.warn(
`Couldn't clean up index for foreign key ${update.table}.${constraint.column}->${constraint.references}`
);
logger.warn(err);
}
}
try {
await knex.schema.alterTable(update.table, (table) => {
table.foreign(constraint.column).references(constraint.references);
});
} catch (err) {
logger.warn(`Couldn't add foreign key to ${update.table}.${constraint.column}->${constraint.references}`);
logger.warn(err);
}
}
}
}

13
api/src/database/migrations/20210521A-add-collections-icon-color.ts Normal file
View File

@@ -0,0 +1,13 @@
import { Knex } from 'knex';
export async function up(knex: Knex): Promise<void> {
await knex.schema.alterTable('directus_collections', (table) => {
table.string('color').nullable();
});
}
export async function down(knex: Knex): Promise<void> {
await knex.schema.alterTable('directus_collections', (table) => {
table.dropColumn('color');
});
}

6
api/src/database/run-ast.ts
View File

@@ -5,7 +5,7 @@ import { Item, Query, SchemaOverview } from '../types';
import { AST, FieldNode, NestedCollectionNode } from '../types/ast';
import applyQuery from '../utils/apply-query';
import { toArray } from '../utils/to-array';
import database from './index';
import getDatabase from './index';
type RunASTOptions = {
/**
@@ -39,7 +39,7 @@ export default async function runAST(
): Promise<null | Item | Item[]> {
const ast = cloneDeep(originalAST);
const knex = options?.knex || database;
const knex = options?.knex || getDatabase();
if (ast.type === 'm2a') {
const results: { [collection: string]: null | Item | Item[] } = {};
@@ -298,7 +298,7 @@ function mergeWithParentItems(
});
// We re-apply the requested limit here. This forces the _n_ nested items per parent concept
if (nested) {
if (nested && nestedNode.query.limit !== -1) {
itemChildren = itemChildren.slice(0, nestedNode.query.limit ?? 100);
}

14
api/src/database/system-data/fields/collections.yaml
View File

@@ -18,16 +18,22 @@ fields:
readonly: true
width: half
- field: note
interface: input
options:
placeholder: A description of this collection...
width: half
- field: icon
interface: select-icon
options:
width: half
- field: note
interface: input
- field: color
interface: select-color
options:
placeholder: A description of this collection...
width: full
placeholder: Choose a color...
width: half
- field: display_template
interface: system-display-template