Insights 2.0 (#14096)

* query function added to list

* dashboard reading query, adding to object

* typecasting of filter vals needed still

* numbers accepting strings too

* json-to-graphql-query => devD

* fixed unneeded return in list index.ts

* stitching and calling but not actually calling

* calls on panel change

* query object += new panel before dashboard save

* uuid generated in app not api

* fixed panel ids in query

* fixed the tests I just wrote

* passing the query data down!

* list showing data

* objDiff test moved to test

* metric bug fixes + data

* dashboard logic

* time series conversion started

* timeseries GQL query almost there

* query querying

* chart loading

* aggregate handling improved

* error handling for aggregate+filter errors

* removed query on empty queryObj

* maybe more error handling

* more error handling working

* improvements to erorr handling

* stitchGQL() error return type corrected

* added string fields to COUNT

* pushing up but needs work

* not an endless recursion

* its not pretty but it works.

* throws an error

* system collections supported

* refactor to solve some errors

* loading correct

* metric function fixed

* data loading but not blocking rendering

* removed redundant code.

* relational fields

* deep nesting relations

* options.precision has a default

* relational fields fix. (thanks azri)

* the limit

* limit and time series

* range has a default

* datat to workspace

* v-if

* panels loading

* workspaces dont get data anymore

* package.json

* requested changes

* loading

* get groups util

* timeseries => script setup

* list => script setup

* metric => script setup

* label => script setup

* declare optional props

* loadingPanels: only loading spinner on loading panels

* remove unneeded parseDate!!

* applyDataToPanels tests

* -.only

* remove unneeded steps

* processQuery tests

* tests

* removed unused var

* jest.config and some queryCaller tests

* one more test

* query tests

* typo

* clean up

* fix some but not all bugs

* bugs from merge fixed

* Start cleaning up 🧹

* Refactor custom input type

* Small tweaks in list index

* Cleanup imports

* Require Query object to be returned from query prop

* Tweak return statement

* Fix imports

* Cleanup metric watch effect

* Tweaks tweaks tweaks

* Don't rely on options, simplify fetch logic

* Add paths to validation errors

* [WIP] Start handling things in the store

* Rework query fetching logic into store

* Clean up data passing

* Use composition setup for insights store

* Remove outdated

* Fix missing return

* Allow batch updating in REST API

Allows sending an array of partial items to the endpoints, updating all to their own values

* Add batch update to graphql

* Start integrating edits

* Readd clear

* Add deletion

* Add duplication

* Finish create flow

* Resolve cache refresh on panel config

* Prevent warnings about component name

* Improve loading state

* Finalize dashboard overhaul

* Add auto-refresh sidebar detail

* Add efficient panel reloading

* Set/remove errors on succeeded requests

* Move options rendering to shared

* Fix wrong imports, render options in app

* Selectively reload panels with changed variables

* Ensure newly added panels don't lose data

* Only refresh panel if data query changed

* Never use empty filter object in metric query

* Add default value support to variable panel

* Centralize no-data state

* Only reload data on var change when query is altered

* Fix build

* Fix time series order

* Remove unused utils

* Remove no-longer-used logic

* Mark batch update result as non-nullable in GraphQL schema

* Interim flows fix

* Skip parsing undefined keys

* Refresh insights dashboard when discarding changes

* Don't submit primary key when updating batch

* Handle null prop field better

* Tweak panel padding

Co-authored-by: jaycammarano <jay.cammarano@gmail.com>
Co-authored-by: Azri Kahar <42867097+azrikahar@users.noreply.github.com>
Co-authored-by: ian <licitdev@gmail.com>

api/src/services/flows.ts

@@ -35,6 +35,15 @@ export class FlowsService extends ItemsService<FlowRaw> {
 		return result;
 	}
 
+	async updateBatch(data: Partial<Item>[], opts?: MutationOptions): Promise<PrimaryKey[]> {
+		const flowManager = getFlowManager();
+
+		const result = await super.updateBatch(data, opts);
+		await flowManager.reload();
+
+		return result;
+	}
+
 	async updateMany(keys: PrimaryKey[], data: Partial<Item>, opts?: MutationOptions): Promise<PrimaryKey[]> {
 		const flowManager = getFlowManager();
 

api/src/services/graphql/index.ts

@@ -27,6 +27,7 @@ import {
 	GraphQLUnionType,
 	InlineFragmentNode,
 	IntValueNode,
+	NoSchemaIntrospectionCustomRule,
 	ObjectFieldNode,
 	ObjectValueNode,
 	SelectionNode,
@@ -46,71 +47,56 @@ import {
 import { Knex } from 'knex';
 import { flatten, get, isObject, mapKeys, merge, omit, pick, set, transform, uniq } from 'lodash';
 import ms from 'ms';
-import { clearSystemCache, getCache } from '../cache';
-import { DEFAULT_AUTH_PROVIDER } from '../constants';
-import getDatabase from '../database';
-import env from '../env';
-import { ForbiddenException, GraphQLValidationException, InvalidPayloadException } from '../exceptions';
-import { getExtensionManager } from '../extensions';
-import { AbstractServiceOptions, GraphQLParams, Item } from '../types';
-import { generateHash } from '../utils/generate-hash';
-import { getGraphQLType } from '../utils/get-graphql-type';
-import { reduceSchema } from '../utils/reduce-schema';
-import { sanitizeQuery } from '../utils/sanitize-query';
-import { validateQuery } from '../utils/validate-query';
-import { ActivityService } from './activity';
-import { AuthenticationService } from './authentication';
-import { CollectionsService } from './collections';
-import { FieldsService } from './fields';
-import { FilesService } from './files';
-import { FlowsService } from './flows';
-import { FoldersService } from './folders';
-import { ItemsService } from './items';
-import { NotificationsService } from './notifications';
-import { OperationsService } from './operations';
-import { PermissionsService } from './permissions';
-import { PresetsService } from './presets';
-import { RelationsService } from './relations';
-import { RevisionsService } from './revisions';
-import { RolesService } from './roles';
-import { ServerService } from './server';
-import { SettingsService } from './settings';
-import { SharesService } from './shares';
-import { SpecificationService } from './specifications';
-import { TFAService } from './tfa';
-import { UsersService } from './users';
-import { UtilsService } from './utils';
-import { WebhooksService } from './webhooks';
+import { clearSystemCache, getCache } from '../../cache';
+import { DEFAULT_AUTH_PROVIDER } from '../../constants';
+import getDatabase from '../../database';
+import env from '../../env';
+import { ForbiddenException, GraphQLValidationException, InvalidPayloadException } from '../../exceptions';
+import { getExtensionManager } from '../../extensions';
+import { AbstractServiceOptions, GraphQLParams, Item } from '../../types';
+import { generateHash } from '../../utils/generate-hash';
+import { getGraphQLType } from '../../utils/get-graphql-type';
+import { reduceSchema } from '../../utils/reduce-schema';
+import { sanitizeQuery } from '../../utils/sanitize-query';
+import { validateQuery } from '../../utils/validate-query';
+import { ActivityService } from '../activity';
+import { AuthenticationService } from '../authentication';
+import { CollectionsService } from '../collections';
+import { FieldsService } from '../fields';
+import { FilesService } from '../files';
+import { FlowsService } from '../flows';
+import { FoldersService } from '../folders';
+import { ItemsService } from '../items';
+import { NotificationsService } from '../notifications';
+import { OperationsService } from '../operations';
+import { PermissionsService } from '../permissions';
+import { PresetsService } from '../presets';
+import { RelationsService } from '../relations';
+import { RevisionsService } from '../revisions';
+import { RolesService } from '../roles';
+import { ServerService } from '../server';
+import { SettingsService } from '../settings';
+import { SharesService } from '../shares';
+import { SpecificationService } from '../specifications';
+import { TFAService } from '../tfa';
+import { UsersService } from '../users';
+import { UtilsService } from '../utils';
+import { WebhooksService } from '../webhooks';
 
-const GraphQLVoid = new GraphQLScalarType({
-	name: 'Void',
+import { GraphQLDate } from './types/date';
+import { GraphQLGeoJSON } from './types/geojson';
+import { GraphQLStringOrFloat } from './types/string-or-float';
+import { GraphQLVoid } from './types/void';
 
-	description: 'Represents NULL values',
+import { PrimaryKey } from '@directus/shared/types';
 
-	serialize() {
-		return null;
-	},
+import { addPathToValidationError } from './utils/add-path-to-validation-error';
 
-	parseValue() {
-		return null;
-	},
+const validationRules = Array.from(specifiedRules);
 
-	parseLiteral() {
-		return null;
-	},
-});
-
-export const GraphQLGeoJSON = new GraphQLScalarType({
-	...GraphQLJSON,
-	name: 'GraphQLGeoJSON',
-	description: 'GeoJSON value',
-});
-
-export const GraphQLDate = new GraphQLScalarType({
-	...GraphQLString,
-	name: 'Date',
-	description: 'ISO8601 Date values',
-});
+if (env.GRAPHQL_INTROSPECTION === false) {
+	validationRules.push(NoSchemaIntrospectionCustomRule);
+}
 
 /**
  * These should be ignored in the context of GraphQL, and/or are replaced by a custom resolver (for non-standard structures)
@@ -122,6 +108,7 @@ const SYSTEM_DENY_LIST = [
 	'directus_migrations',
 	'directus_sessions',
 ];
+
 const READ_ONLY = ['directus_activity', 'directus_revisions'];
 
 export class GraphQLService {
@@ -148,18 +135,9 @@ export class GraphQLService {
 	}: GraphQLParams): Promise<FormattedExecutionResult> {
 		const schema = this.getSchema();
 
-		const validationErrors = validate(schema, document, [
-			...specifiedRules,
-			(context) => ({
-				Field(node) {
-					if (env.GRAPHQL_INTROSPECTION === false && (node.name.value === '__schema' || node.name.value === '__type')) {
-						context.reportError(
-							new GraphQLError('GraphQL introspection is not allowed. The query contained __schema or __type.', [node])
-						);
-					}
-				},
-			}),
-		]);
+		const validationErrors = validate(schema, document, validationRules).map((validationError) =>
+			addPathToValidationError(validationError)
+		);
 
 		if (validationErrors.length > 0) {
 			throw new GraphQLValidationException({ graphqlErrors: validationErrors });
@@ -312,6 +290,10 @@ export class GraphQLService {
 								`update_${collection.collection}_items`
 							);
 
+							acc[`update_${collectionName}_batch`] = UpdateCollectionTypes[collection.collection].getResolver(
+								`update_${collection.collection}_batch`
+							);
+
 							acc[`update_${collectionName}_item`] = UpdateCollectionTypes[collection.collection].getResolver(
 								`update_${collection.collection}_item`
 							);
@@ -653,32 +635,33 @@ export class GraphQLService {
 				},
 			});
 
+			// Uses StringOrFloat rather than Float to support api dynamic variables (like `$NOW`)
 			const NumberFilterOperators = schemaComposer.createInputTC({
 				name: 'number_filter_operators',
 				fields: {
 					_eq: {
-						type: GraphQLFloat,
+						type: GraphQLStringOrFloat,
 					},
 					_neq: {
-						type: GraphQLFloat,
+						type: GraphQLStringOrFloat,
 					},
 					_in: {
-						type: new GraphQLList(GraphQLFloat),
+						type: new GraphQLList(GraphQLStringOrFloat),
 					},
 					_nin: {
-						type: new GraphQLList(GraphQLFloat),
+						type: new GraphQLList(GraphQLStringOrFloat),
 					},
 					_gt: {
-						type: GraphQLFloat,
+						type: GraphQLStringOrFloat,
 					},
 					_gte: {
-						type: GraphQLFloat,
+						type: GraphQLStringOrFloat,
 					},
 					_lt: {
-						type: GraphQLFloat,
+						type: GraphQLStringOrFloat,
 					},
 					_lte: {
-						type: GraphQLFloat,
+						type: GraphQLStringOrFloat,
 					},
 					_null: {
 						type: GraphQLBoolean,
@@ -1143,6 +1126,27 @@ export class GraphQLService {
 								await self.resolveMutation(args, info),
 						});
 					} else {
+						UpdateCollectionTypes[collection.collection].addResolver({
+							name: `update_${collection.collection}_batch`,
+							type: collectionIsReadable
+								? new GraphQLNonNull(
+										new GraphQLList(new GraphQLNonNull(ReadCollectionTypes[collection.collection].getType()))
+								  )
+								: GraphQLBoolean,
+							args: {
+								...(collectionIsReadable
+									? ReadCollectionTypes[collection.collection].getResolver(collection.collection).getArgs()
+									: {}),
+								data: [
+									toInputObjectType(UpdateCollectionTypes[collection.collection]).setTypeName(
+										`update_${collection.collection}_input`
+									).NonNull,
+								],
+							},
+							resolve: async ({ args, info }: { args: Record<string, any>; info: GraphQLResolveInfo }) =>
+								await self.resolveMutation(args, info),
+						});
+
 						UpdateCollectionTypes[collection.collection].addResolver({
 							name: `update_${collection.collection}_items`,
 							type: collectionIsReadable
@@ -1290,12 +1294,15 @@ export class GraphQLService {
 		const query = this.getQuery(args, selections || [], info.variableValues);
 
 		const singleton =
+			collection.endsWith('_batch') === false &&
 			collection.endsWith('_items') === false &&
 			collection.endsWith('_item') === false &&
 			collection in this.schema.collections;
 
-		const single = collection.endsWith('_items') === false;
+		const single = collection.endsWith('_items') === false && collection.endsWith('_batch') === false;
+		const batchUpdate = action === 'update' && collection.endsWith('_batch');
 
+		if (collection.endsWith('_batch')) collection = collection.slice(0, -6);
 		if (collection.endsWith('_items')) collection = collection.slice(0, -6);
 		if (collection.endsWith('_item')) collection = collection.slice(0, -5);
 
@@ -1329,7 +1336,14 @@ export class GraphQLService {
 				}
 
 				if (action === 'update') {
-					const keys = await service.updateMany(args.ids, args.data);
+					const keys: PrimaryKey[] = [];
+
+					if (batchUpdate) {
+						keys.push(...(await service.updateBatch(args.data)));
+					} else {
+						keys.push(...(await service.updateMany(args.ids, args.data)));
+					}
+
 					return hasQuery ? await service.readMany(keys, query) : true;
 				}
 

api/src/services/graphql/types/date.ts

@@ -0,0 +1,7 @@
+import { GraphQLString, GraphQLScalarType } from 'graphql';
+
+export const GraphQLDate = new GraphQLScalarType({
+	...GraphQLString,
+	name: 'Date',
+	description: 'ISO8601 Date values',
+});

api/src/services/graphql/types/geojson.ts

@@ -0,0 +1,8 @@
+import { GraphQLScalarType } from 'graphql';
+import { GraphQLJSON } from 'graphql-compose';
+
+export const GraphQLGeoJSON = new GraphQLScalarType({
+	...GraphQLJSON,
+	name: 'GraphQLGeoJSON',
+	description: 'GeoJSON value',
+});

api/src/services/graphql/types/string-or-float.ts

@@ -0,0 +1,35 @@
+import { GraphQLScalarType, Kind } from 'graphql';
+
+/**
+ * Adopted from https://kamranicus.com/handling-multiple-scalar-types-in-graphql/
+ */
+
+export const GraphQLStringOrFloat = new GraphQLScalarType({
+	name: 'GraphQLStringOrFloat',
+	description: 'A Float or a String',
+	serialize(value) {
+		if (typeof value !== 'string' && typeof value !== 'number') {
+			throw new Error('Value must be either a String or a Float');
+		}
+
+		return value;
+	},
+	parseValue(value) {
+		if (typeof value !== 'string' && typeof value !== 'number') {
+			throw new Error('Value must be either a String or a Float');
+		}
+
+		return value;
+	},
+	parseLiteral(ast) {
+		switch (ast.kind) {
+			case Kind.INT:
+			case Kind.FLOAT:
+				return Number(ast.value);
+			case Kind.STRING:
+				return ast.value;
+			default:
+				throw new Error('Value must be either a String or a Float');
+		}
+	},
+});

api/src/services/graphql/types/void.ts

@@ -0,0 +1,19 @@
+import { GraphQLScalarType } from 'graphql';
+
+export const GraphQLVoid = new GraphQLScalarType({
+	name: 'Void',
+
+	description: 'Represents NULL values',
+
+	serialize() {
+		return null;
+	},
+
+	parseValue() {
+		return null;
+	},
+
+	parseLiteral() {
+		return null;
+	},
+});

api/src/services/graphql/utils/add-path-to-validation-error.ts

@@ -0,0 +1,21 @@
+import { GraphQLError, Token, locatedError } from 'graphql';
+
+export function addPathToValidationError(validationError: GraphQLError): GraphQLError {
+	const token = validationError.nodes?.[0]?.loc?.startToken;
+
+	if (!token) return validationError;
+
+	let prev: Token | null = token;
+
+	const queryRegex = /query_[A-Za-z0-9]{8}/;
+
+	while (prev) {
+		if (prev.kind === 'Name' && prev.value && queryRegex.test(prev.value)) {
+			return locatedError(validationError, validationError.nodes, [prev.value]);
+		}
+
+		prev = prev.prev;
+	}
+
+	return locatedError(validationError, validationError.nodes);
+}

api/src/services/import-export.ts

@@ -1,5 +1,5 @@
 import { Accountability, Query, SchemaOverview } from '@directus/shared/types';
-import { toArray } from '@directus/shared/utils';
+import { parseJSON, toArray } from '@directus/shared/utils';
 import { queue } from 'async';
 import csv from 'csv-parser';
 import destroyStream from 'destroy';
@@ -22,7 +22,6 @@ import {
 import logger from '../logger';
 import { AbstractServiceOptions, File } from '../types';
 import { getDateFormatted } from '../utils/get-date-formatted';
-import { parseJSON } from '../utils/parse-json';
 import { FilesService } from './files';
 import { ItemsService } from './items';
 import { NotificationsService } from './notifications';

api/src/services/items.ts

@@ -1,20 +1,20 @@
 import { Accountability, Action, PermissionsAction, Query, SchemaOverview } from '@directus/shared/types';
 import Keyv from 'keyv';
 import { Knex } from 'knex';
-import { assign, clone, cloneDeep, pick, without } from 'lodash';
+import { assign, clone, cloneDeep, omit, pick, without } from 'lodash';
 import { getCache } from '../cache';
 import getDatabase from '../database';
 import runAST from '../database/run-ast';
 import emitter from '../emitter';
 import env from '../env';
-import { ForbiddenException } from '../exceptions';
+import { ForbiddenException, InvalidPayloadException } from '../exceptions';
 import { translateDatabaseError } from '../exceptions/database/translate';
 import { AbstractService, AbstractServiceOptions, Item as AnyItem, MutationOptions, PrimaryKey } from '../types';
 import getASTFromQuery from '../utils/get-ast-from-query';
+import { validateKeys } from '../utils/validate-keys';
 import { AuthorizationService } from './authorization';
 import { ActivityService, RevisionsService } from './index';
 import { PayloadService } from './payload';
-import { validateKeys } from '../utils/validate-keys';
 
 export type QueryOptions = {
 	stripNonRequested?: boolean;
@@ -371,7 +371,31 @@ export class ItemsService<Item extends AnyItem = AnyItem> implements AbstractSer
 	}
 
 	/**
-	 * Update many items by primary key
+	 * Update multiple items in a single transaction
+	 */
+	async updateBatch(data: Partial<Item>[], opts?: MutationOptions): Promise<PrimaryKey[]> {
+		const primaryKeyField = this.schema.collections[this.collection].primary;
+
+		const keys: PrimaryKey[] = [];
+
+		await this.knex.transaction(async (trx) => {
+			const service = new ItemsService(this.collection, {
+				accountability: this.accountability,
+				knex: trx,
+				schema: this.schema,
+			});
+
+			for (const item of data) {
+				if (!item[primaryKeyField]) throw new InvalidPayloadException(`Item in update misses primary key.`);
+				keys.push(await service.updateOne(item[primaryKeyField]!, omit(item, primaryKeyField), opts));
+			}
+		});
+
+		return keys;
+	}
+
+	/**
+	 * Update many items by primary key, setting all items to the same change
 	 */
 	async updateMany(keys: PrimaryKey[], data: Partial<Item>, opts?: MutationOptions): Promise<PrimaryKey[]> {
 		const primaryKeyField = this.schema.collections[this.collection].primary;

api/src/services/operations.ts

@@ -35,6 +35,15 @@ export class OperationsService extends ItemsService<OperationRaw> {
 		return result;
 	}
 
+	async updateBatch(data: Partial<Item>[], opts?: MutationOptions): Promise<PrimaryKey[]> {
+		const flowManager = getFlowManager();
+
+		const result = await super.updateBatch(data, opts);
+		await flowManager.reload();
+
+		return result;
+	}
+
 	async updateMany(keys: PrimaryKey[], data: Partial<Item>, opts?: MutationOptions): Promise<PrimaryKey[]> {
 		const flowManager = getFlowManager();
 

api/src/services/payload.ts

@@ -1,5 +1,5 @@
 import { Accountability, Query, SchemaOverview } from '@directus/shared/types';
-import { toArray } from '@directus/shared/utils';
+import { parseJSON, toArray } from '@directus/shared/utils';
 import { format } from 'date-fns';
 import { unflatten } from 'flat';
 import Joi from 'joi';
@@ -12,7 +12,6 @@ import { getHelpers, Helpers } from '../database/helpers';
 import { ForbiddenException, InvalidPayloadException } from '../exceptions';
 import { AbstractServiceOptions, Alterations, Item, PrimaryKey } from '../types';
 import { generateHash } from '../utils/generate-hash';
-import { parseJSON } from '../utils/parse-json';
 import { ItemsService } from './items';
 
 type Action = 'create' | 'read' | 'update';

api/src/services/permissions.ts

@@ -102,6 +102,12 @@ export class PermissionsService extends ItemsService {
 		return res;
 	}
 
+	async updateBatch(data: Partial<Item>[], opts?: MutationOptions) {
+		const res = await super.updateBatch(data, opts);
+		await clearSystemCache();
+		return res;
+	}
+
 	async updateMany(keys: PrimaryKey[], data: Partial<Item>, opts?: MutationOptions) {
 		const res = await super.updateMany(keys, data, opts);
 		await clearSystemCache();

api/src/services/roles.ts

@@ -79,6 +79,19 @@ export class RolesService extends ItemsService {
 		return super.updateOne(key, data, opts);
 	}
 
+	async updateBatch(data: Record<string, any>[], opts?: MutationOptions): Promise<PrimaryKey[]> {
+		const primaryKeyField = this.schema.collections[this.collection].primary;
+
+		const keys = data.map((item) => item[primaryKeyField]);
+		const setsToNoAdmin = data.some((item) => item.admin_access === false);
+
+		if (setsToNoAdmin) {
+			await this.checkForOtherAdminRoles(keys);
+		}
+
+		return super.updateBatch(data, opts);
+	}
+
 	async updateMany(keys: PrimaryKey[], data: Record<string, any>, opts?: MutationOptions): Promise<PrimaryKey[]> {
 		if ('admin_access' in data && data.admin_access === false) {
 			await this.checkForOtherAdminRoles(keys);

api/src/services/users.ts

@@ -182,6 +182,27 @@ export class UsersService extends ItemsService {
 		return key;
 	}
 
+	async updateBatch(data: Partial<Item>[], opts?: MutationOptions): Promise<PrimaryKey[]> {
+		const primaryKeyField = this.schema.collections[this.collection].primary;
+
+		const keys: PrimaryKey[] = [];
+
+		await this.knex.transaction(async (trx) => {
+			const service = new UsersService({
+				accountability: this.accountability,
+				knex: trx,
+				schema: this.schema,
+			});
+
+			for (const item of data) {
+				if (!item[primaryKeyField]) throw new InvalidPayloadException(`User in update misses primary key.`);
+				keys.push(await service.updateOne(item[primaryKeyField]!, item, opts));
+			}
+		});
+
+		return keys;
+	}
+
 	/**
 	 * Update many users by primary key
 	 */