From 1c3e94d8308e85a03dc5da170f3fc33bfc9b3196 Mon Sep 17 00:00:00 2001 From: Rijk van Zanten Date: Thu, 17 Mar 2022 15:43:45 -0400 Subject: [PATCH] Add new export experience (#12201) * Use script setup * Start on export dialog * Use new system field interface, replace limit with numeric input * Set placeholder * Add sort config * Use folder picker, correct layoutQuery use * Add local download button * Allow writing exports to file * Add notification after export * Fix sort config, use new export endpoint * Setup notification hints * Add information notice * Fix local limit, cancel button * Add (basic) docs for export functionality * Fix json export file format * Implement xml batch stitching * Resolve review points --- api/package.json | 1 + api/src/controllers/utils.ts | 29 +- api/src/env.ts | 2 + api/src/middleware/respond.ts | 36 +- api/src/services/import-export.ts | 345 +++++++++++ api/src/services/import.ts | 138 ----- api/src/services/index.ts | 2 +- api/src/utils/get-date-formatted.ts | 11 + .../interfaces/_system/system-fields/index.ts | 12 + .../_system/system-fields/system-fields.vue | 87 +++ app/src/lang/translations/en-US.yaml | 17 +- app/src/modules/content/routes/collection.vue | 1 + .../files/components/navigation-folder.vue | 2 +- app/src/modules/files/routes/collection.vue | 3 +- app/src/modules/files/routes/item.vue | 2 +- app/src/modules/users/routes/collection.vue | 1 + .../components/export-sidebar-detail.vue | 580 ++++++++++++------ .../folder-picker-list-item.vue | 0 .../folder-picker}/folder-picker.vue | 11 +- docs/configuration/config-options.md | 1 + docs/reference/system/utilities.md | 69 +++ package-lock.json | 144 +++-- .../shared/src/composables/use-collection.ts | 6 +- packages/shared/src/types/collection.ts | 7 + packages/shared/src/types/notifications.ts | 4 +- 25 files changed, 1095 insertions(+), 416 deletions(-) create mode 100644 api/src/services/import-export.ts delete mode 100644 api/src/services/import.ts create mode 100644 api/src/utils/get-date-formatted.ts create mode 100644 app/src/interfaces/_system/system-fields/index.ts create mode 100644 app/src/interfaces/_system/system-fields/system-fields.vue rename app/src/{modules/files/components => views/private/components/folder-picker}/folder-picker-list-item.vue (100%) rename app/src/{modules/files/components => views/private/components/folder-picker}/folder-picker.vue (91%) diff --git a/api/package.json b/api/package.json index ca73988bb1..1ce167ab7b 100644 --- a/api/package.json +++ b/api/package.json @@ -153,6 +153,7 @@ "sharp": "^0.29.0", "stream-json": "^1.7.1", "supertest": "^6.1.6", + "tmp-promise": "^3.0.3", "update-check": "^1.5.4", "uuid": "^8.3.2", "uuid-validate": "0.0.3", diff --git a/api/src/controllers/utils.ts b/api/src/controllers/utils.ts index d1e8bb5848..1627f30d0b 100644 --- a/api/src/controllers/utils.ts +++ b/api/src/controllers/utils.ts @@ -10,7 +10,7 @@ import { } from '../exceptions'; import collectionExists from '../middleware/collection-exists'; import { respond } from '../middleware/respond'; -import { RevisionsService, UtilsService, ImportService } from '../services'; +import { RevisionsService, UtilsService, ImportService, ExportService } from '../services'; import asyncHandler from '../utils/async-handler'; import Busboy, { BusboyHeaders } from 'busboy'; import { flushCaches } from '../cache'; @@ -136,6 +136,33 @@ router.post( }) ); +router.post( + '/export/:collection', + collectionExists, + asyncHandler(async (req, res, next) => { + if (!req.body.query) { + throw new InvalidPayloadException(`"query" is required.`); + } + + if (!req.body.format) { + throw new InvalidPayloadException(`"format" is required.`); + } + + const service = new ExportService({ + accountability: req.accountability, + schema: req.schema, + }); + + // We're not awaiting this, as it's supposed to run async in the background + service.exportToFile(req.params.collection, req.body.query, req.body.format, { + file: req.body.file, + }); + + return next(); + }), + respond +); + router.post( '/cache/clear', asyncHandler(async (req, res) => { diff --git a/api/src/env.ts b/api/src/env.ts index 0a35b3d27f..b776f49cbe 100644 --- a/api/src/env.ts +++ b/api/src/env.ts @@ -82,6 +82,8 @@ const defaults: Record = { SERVE_APP: true, RELATIONAL_BATCH_SIZE: 25000, + + EXPORT_BATCH_SIZE: 5000, }; // Allows us to force certain environment variable into a type, instead of relying diff --git a/api/src/middleware/respond.ts b/api/src/middleware/respond.ts index d48b1ad458..7632120bf5 100644 --- a/api/src/middleware/respond.ts +++ b/api/src/middleware/respond.ts @@ -1,14 +1,13 @@ import { RequestHandler } from 'express'; -import { Transform, transforms } from 'json2csv'; import ms from 'ms'; -import { PassThrough } from 'stream'; import { getCache } from '../cache'; import env from '../env'; import asyncHandler from '../utils/async-handler'; import { getCacheKey } from '../utils/get-cache-key'; -import { parse as toXML } from 'js2xmlparser'; import { getCacheControlHeader } from '../utils/get-cache-headers'; import logger from '../logger'; +import { ExportService } from '../services'; +import { getDateFormatted } from '../utils/get-date-formatted'; export const respond: RequestHandler = asyncHandler(async (req, res) => { const { cache } = getCache(); @@ -38,6 +37,8 @@ export const respond: RequestHandler = asyncHandler(async (req, res) => { } if (req.sanitizedQuery.export) { + const exportService = new ExportService({ accountability: req.accountability, schema: req.schema }); + let filename = ''; if (req.collection) { @@ -51,30 +52,19 @@ export const respond: RequestHandler = asyncHandler(async (req, res) => { if (req.sanitizedQuery.export === 'json') { res.attachment(`${filename}.json`); res.set('Content-Type', 'application/json'); - return res.status(200).send(JSON.stringify(res.locals.payload?.data || null, null, '\t')); + return res.status(200).send(exportService.transform(res.locals.payload?.data, 'json')); } if (req.sanitizedQuery.export === 'xml') { res.attachment(`${filename}.xml`); res.set('Content-Type', 'text/xml'); - return res.status(200).send(toXML('data', res.locals.payload?.data)); + return res.status(200).send(exportService.transform(res.locals.payload?.data, 'xml')); } if (req.sanitizedQuery.export === 'csv') { res.attachment(`${filename}.csv`); res.set('Content-Type', 'text/csv'); - const stream = new PassThrough(); - - if (!res.locals.payload?.data || res.locals.payload.data.length === 0) { - stream.end(Buffer.from('')); - return stream.pipe(res); - } else { - stream.end(Buffer.from(JSON.stringify(res.locals.payload.data), 'utf-8')); - const json2csv = new Transform({ - transforms: [transforms.flatten({ separator: '.' })], - }); - return stream.pipe(json2csv).pipe(res); - } + return res.status(200).send(exportService.transform(res.locals.payload?.data, 'csv')); } } @@ -86,15 +76,3 @@ export const respond: RequestHandler = asyncHandler(async (req, res) => { return res.status(204).end(); } }); - -function getDateFormatted() { - const date = new Date(); - - let month = String(date.getMonth() + 1); - if (month.length === 1) month = '0' + month; - - let day = String(date.getDate()); - if (day.length === 1) day = '0' + day; - - return `${date.getFullYear()}-${month}-${day} at ${date.getHours()}.${date.getMinutes()}.${date.getSeconds()}`; -} diff --git a/api/src/services/import-export.ts b/api/src/services/import-export.ts new file mode 100644 index 0000000000..417f8afada --- /dev/null +++ b/api/src/services/import-export.ts @@ -0,0 +1,345 @@ +import { Knex } from 'knex'; +import getDatabase from '../database'; +import { AbstractServiceOptions, File } from '../types'; +import { Accountability, Query, SchemaOverview } from '@directus/shared/types'; +import { + ForbiddenException, + InvalidPayloadException, + ServiceUnavailableException, + UnsupportedMediaTypeException, +} from '../exceptions'; +import StreamArray from 'stream-json/streamers/StreamArray'; +import { ItemsService } from './items'; +import { queue } from 'async'; +import destroyStream from 'destroy'; +import csv from 'csv-parser'; +import { set, transform } from 'lodash'; +import { parse as toXML } from 'js2xmlparser'; +import { Parser as CSVParser, transforms as CSVTransforms } from 'json2csv'; +import { appendFile, createReadStream } from 'fs-extra'; +import { file as createTmpFile } from 'tmp-promise'; +import env from '../env'; +import { FilesService } from './files'; +import { getDateFormatted } from '../utils/get-date-formatted'; +import { toArray } from '@directus/shared/utils'; +import { NotificationsService } from './notifications'; +import logger from '../logger'; + +export class ImportService { + knex: Knex; + accountability: Accountability | null; + schema: SchemaOverview; + + constructor(options: AbstractServiceOptions) { + this.knex = options.knex || getDatabase(); + this.accountability = options.accountability || null; + this.schema = options.schema; + } + + async import(collection: string, mimetype: string, stream: NodeJS.ReadableStream): Promise { + if (collection.startsWith('directus_')) throw new ForbiddenException(); + + const createPermissions = this.accountability?.permissions?.find( + (permission) => permission.collection === collection && permission.action === 'create' + ); + + const updatePermissions = this.accountability?.permissions?.find( + (permission) => permission.collection === collection && permission.action === 'update' + ); + + if (this.accountability?.admin !== true && (!createPermissions || !updatePermissions)) { + throw new ForbiddenException(); + } + + switch (mimetype) { + case 'application/json': + return await this.importJSON(collection, stream); + case 'text/csv': + case 'application/vnd.ms-excel': + return await this.importCSV(collection, stream); + default: + throw new UnsupportedMediaTypeException(`Can't import files of type "${mimetype}"`); + } + } + + importJSON(collection: string, stream: NodeJS.ReadableStream): Promise { + const extractJSON = StreamArray.withParser(); + + return this.knex.transaction((trx) => { + const service = new ItemsService(collection, { + knex: trx, + schema: this.schema, + accountability: this.accountability, + }); + + const saveQueue = queue(async (value: Record) => { + return await service.upsertOne(value); + }); + + return new Promise((resolve, reject) => { + stream.pipe(extractJSON); + + extractJSON.on('data', ({ value }) => { + saveQueue.push(value); + }); + + extractJSON.on('error', (err) => { + destroyStream(stream); + destroyStream(extractJSON); + + reject(new InvalidPayloadException(err.message)); + }); + + saveQueue.error((err) => { + reject(err); + }); + + extractJSON.on('end', () => { + saveQueue.drain(() => { + return resolve(); + }); + }); + }); + }); + } + + importCSV(collection: string, stream: NodeJS.ReadableStream): Promise { + return this.knex.transaction((trx) => { + const service = new ItemsService(collection, { + knex: trx, + schema: this.schema, + accountability: this.accountability, + }); + + const saveQueue = queue(async (value: Record) => { + return await service.upsertOne(value); + }); + + return new Promise((resolve, reject) => { + stream + .pipe(csv()) + .on('data', (value: Record) => { + const obj = transform(value, (result: Record, value, key) => { + if (value.length === 0) { + delete result[key]; + } else { + try { + const parsedJson = JSON.parse(value); + set(result, key, parsedJson); + } catch { + set(result, key, value); + } + } + }); + + saveQueue.push(obj); + }) + .on('error', (err) => { + destroyStream(stream); + reject(new InvalidPayloadException(err.message)); + }) + .on('end', () => { + saveQueue.drain(() => { + return resolve(); + }); + }); + + saveQueue.error((err) => { + reject(err); + }); + }); + }); + } +} + +export class ExportService { + knex: Knex; + accountability: Accountability | null; + schema: SchemaOverview; + + constructor(options: AbstractServiceOptions) { + this.knex = options.knex || getDatabase(); + this.accountability = options.accountability || null; + this.schema = options.schema; + } + + /** + * Export the query results as a named file. Will query in batches, and keep appending a tmp file + * until all the data is retrieved. Uploads the result as a new file using the regular + * FilesService upload method. + */ + async exportToFile( + collection: string, + query: Partial, + format: 'xml' | 'csv' | 'json', + options?: { + file?: Partial; + } + ) { + try { + const mimeTypes = { + xml: 'text/xml', + csv: 'text/csv', + json: 'application/json', + }; + + const database = getDatabase(); + + const { path, cleanup } = await createTmpFile(); + + await database.transaction(async (trx) => { + const service = new ItemsService(collection, { + accountability: this.accountability, + schema: this.schema, + knex: trx, + }); + + const totalCount = await service + .readByQuery({ + ...query, + aggregate: { + count: ['*'], + }, + }) + .then((result) => Number(result?.[0]?.count ?? 0)); + + const count = query.limit ? Math.min(totalCount, query.limit) : totalCount; + + const requestedLimit = query.limit ?? -1; + const batchesRequired = Math.ceil(count / env.EXPORT_BATCH_SIZE); + + let readCount = 0; + + for (let batch = 0; batch <= batchesRequired; batch++) { + let limit = env.EXPORT_BATCH_SIZE; + + if (requestedLimit > 0 && env.EXPORT_BATCH_SIZE > requestedLimit - readCount) { + limit = requestedLimit - readCount; + } + + const result = await service.readByQuery({ + ...query, + limit, + page: batch, + }); + + readCount += result.length; + + if (result.length) { + await appendFile( + path, + this.transform(result, format, { + includeHeader: batch === 0, + includeFooter: batch + 1 === batchesRequired, + }) + ); + } + } + }); + + const filesService = new FilesService({ + accountability: this.accountability, + schema: this.schema, + }); + + const storage: string = toArray(env.STORAGE_LOCATIONS)[0]; + + const title = `export-${collection}-${getDateFormatted()}`; + const filename = `${title}.${format}`; + + const fileWithDefaults: Partial & { storage: string; filename_download: string } = { + ...(options?.file ?? {}), + title: options?.file?.title ?? title, + filename_download: options?.file?.filename_download ?? filename, + storage: options?.file?.storage ?? storage, + type: mimeTypes[format], + }; + + const savedFile = await filesService.uploadOne(createReadStream(path), fileWithDefaults); + + if (this.accountability?.user) { + const notificationsService = new NotificationsService({ + accountability: this.accountability, + schema: this.schema, + }); + + await notificationsService.createOne({ + recipient: this.accountability.user, + sender: this.accountability.user, + subject: `Your export of ${collection} is ready`, + collection: `directus_files`, + item: savedFile, + }); + } + + await cleanup(); + } catch (err: any) { + logger.error(err, `Couldn't export ${collection}: ${err.message}`); + + if (this.accountability?.user) { + const notificationsService = new NotificationsService({ + accountability: this.accountability, + schema: this.schema, + }); + + await notificationsService.createOne({ + recipient: this.accountability.user, + sender: this.accountability.user, + subject: `Your export of ${collection} failed`, + message: `Please contact your system administrator for more information.`, + }); + } + } + } + + /** + * Transform a given input object / array to the given type + */ + transform( + input: Record[], + format: 'xml' | 'csv' | 'json', + options?: { + includeHeader?: boolean; + includeFooter?: boolean; + } + ): string { + if (format === 'json') { + let string = JSON.stringify(input || null, null, '\t'); + + if (options?.includeHeader === false) string = string.split('\n').slice(1).join('\n'); + + if (options?.includeFooter === false) { + const lines = string.split('\n'); + string = lines.slice(0, lines.length - 1).join('\n'); + string += ',\n'; + } + + return string; + } + + if (format === 'xml') { + let string = toXML('data', input); + + if (options?.includeHeader === false) string = string.split('\n').slice(2).join('\n'); + + if (options?.includeFooter === false) { + const lines = string.split('\n'); + string = lines.slice(0, lines.length - 1).join('\n'); + string += '\n'; + } + + return string; + } + + if (format === 'csv') { + const parser = new CSVParser({ + transforms: [CSVTransforms.flatten({ separator: '.' })], + header: options?.includeHeader !== false, + }); + + return parser.parse(input); + } + + throw new ServiceUnavailableException(`Illegal export type used: "${format}"`, { service: 'export' }); + } +} diff --git a/api/src/services/import.ts b/api/src/services/import.ts deleted file mode 100644 index 6c3f9bde83..0000000000 --- a/api/src/services/import.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { Knex } from 'knex'; -import getDatabase from '../database'; -import { AbstractServiceOptions } from '../types'; -import { Accountability, SchemaOverview } from '@directus/shared/types'; -import { ForbiddenException, InvalidPayloadException, UnsupportedMediaTypeException } from '../exceptions'; -import StreamArray from 'stream-json/streamers/StreamArray'; -import { ItemsService } from './items'; -import { queue } from 'async'; -import destroyStream from 'destroy'; -import csv from 'csv-parser'; -import { set, transform } from 'lodash'; - -export class ImportService { - knex: Knex; - accountability: Accountability | null; - schema: SchemaOverview; - - constructor(options: AbstractServiceOptions) { - this.knex = options.knex || getDatabase(); - this.accountability = options.accountability || null; - this.schema = options.schema; - } - - async import(collection: string, mimetype: string, stream: NodeJS.ReadableStream): Promise { - if (collection.startsWith('directus_')) throw new ForbiddenException(); - - const createPermissions = this.accountability?.permissions?.find( - (permission) => permission.collection === collection && permission.action === 'create' - ); - - const updatePermissions = this.accountability?.permissions?.find( - (permission) => permission.collection === collection && permission.action === 'update' - ); - - if (this.accountability?.admin !== true && (!createPermissions || !updatePermissions)) { - throw new ForbiddenException(); - } - - switch (mimetype) { - case 'application/json': - return await this.importJSON(collection, stream); - case 'text/csv': - case 'application/vnd.ms-excel': - return await this.importCSV(collection, stream); - default: - throw new UnsupportedMediaTypeException(`Can't import files of type "${mimetype}"`); - } - } - - importJSON(collection: string, stream: NodeJS.ReadableStream): Promise { - const extractJSON = StreamArray.withParser(); - - return this.knex.transaction((trx) => { - const service = new ItemsService(collection, { - knex: trx, - schema: this.schema, - accountability: this.accountability, - }); - - const saveQueue = queue(async (value: Record) => { - return await service.upsertOne(value); - }); - - return new Promise((resolve, reject) => { - stream.pipe(extractJSON); - - extractJSON.on('data', ({ value }) => { - saveQueue.push(value); - }); - - extractJSON.on('error', (err) => { - destroyStream(stream); - destroyStream(extractJSON); - - reject(new InvalidPayloadException(err.message)); - }); - - saveQueue.error((err) => { - reject(err); - }); - - extractJSON.on('end', () => { - saveQueue.drain(() => { - return resolve(); - }); - }); - }); - }); - } - - importCSV(collection: string, stream: NodeJS.ReadableStream): Promise { - return this.knex.transaction((trx) => { - const service = new ItemsService(collection, { - knex: trx, - schema: this.schema, - accountability: this.accountability, - }); - - const saveQueue = queue(async (value: Record) => { - return await service.upsertOne(value); - }); - - return new Promise((resolve, reject) => { - stream - .pipe(csv()) - .on('data', (value: Record) => { - const obj = transform(value, (result: Record, value, key) => { - if (value.length === 0) { - delete result[key]; - } else { - try { - const parsedJson = JSON.parse(value); - set(result, key, parsedJson); - } catch { - set(result, key, value); - } - } - }); - - saveQueue.push(obj); - }) - .on('error', (err) => { - destroyStream(stream); - reject(new InvalidPayloadException(err.message)); - }) - .on('end', () => { - saveQueue.drain(() => { - return resolve(); - }); - }); - - saveQueue.error((err) => { - reject(err); - }); - }); - }); - } -} diff --git a/api/src/services/index.ts b/api/src/services/index.ts index 8668b346d6..36376ddef3 100644 --- a/api/src/services/index.ts +++ b/api/src/services/index.ts @@ -10,7 +10,7 @@ export * from './fields'; export * from './files'; export * from './folders'; export * from './graphql'; -export * from './import'; +export * from './import-export'; export * from './mail'; export * from './meta'; export * from './notifications'; diff --git a/api/src/utils/get-date-formatted.ts b/api/src/utils/get-date-formatted.ts new file mode 100644 index 0000000000..339c9ab067 --- /dev/null +++ b/api/src/utils/get-date-formatted.ts @@ -0,0 +1,11 @@ +export function getDateFormatted() { + const date = new Date(); + + let month = String(date.getMonth() + 1); + if (month.length === 1) month = '0' + month; + + let day = String(date.getDate()); + if (day.length === 1) day = '0' + day; + + return `${date.getFullYear()}${month}${day}-${date.getHours()}${date.getMinutes()}${date.getSeconds()}`; +} diff --git a/app/src/interfaces/_system/system-fields/index.ts b/app/src/interfaces/_system/system-fields/index.ts new file mode 100644 index 0000000000..7342e09c17 --- /dev/null +++ b/app/src/interfaces/_system/system-fields/index.ts @@ -0,0 +1,12 @@ +import { defineInterface } from '@directus/shared/utils'; +import InterfaceSystemFields from './system-fields.vue'; + +export default defineInterface({ + id: 'system-fields', + name: '$t:interfaces.fields.name', + icon: 'search', + component: InterfaceSystemFields, + types: ['csv', 'json'], + options: [], + system: true, +}); diff --git a/app/src/interfaces/_system/system-fields/system-fields.vue b/app/src/interfaces/_system/system-fields/system-fields.vue new file mode 100644 index 0000000000..4a52963c38 --- /dev/null +++ b/app/src/interfaces/_system/system-fields/system-fields.vue @@ -0,0 +1,87 @@ + + + + + diff --git a/app/src/lang/translations/en-US.yaml b/app/src/lang/translations/en-US.yaml index ce19a45cea..73f0c37fa5 100644 --- a/app/src/lang/translations/en-US.yaml +++ b/app/src/lang/translations/en-US.yaml @@ -42,6 +42,7 @@ duplicate_field: Duplicate Field half_width: Half Width full_width: Full Width group: Group +export_items: Export Items and: And or: Or fill_width: Fill Width @@ -344,8 +345,6 @@ label_import: Import label_export: Export import_export: Import / Export format: Format -use_current_filters_settings: Use Current Filters & Settings -export_data_button: Start Export last_page: Last Page last_access: Last Access fill_template: Fill with Template Value @@ -417,6 +416,14 @@ replace_from_library: Replace File from Library replace_from_url: Replace File from URL no_file_selected: No File Selected download_file: Download File +start_export: Start Export +not_available_for_local_downloads: Not available for local downloads +exporting_all_items_in_collection: Exporting all {total} items within {collection}. +exporting_limited_items_in_collection: Exporting {limit} out of {total} items within {collection}. +exporting_no_items_to_export: No items to export. Adjust the exporting configuration below. +exporting_download_hint: Once completed, this {format} file will automatically be downloaded to your device. +exporting_batch_hint: This export will be processed in batches, and once completed, the {format} file will be saved to the File Library. +exporting_batch_hint_forced: Due to the large number of items, this export must be processed in batches, and once completed, the {format} file will be saved to the File Library. collection_key: Collection Key name: Name primary_key_field: Primary Key Field @@ -723,6 +730,7 @@ no_data: No Data create_dashboard: Create Dashboard dashboard_name: Dashboard Name full_screen: Full Screen +full_text_search: Full-Text Search edit_panels: Edit Panels center_align: Center Align left_align: Left Align @@ -1273,6 +1281,9 @@ sign_out: Sign Out sign_out_confirm: Are you sure you want to sign out? something_went_wrong: Something went wrong. sort_direction: Sort Direction +export_location: Export Location +export_started: Export Started +export_started_copy: Your export has started. You'll be notified when it's ready to download. sort_asc: Sort Ascending sort_desc: Sort Descending template: Template @@ -1300,6 +1311,8 @@ interfaces: no_rules: No configured rules change_value: Click to change value placeholder: Drag rules here + fields: + name: Fields group-accordion: name: Accordion description: Display fields or groups as accordion sections diff --git a/app/src/modules/content/routes/collection.vue b/app/src/modules/content/routes/collection.vue index 793ad86f98..46c3333725 100644 --- a/app/src/modules/content/routes/collection.vue +++ b/app/src/modules/content/routes/collection.vue @@ -248,6 +248,7 @@ :collection="collection" :filter="mergeFilters(filter, archiveFilter)" :search="search" + :layout-query="layoutQuery" @refresh="refresh" /> diff --git a/app/src/modules/files/components/navigation-folder.vue b/app/src/modules/files/components/navigation-folder.vue index d022201a47..561ba68e0c 100644 --- a/app/src/modules/files/components/navigation-folder.vue +++ b/app/src/modules/files/components/navigation-folder.vue @@ -118,7 +118,7 @@ import { useI18n } from 'vue-i18n'; import { defineComponent, PropType, ref } from 'vue'; import useFolders, { Folder } from '@/composables/use-folders'; import api from '@/api'; -import FolderPicker from './folder-picker.vue'; +import FolderPicker from '@/views/private/components/folder-picker/folder-picker.vue'; import { useRouter } from 'vue-router'; import { unexpectedError } from '@/utils/unexpected-error'; diff --git a/app/src/modules/files/routes/collection.vue b/app/src/modules/files/routes/collection.vue index 0071fb60e1..f3f058b274 100644 --- a/app/src/modules/files/routes/collection.vue +++ b/app/src/modules/files/routes/collection.vue @@ -167,6 +167,7 @@ @@ -191,7 +192,7 @@ import usePreset from '@/composables/use-preset'; import LayoutSidebarDetail from '@/views/private/components/layout-sidebar-detail'; import AddFolder from '../components/add-folder.vue'; import SearchInput from '@/views/private/components/search-input'; -import FolderPicker from '../components/folder-picker.vue'; +import FolderPicker from '@/views/private/components/folder-picker/folder-picker.vue'; import emitter, { Events } from '@/events'; import { useRouter, onBeforeRouteLeave, onBeforeRouteUpdate } from 'vue-router'; import { useNotificationsStore, useUserStore, usePermissionsStore } from '@/stores'; diff --git a/app/src/modules/files/routes/item.vue b/app/src/modules/files/routes/item.vue index 85ee3065a9..6fa8ec76db 100644 --- a/app/src/modules/files/routes/item.vue +++ b/app/src/modules/files/routes/item.vue @@ -183,7 +183,7 @@ import FilePreview from '@/views/private/components/file-preview'; import ImageEditor from '@/views/private/components/image-editor'; import { Field } from '@directus/shared/types'; import FileInfoSidebarDetail from '../components/file-info-sidebar-detail.vue'; -import FolderPicker from '../components/folder-picker.vue'; +import FolderPicker from '@/views/private/components/folder-picker/folder-picker.vue'; import api, { addTokenToURL } from '@/api'; import { getRootPath } from '@/utils/get-root-path'; import FilesNotFound from './not-found.vue'; diff --git a/app/src/modules/users/routes/collection.vue b/app/src/modules/users/routes/collection.vue index 0bbf6c9a63..c6064430f5 100644 --- a/app/src/modules/users/routes/collection.vue +++ b/app/src/modules/users/routes/collection.vue @@ -151,6 +151,7 @@ diff --git a/app/src/views/private/components/export-sidebar-detail.vue b/app/src/views/private/components/export-sidebar-detail.vue index b0f84743e6..20420d3ed4 100644 --- a/app/src/views/private/components/export-sidebar-detail.vue +++ b/app/src/views/private/components/export-sidebar-detail.vue @@ -51,45 +51,177 @@
-

{{ t('label_export') }}

- - -
- -
- - {{ t('export_data_button') }} + + {{ t('export_items') }}
+ + + +
+
+

{{ t('format') }}

+ +
+ +
+

{{ t('limit') }}

+ +
+ +
+

{{ t('export_location') }}

+ +
+ +
+

{{ t('folder') }}

+ + {{ t('not_available_for_local_downloads') }} +
+ + +
+

+ + + + + +

+ +

+ + + + + +

+
+ + + + +
+

{{ t('sort_field') }}

+ +
+
+

{{ t('sort_direction') }}

+ +
+ +
+

{{ t('full_text_search') }}

+ +
+
+

{{ t('filter') }}

+ +
+
+

{{ t('field', 2) }}

+ +
+
+ - diff --git a/docs/configuration/config-options.md b/docs/configuration/config-options.md index 6acfb0ef23..18da76eea1 100644 --- a/docs/configuration/config-options.md +++ b/docs/configuration/config-options.md @@ -799,3 +799,4 @@ Allows you to configure hard technical limits, to prevent abuse and optimize for | Variable | Description | Default Value | | ----------------------- | ----------------------------------------------------------------------------------------- | ------------- | | `RELATIONAL_BATCH_SIZE` | How many rows are read into memory at a time when constructing nested relational datasets | 25000 | +| `EXPORT_BATCH_SIZE` | How many rows are read into memory at a time when constructing exports | 5000 | diff --git a/docs/reference/system/utilities.md b/docs/reference/system/utilities.md index ea30ad2df0..3dd28626a9 100644 --- a/docs/reference/system/utilities.md +++ b/docs/reference/system/utilities.md @@ -270,6 +270,75 @@ n/a --- +## Export Data to a File + +Export a larger data set to a file in the File Library + +
+
+ +### Query Parameters + +Doesn't use any query parameters. + +### Request Body + +
+ +`format` **Required**\ +What file format to save the export to. One of `csv`, `xml`, `json`. + +`query` **Required**\ +The query object to use for the export. Supports the [global query parameters](/reference/query). + +`file` **File Object**\ +Partial file object to tweak where / how the export file is saved. + +
+ +### Returns + +Empty body + +
+
+ +### REST API + +``` +POST /utils/export/:collection +``` + +##### Example + +``` +POST /utils/export/articles +``` + +```json +{ + "query": { + "filter": { + "status": { + "_eq": "published" + } + } + }, + "file": { + "folder": "34e95c19-cc50-42f2-83c8-b97616ac2390" + } +} +``` + +### GraphQL + +n/a + +
+
+ +--- + ## Clear the Internal Cache Resets both the data and schema cache of Directus. This endpoint is only available to admin users. diff --git a/package-lock.json b/package-lock.json index f97860b3be..4a1e13f6a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -54,20 +54,20 @@ }, "api": { "name": "directus", - "version": "9.5.2", + "version": "9.6.0", "license": "GPL-3.0-only", "dependencies": { "@aws-sdk/client-ses": "^3.40.0", - "@directus/app": "9.5.2", - "@directus/drive": "9.5.2", - "@directus/drive-azure": "9.5.2", - "@directus/drive-gcs": "9.5.2", - "@directus/drive-s3": "9.5.2", - "@directus/extensions-sdk": "9.5.2", - "@directus/format-title": "9.5.2", - "@directus/schema": "9.5.2", - "@directus/shared": "9.5.2", - "@directus/specs": "9.5.2", + "@directus/app": "9.6.0", + "@directus/drive": "9.6.0", + "@directus/drive-azure": "9.6.0", + "@directus/drive-gcs": "9.6.0", + "@directus/drive-s3": "9.6.0", + "@directus/extensions-sdk": "9.6.0", + "@directus/format-title": "9.6.0", + "@directus/schema": "9.6.0", + "@directus/shared": "9.6.0", + "@directus/specs": "9.6.0", "@godaddy/terminus": "^4.9.0", "@rollup/plugin-alias": "^3.1.9", "@rollup/plugin-virtual": "^2.0.3", @@ -133,6 +133,7 @@ "sharp": "^0.29.0", "stream-json": "^1.7.1", "supertest": "^6.1.6", + "tmp-promise": "^3.0.3", "update-check": "^1.5.4", "uuid": "^8.3.2", "uuid-validate": "0.0.3", @@ -207,12 +208,12 @@ }, "app": { "name": "@directus/app", - "version": "9.5.2", + "version": "9.6.0", "devDependencies": { - "@directus/docs": "9.5.2", - "@directus/extensions-sdk": "9.5.2", - "@directus/format-title": "9.5.2", - "@directus/shared": "9.5.2", + "@directus/docs": "9.6.0", + "@directus/extensions-sdk": "9.6.0", + "@directus/format-title": "9.6.0", + "@directus/shared": "9.6.0", "@fortawesome/fontawesome-svg-core": "1.2.36", "@fortawesome/free-brands-svg-icons": "5.15.4", "@fullcalendar/core": "5.10.1", @@ -351,7 +352,7 @@ }, "docs": { "name": "@directus/docs", - "version": "9.5.2", + "version": "9.6.0", "license": "ISC", "devDependencies": { "directory-tree": "3.0.1", @@ -47360,6 +47361,14 @@ "node": ">=8.17.0" } }, + "node_modules/tmp-promise": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", + "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", + "dependencies": { + "tmp": "^0.2.0" + } + }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -52063,11 +52072,11 @@ }, "packages/cli": { "name": "@directus/cli", - "version": "9.5.2", + "version": "9.6.0", "license": "MIT", "dependencies": { - "@directus/format-title": "9.5.2", - "@directus/sdk": "9.5.2", + "@directus/format-title": "9.6.0", + "@directus/sdk": "9.6.0", "@types/yargs": "^17.0.0", "app-module-path": "^2.2.0", "chalk": "^4.1.0", @@ -52211,11 +52220,11 @@ } }, "packages/create-directus-extension": { - "version": "9.5.2", + "version": "9.6.0", "license": "GPL-3.0-only", "dependencies": { - "@directus/extensions-sdk": "9.5.2", - "@directus/shared": "9.5.2", + "@directus/extensions-sdk": "9.6.0", + "@directus/shared": "9.6.0", "inquirer": "^8.1.2" }, "bin": { @@ -52224,7 +52233,7 @@ } }, "packages/create-directus-project": { - "version": "9.5.2", + "version": "9.6.0", "license": "GPL-3.0-only", "dependencies": { "chalk": "^4.1.1", @@ -52241,7 +52250,7 @@ }, "packages/drive": { "name": "@directus/drive", - "version": "9.5.2", + "version": "9.6.0", "license": "MIT", "dependencies": { "fs-extra": "^10.0.0", @@ -52260,11 +52269,11 @@ }, "packages/drive-azure": { "name": "@directus/drive-azure", - "version": "9.5.2", + "version": "9.6.0", "license": "MIT", "dependencies": { "@azure/storage-blob": "^12.6.0", - "@directus/drive": "9.5.2", + "@directus/drive": "9.6.0", "normalize-path": "^3.0.0" }, "devDependencies": { @@ -52386,10 +52395,10 @@ }, "packages/drive-gcs": { "name": "@directus/drive-gcs", - "version": "9.5.2", + "version": "9.6.0", "license": "MIT", "dependencies": { - "@directus/drive": "9.5.2", + "@directus/drive": "9.6.0", "@google-cloud/storage": "^5.8.5", "lodash": "4.17.21", "normalize-path": "^3.0.0" @@ -52499,10 +52508,10 @@ }, "packages/drive-s3": { "name": "@directus/drive-s3", - "version": "9.5.2", + "version": "9.6.0", "license": "MIT", "dependencies": { - "@directus/drive": "9.5.2", + "@directus/drive": "9.6.0", "aws-sdk": "^2.928.0", "normalize-path": "^3.0.0" }, @@ -52716,9 +52725,9 @@ }, "packages/extensions-sdk": { "name": "@directus/extensions-sdk", - "version": "9.5.2", + "version": "9.6.0", "dependencies": { - "@directus/shared": "9.5.2", + "@directus/shared": "9.6.0", "@rollup/plugin-commonjs": "^21.0.1", "@rollup/plugin-json": "^4.1.0", "@rollup/plugin-node-resolve": "^13.1.3", @@ -52749,7 +52758,7 @@ }, "packages/format-title": { "name": "@directus/format-title", - "version": "9.5.2", + "version": "9.6.0", "license": "MIT", "devDependencies": { "@rollup/plugin-commonjs": "21.0.1", @@ -52805,10 +52814,10 @@ }, "packages/gatsby-source-directus": { "name": "@directus/gatsby-source-directus", - "version": "9.5.2", + "version": "9.6.0", "license": "MIT", "dependencies": { - "@directus/sdk": "9.5.2", + "@directus/sdk": "9.6.0", "gatsby-source-filesystem": "4.2.0", "gatsby-source-graphql": "4.2.0", "ms": "2.1.3" @@ -52816,7 +52825,7 @@ }, "packages/schema": { "name": "@directus/schema", - "version": "9.5.2", + "version": "9.6.0", "license": "GPL-3.0", "dependencies": { "knex-schema-inspector": "1.7.3", @@ -52829,7 +52838,7 @@ }, "packages/sdk": { "name": "@directus/sdk", - "version": "9.5.2", + "version": "9.6.0", "license": "MIT", "dependencies": { "axios": "^0.24.0" @@ -53006,7 +53015,7 @@ }, "packages/shared": { "name": "@directus/shared", - "version": "9.5.2", + "version": "9.6.0", "dependencies": { "date-fns": "2.24.0", "fs-extra": "10.0.0", @@ -53073,7 +53082,7 @@ }, "packages/specs": { "name": "@directus/specs", - "version": "9.5.2", + "version": "9.6.0", "license": "GPL-3.0", "dependencies": { "openapi3-ts": "^2.0.1" @@ -55465,10 +55474,10 @@ "@directus/app": { "version": "file:app", "requires": { - "@directus/docs": "9.5.2", - "@directus/extensions-sdk": "9.5.2", - "@directus/format-title": "9.5.2", - "@directus/shared": "9.5.2", + "@directus/docs": "9.6.0", + "@directus/extensions-sdk": "9.6.0", + "@directus/format-title": "9.6.0", + "@directus/shared": "9.6.0", "@fortawesome/fontawesome-svg-core": "1.2.36", "@fortawesome/free-brands-svg-icons": "5.15.4", "@fullcalendar/core": "5.10.1", @@ -55583,8 +55592,8 @@ "@directus/cli": { "version": "file:packages/cli", "requires": { - "@directus/format-title": "9.5.2", - "@directus/sdk": "9.5.2", + "@directus/format-title": "9.6.0", + "@directus/sdk": "9.6.0", "@types/figlet": "1.5.4", "@types/fs-extra": "9.0.13", "@types/jest": "27.0.3", @@ -55772,7 +55781,7 @@ "version": "file:packages/drive-azure", "requires": { "@azure/storage-blob": "^12.6.0", - "@directus/drive": "9.5.2", + "@directus/drive": "9.6.0", "@types/fs-extra": "9.0.13", "@types/jest": "27.0.3", "@types/node": "16.11.9", @@ -55848,7 +55857,7 @@ "@directus/drive-gcs": { "version": "file:packages/drive-gcs", "requires": { - "@directus/drive": "9.5.2", + "@directus/drive": "9.6.0", "@google-cloud/storage": "^5.8.5", "@lukeed/uuid": "2.0.0", "@types/fs-extra": "9.0.13", @@ -55915,7 +55924,7 @@ "@directus/drive-s3": { "version": "file:packages/drive-s3", "requires": { - "@directus/drive": "9.5.2", + "@directus/drive": "9.6.0", "@lukeed/uuid": "2.0.0", "@types/fs-extra": "9.0.13", "@types/jest": "27.0.3", @@ -55993,7 +56002,7 @@ "@directus/extensions-sdk": { "version": "file:packages/extensions-sdk", "requires": { - "@directus/shared": "9.5.2", + "@directus/shared": "9.6.0", "@rollup/plugin-commonjs": "^21.0.1", "@rollup/plugin-json": "^4.1.0", "@rollup/plugin-node-resolve": "^13.1.3", @@ -56058,7 +56067,7 @@ "@directus/gatsby-source-directus": { "version": "file:packages/gatsby-source-directus", "requires": { - "@directus/sdk": "9.5.2", + "@directus/sdk": "9.6.0", "gatsby-source-filesystem": "4.2.0", "gatsby-source-graphql": "4.2.0", "ms": "2.1.3" @@ -70249,8 +70258,8 @@ "create-directus-extension": { "version": "file:packages/create-directus-extension", "requires": { - "@directus/extensions-sdk": "9.5.2", - "@directus/shared": "9.5.2", + "@directus/extensions-sdk": "9.6.0", + "@directus/shared": "9.6.0", "inquirer": "^8.1.2" } }, @@ -71471,16 +71480,16 @@ "version": "file:api", "requires": { "@aws-sdk/client-ses": "^3.40.0", - "@directus/app": "9.5.2", - "@directus/drive": "9.5.2", - "@directus/drive-azure": "9.5.2", - "@directus/drive-gcs": "9.5.2", - "@directus/drive-s3": "9.5.2", - "@directus/extensions-sdk": "9.5.2", - "@directus/format-title": "9.5.2", - "@directus/schema": "9.5.2", - "@directus/shared": "9.5.2", - "@directus/specs": "9.5.2", + "@directus/app": "9.6.0", + "@directus/drive": "9.6.0", + "@directus/drive-azure": "9.6.0", + "@directus/drive-gcs": "9.6.0", + "@directus/drive-s3": "9.6.0", + "@directus/extensions-sdk": "9.6.0", + "@directus/format-title": "9.6.0", + "@directus/schema": "9.6.0", + "@directus/shared": "9.6.0", + "@directus/specs": "9.6.0", "@godaddy/terminus": "^4.9.0", "@keyv/redis": "^2.1.2", "@rollup/plugin-alias": "^3.1.9", @@ -71599,6 +71608,7 @@ "stream-json": "^1.7.1", "supertest": "^6.1.6", "tedious": "^13.0.0", + "tmp-promise": "*", "ts-jest": "27.1.3", "ts-node-dev": "1.1.8", "typescript": "4.5.2", @@ -91357,6 +91367,14 @@ "rimraf": "^3.0.0" } }, + "tmp-promise": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", + "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", + "requires": { + "tmp": "^0.2.0" + } + }, "tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", diff --git a/packages/shared/src/composables/use-collection.ts b/packages/shared/src/composables/use-collection.ts index 42b5b3f062..dd538af0a7 100644 --- a/packages/shared/src/composables/use-collection.ts +++ b/packages/shared/src/composables/use-collection.ts @@ -1,9 +1,9 @@ import { useStores } from './use-system'; -import { Collection, Field } from '../types'; +import { AppCollection, Field } from '../types'; import { computed, ref, Ref, ComputedRef } from 'vue'; type UsableCollection = { - info: ComputedRef; + info: ComputedRef; fields: ComputedRef; defaults: Record; primaryKeyField: ComputedRef; @@ -22,7 +22,7 @@ export function useCollection(collectionKey: string | Ref): Usabl const info = computed(() => { return ( - (collectionsStore.collections as Collection[]).find(({ collection: key }) => key === collection.value) || null + (collectionsStore.collections as AppCollection[]).find(({ collection: key }) => key === collection.value) || null ); }); diff --git a/packages/shared/src/types/collection.ts b/packages/shared/src/types/collection.ts index e343a3b19e..020fcec8c2 100644 --- a/packages/shared/src/types/collection.ts +++ b/packages/shared/src/types/collection.ts @@ -34,4 +34,11 @@ export interface Collection { schema: Table | null; } +export interface AppCollection extends Collection { + name: string; + icon: string; + type: CollectionType; + color?: string | null; +} + export type CollectionType = 'alias' | 'table' | 'unknown'; diff --git a/packages/shared/src/types/notifications.ts b/packages/shared/src/types/notifications.ts index 5db052efd9..4512c7bb38 100644 --- a/packages/shared/src/types/notifications.ts +++ b/packages/shared/src/types/notifications.ts @@ -1,3 +1,5 @@ +import { PrimaryKey } from './items'; + export type Notification = { id: string; status: string; @@ -7,5 +9,5 @@ export type Notification = { subject: string; message: string | null; collection: string | null; - item: string | null; + item: PrimaryKey | null; };