github/directus
1
1
Fork 0
mirror of https://github.com/directus/directus.git synced 2026-04-25 03:00:53 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add max concurrency and max image transform size support (#5795)

Browse Source 
* Add assets concurrency and max size controls

* Render no-thumbnail images nicer in app

* Document new asset environment variables

* Update package-lock
This commit is contained in:
Rijk van Zanten
2021-05-20 18:18:10 -04:00
committed by GitHub
parent 4264b87004
commit 8d3102fbad
11 changed files with 146 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
api/package.json
View File

@@ -77,6 +77,7 @@
"@godaddy/terminus": "^4.7.2",
"argon2": "^0.27.0",
"async": "^3.2.0",
"async-mutex": "^0.3.1",
"atob": "^2.1.2",
"axios": "^0.21.0",
"body-parser": "^1.19.0",

5
api/src/env.ts
View File

@@ -51,7 +51,6 @@ const defaults: Record<string, any> = {
CACHE_TTL: '30m',
CACHE_NAMESPACE: 'system-cache',
CACHE_AUTO_PURGE: false,
ASSETS_CACHE_TTL: '30m',
OAUTH_PROVIDERS: '',
@@ -63,6 +62,10 @@ const defaults: Record<string, any> = {
EMAIL_SENDMAIL_PATH: '/usr/sbin/sendmail',
TELEMETRY: true,
ASSETS_CACHE_TTL: '30m',
ASSETS_TRANSFORM_MAX_CONCURRENT: 4,
ASSETS_TRANSFORM_IMAGE_MAX_DIMENSION: 6000,
};
// Allows us to force certain environment variable into a type, instead of relying

7
api/src/exceptions/illegal-asset-transformation.ts Normal file
View File

@@ -0,0 +1,7 @@
import { BaseException } from './base';
export class IllegalAssetTransformation extends BaseException {
constructor(message: string) {
super(message, 400, 'ILLEGAL_ASSET_TRANSFORMATION');
}
}

1
api/src/exceptions/index.ts
View File

@@ -3,6 +3,7 @@ export * from './failed-validation';
export * from './forbidden';
export * from './graphql-validation';
export * from './hit-rate-limit';
export * from './illegal-asset-transformation';
export * from './invalid-credentials';
export * from './invalid-ip';
export * from './invalid-otp';

61
api/src/services/assets.ts
View File

@@ -3,10 +3,19 @@ import { Knex } from 'knex';
import path from 'path';
import sharp, { ResizeOptions } from 'sharp';
import database from '../database';
import { RangeNotSatisfiableException } from '../exceptions';
import { RangeNotSatisfiableException, IllegalAssetTransformation } from '../exceptions';
import storage from '../storage';
import { AbstractServiceOptions, Accountability, Transformation } from '../types';
import { AuthorizationService } from './authorization';
import { Semaphore } from 'async-mutex';
import env from '../env';
import { File } from '../types';
sharp.concurrency(1);
// Note: don't put this in the service. The service can be initialized in multiple places, but they
// should all share the same semaphore instance.
const semaphore = new Semaphore(env.ASSETS_MAX_CONCURRENT_TRANSFORMATIONS);
export class AssetsService {
knex: Knex;
@@ -35,7 +44,7 @@ export class AssetsService {
await this.authorizationService.checkAccess('read', 'directus_files', id);
}
const file = await database.select('*').from('directus_files').where({ id }).first();
const file = (await database.select('*').from('directus_files').where({ id }).first()) as File;
if (range) {
if (range.start >= file.filesize || (range.end && range.end >= file.filesize)) {
@@ -46,7 +55,7 @@ export class AssetsService {
const type = file.type;
// We can only transform JPEG, PNG, and WebP
if (Object.keys(transformation).length > 0 && ['image/jpeg', 'image/png', 'image/webp'].includes(type)) {
if (type && Object.keys(transformation).length > 0 && ['image/jpeg', 'image/png', 'image/webp'].includes(type)) {
const resizeOptions = this.parseTransformation(transformation);
const assetFilename =
@@ -64,19 +73,45 @@ export class AssetsService {
};
}
const readStream = storage.disk(file.storage).getStream(file.filename_disk, range);
const transformer = sharp().rotate().resize(resizeOptions);
if (transformation.quality) {
transformer.toFormat(type.substring(6), { quality: Number(transformation.quality) });
// Check image size before transforming. Processing an image that's too large for the
// system memory will kill the API. Sharp technically checks for this too in it's
// limitInputPixels, but we should have that check applied before starting the read streams
const { width, height } = file;
if (
!width ||
!height ||
width > env.ASSETS_TRANSFORM_IMAGE_MAX_DIMENSION ||
height > env.ASSETS_TRANSFORM_IMAGE_MAX_DIMENSION
) {
throw new IllegalAssetTransformation(
`Image is too large to be transformed, or image size couldn't be determined.`
);
}
await storage.disk(file.storage).put(assetFilename, readStream.pipe(transformer), type);
return await semaphore.runExclusive(async () => {
const readStream = storage.disk(file.storage).getStream(file.filename_disk, range);
const transformer = sharp({
limitInputPixels: Math.pow(env.ASSETS_TRANSFORM_IMAGE_MAX_DIMENSION, 2),
sequentialRead: true,
})
.rotate()
.resize(resizeOptions);
return {
stream: storage.disk(file.storage).getStream(assetFilename, range),
stat: await storage.disk(file.storage).getStat(assetFilename),
file,
};
if (transformation.quality) {
transformer.toFormat(type.substring(6) as 'jpeg' | 'png' | 'webp', {
quality: Number(transformation.quality),
});
}
await storage.disk(file.storage).put(assetFilename, readStream.pipe(transformer), type);
return {
stream: storage.disk(file.storage).getStream(assetFilename, range),
stat: await storage.disk(file.storage).getStat(assetFilename),
file,
};
});
} else {
const readStream = storage.disk(file.storage).getStream(file.filename_disk, range);
const stat = await storage.disk(file.storage).getStat(file.filename_disk);

14
app/src/displays/file/file.vue
View File

@@ -1,11 +1,12 @@
<template>
<img
v-if="imageThumbnail"
v-if="imageThumbnail && !imgError"
:src="imageThumbnail"
:class="{ 'is-svg': value && value.type.includes('svg') }"
:alt="value.title"
@error="imgError = true"
/>
<div ref="previewEl" v-else class="preview" :class="{ 'has-file': value }" :style="{ width: height + 'px' }">
<div ref="previewEl" v-else class="preview">
<span class="extension" v-if="fileExtension">
{{ fileExtension }}
</span>
@@ -35,6 +36,7 @@ export default defineComponent({
},
setup(props) {
const previewEl = ref<Element>();
const imgError = ref(false);
const fileExtension = computed(() => {
if (!props.value) return null;
@@ -50,16 +52,17 @@ export default defineComponent({
const { height } = useElementSize(previewEl);
return { fileExtension, imageThumbnail, previewEl, height };
return { fileExtension, imageThumbnail, previewEl, height, imgError };
},
});
</script>
<style lang="scss" scoped>
img {
width: auto;
max-height: 100%;
height: 100%;
object-fit: cover;
border-radius: var(--border-radius);
aspect-ratio: 1;
}
.preview {
@@ -73,6 +76,7 @@ img {
overflow: hidden;
background-color: var(--background-normal);
border-radius: var(--border-radius);
aspect-ratio: 1;
&.has-file {
background-color: var(--primary-alt);

9
app/src/displays/image/image.vue
View File

@@ -1,10 +1,11 @@
<template>
<img v-if="src" :src="src" role="presentation" :alt="value && value.title" :class="{ circle }" />
<v-icon v-if="imgError" name="" />
<img v-else-if="src" :src="src" role="presentation" :alt="value && value.title" :class="{ circle }" />
<value-null v-else />
</template>
<script lang="ts">
import { defineComponent, PropType, computed } from '@vue/composition-api';
import { defineComponent, PropType, computed, ref } from '@vue/composition-api';
import ValueNull from '@/views/private/components/value-null';
import { getRootPath } from '@/utils/get-root-path';
import { addTokenToURL } from '@/api';
@@ -28,13 +29,15 @@ export default defineComponent({
},
},
setup(props) {
const imgError = ref(false);
const src = computed(() => {
if (props.value === null) return null;
const url = getRootPath() + `assets/${props.value.id}?key=system-small-cover`;
return addTokenToURL(url);
});
return { src };
return { src, imgError };
},
});
</script>

22
app/src/layouts/cards/components/card.vue
View File

@@ -6,10 +6,18 @@
</div>
<v-skeleton-loader v-if="loading" />
<template v-else>
<p v-if="type" class="type type-title">{{ type }}</p>
<p v-if="type || imgError" class="type type-title">{{ type }}</p>
<template v-else>
<img class="image" loading="lazy" v-if="imageSource" :src="imageSource" alt="" role="presentation" />
<img class="svg" v-else-if="svgSource" :src="svgSource" alt="" role="presentation" />
<img
class="image"
loading="lazy"
v-if="imageSource"
:src="imageSource"
alt=""
role="presentation"
@error="imgError = true"
/>
<img class="svg" v-else-if="svgSource" :src="svgSource" alt="" role="presentation" @error="imgError = true" />
<v-icon v-else large :name="icon" />
</template>
</template>
@@ -23,7 +31,7 @@
</template>
<script lang="ts">
import { defineComponent, PropType, computed } from '@vue/composition-api';
import { defineComponent, PropType, computed, ref } from '@vue/composition-api';
import router from '@/router';
import { getRootPath } from '@/utils/get-root-path';
import { addTokenToURL } from '@/api';
@@ -80,9 +88,11 @@ export default defineComponent({
},
},
setup(props, { emit }) {
const imgError = ref(false);
const type = computed(() => {
if (!props.file || !props.file.type) return null;
if (props.file.type.startsWith('image')) return null;
if (!imgError.value && props.file.type.startsWith('image')) return null;
return readableMimeType(props.file.type, true);
});
@@ -116,7 +126,7 @@ export default defineComponent({
return props.value.includes(props.item[props.itemKey]) ? 'check_circle' : 'radio_button_unchecked';
});
return { imageSource, svgSource, type, selectionIcon, toggleSelection, handleClick };
return { imageSource, svgSource, type, selectionIcon, toggleSelection, handleClick, imgError };
function toggleSelection() {
if (!props.item) return null;

10
app/src/views/private/components/file-preview/file-preview.vue
View File

@@ -1,12 +1,12 @@
<template>
<div class="file-preview" v-if="type">
<div class="file-preview" v-if="type && !imgError">
<div
v-if="type === 'image'"
class="image"
:class="{ svg: isSVG, 'max-size': inModal === false }"
@click="$emit('click')"
>
<img :src="src" :width="width" :height="height" :alt="title" />
<img :src="src" :width="width" :height="height" :alt="title" @error="imgError = true" />
<v-icon v-if="inModal === false" name="upload" />
</div>
@@ -17,7 +17,7 @@
</template>
<script lang="ts">
import { defineComponent, computed } from '@vue/composition-api';
import { defineComponent, computed, ref } from '@vue/composition-api';
export default defineComponent({
props: {
@@ -47,6 +47,8 @@ export default defineComponent({
},
},
setup(props) {
const imgError = ref(false);
const type = computed<'image' | 'video' | 'audio' | null>(() => {
if (props.mime === null) return null;
@@ -67,7 +69,7 @@ export default defineComponent({
const isSVG = computed(() => props.mime.includes('svg'));
return { type, isSVG };
return { type, isSVG, imgError };
},
});
</script>

26
docs/reference/environment-variables.md
View File

@@ -125,14 +125,13 @@ needs, you can extend the above environment variables to configure any of
## Cache
| Variable | Description | Default Value |
| ------------------ | ---------------------------------------------------------------------------------------------------------- | ---------------- |
| `ASSETS_CACHE_TTL` | How long assets will be cached for in the browser. Sets the `max-age` value of the `Cache-Control` header. | `30m` |
| `CACHE_ENABLED` | Whether or not caching is enabled. | `false` |
| `CACHE_TTL` | How long the cache is persisted. | `30m` |
| `CACHE_AUTO_PURGE` | Automatically purge the cache on `create`/`update`/`delete` actions. | `false` |
| `CACHE_NAMESPACE` | How to scope the cache data. | `directus-cache` |
| `CACHE_STORE` | Where to store the cache data. Either `memory`, `redis`, or `memcache`. | `memory` |
| Variable | Description | Default Value |
| ------------------ | ----------------------------------------------------------------------- | ---------------- |
| `CACHE_ENABLED` | Whether or not caching is enabled. | `false` |
| `CACHE_TTL` | How long the cache is persisted. | `30m` |
| `CACHE_AUTO_PURGE` | Automatically purge the cache on `create`/`update`/`delete` actions. | `false` |
| `CACHE_NAMESPACE` | How to scope the cache data. | `directus-cache` |
| `CACHE_STORE` | Where to store the cache data. Either `memory`, `redis`, or `memcache`. | `memory` |
Based on the `CACHE_STORE` used, you must also provide the following configurations:
@@ -253,6 +252,17 @@ STORAGE_LOCATIONS="local"
STORAGE_LOCAL_ROOT="./uploads"
```
## Assets
| Variable | Description | Default Value |
| -------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------- |
| `ASSETS_CACHE_TTL` | How long assets will be cached for in the browser. Sets the `max-age` value of the `Cache-Control` header. | `30m` |
| `ASSETS_TRANSFORM_MAX_CONCURRENT` | How many file transformations can be done simultaneously | 4 |
| `ASSETS_TRANSFORM_IMAGE_MAX_DIMENSION` | The max pixel dimensions size (width/height) that is allowed to be transformed | 6000 |
Image transformations can be fairly heavy on memory usage. If you're using a system with 1GB or less available memory,
we recommend lowering the allowed concurrent transformations to prevent you from overflowing your server.
## OAuth
| Variable | Description | Default Value |

30
package-lock.json generated
View File

@@ -68,6 +68,7 @@
"@godaddy/terminus": "^4.7.2",
"argon2": "^0.27.0",
"async": "^3.2.0",
"async-mutex": "^0.3.1",
"atob": "^2.1.2",
"axios": "^0.21.0",
"body-parser": "^1.19.0",
@@ -11582,6 +11583,19 @@
"resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz",
"integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ=="
},
"node_modules/async-mutex": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.3.1.tgz",
"integrity": "sha512-vRfQwcqBnJTLzVQo72Sf7KIUbcSUP5hNchx6udI1U6LuPQpfePgdjJzlCe76yFZ8pxlLjn9lwcl/Ya0TSOv0Tw==",
"dependencies": {
"tslib": "^2.1.0"
}
},
"node_modules/async-mutex/node_modules/tslib": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz",
"integrity": "sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w=="
},
"node_modules/async-retry": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.1.tgz",
@@ -63081,6 +63095,21 @@
"resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz",
"integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ=="
},
"async-mutex": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.3.1.tgz",
"integrity": "sha512-vRfQwcqBnJTLzVQo72Sf7KIUbcSUP5hNchx6udI1U6LuPQpfePgdjJzlCe76yFZ8pxlLjn9lwcl/Ya0TSOv0Tw==",
"requires": {
"tslib": "^2.1.0"
},
"dependencies": {
"tslib": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.2.0.tgz",
"integrity": "sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w=="
}
}
},
"async-retry": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.1.tgz",
@@ -67360,6 +67389,7 @@
"@types/uuid-validate": "^0.0.1",
"argon2": "^0.27.0",
"async": "^3.2.0",
"async-mutex": "^0.3.1",
"atob": "^2.1.2",
"axios": "^0.21.0",
"body-parser": "^1.19.0",