feat: simplify the process of importing images (#39)

When importing images, an `?img` query can be used to turn them into `Promise<HTMLImageElement>`:
```
import example from '../images/example.png?img';
```
Such Promise can then be awaited for and passed to other components:
```
<Image image={yield example} />
```
Similarly, an `?anim` query can be used to import all images from a directory as an array.
The imported object is of type `Promise<HTMLImageElement[]>`.

BREAKING CHANGE: change how images are imported

By default, importing images will now return their urls instead of a SpriteData object.
This behavior can be adjusted using the `?img` and `?anim` queries.

Resolves: #19
This commit is contained in:
Jacob
2022-06-22 14:23:43 +02:00
committed by GitHub
parent 0525de3175
commit 0c2341fe25
21 changed files with 455 additions and 711 deletions

View File

@@ -70,22 +70,30 @@ const compiler = webpack({
}),
},
},
{
test: /\.glsl$/i,
type: 'asset/source',
},
{
test: /\.mp4/i,
type: 'asset',
},
{
test: /\.meta/i,
loader: 'meta-loader',
},
{
test: /\.wav$/i,
test: /\.(wav|mp3|ogg|mp4)$/i,
type: 'asset',
},
{
test: /\.(png|jpe?g)$/i,
oneOf: [
{
resourceQuery: /img/,
loader: 'image-loader',
},
{
resourceQuery: /anim/,
loader: 'animation-loader',
},
{
type: 'asset',
},
],
},
{
test: /\.csv$/,
loader: 'csv-loader',
@@ -96,20 +104,8 @@ const compiler = webpack({
},
},
{
test: /\.anim$/i,
use: [
{
loader: 'animation-loader',
},
],
},
{
test: /\.png$/i,
use: [
{
loader: 'sprite-loader',
},
],
test: /\.glsl$/i,
type: 'asset/source',
},
],
},

View File

@@ -1,9 +1,8 @@
const path = require('path');
const {readdirSync} = require('fs');
const loadImage = require('../utils/load-image');
const nameRegex = /[^\d]*(\d+)\.png$/;
const nameRegex = /\D*(\d+)\.png$/;
function loader() {
function animationLoader() {
const callback = this.async();
const directoryPath = path.dirname(this.resourcePath);
@@ -16,20 +15,16 @@ function loader() {
)
.map(([file]) => path.resolve(directoryPath, file));
files.forEach(file => this.addDependency(file));
loadAnimation(files)
.then(result => callback(null, result))
loadAnimation(files, this.importModule)
.then(code => callback(null, code))
.catch(error => callback(error));
}
async function loadAnimation(files) {
const frames = [];
for (const file of files) {
frames.push(await loadImage(file));
}
async function loadAnimation(files, importModule) {
const urls = await Promise.all(files.map(file => importModule(file)));
return `export default ${JSON.stringify(frames)};`;
return `import {loadAnimation} from '@motion-canvas/core/lib/media';
export default loadAnimation(${JSON.stringify(urls)});`;
}
module.exports = loader;
module.exports = animationLoader;

View File

@@ -0,0 +1,15 @@
function imageLoader() {
const callback = this.async();
loadImage(this.resourcePath, this.importModule)
.then(code => callback(null, code))
.catch(error => callback(error));
}
async function loadImage(fileName, importModule) {
const url = await importModule(fileName);
return `import {loadImage} from '@motion-canvas/core/lib/media';
export default loadImage('${url}');`;
}
module.exports = imageLoader;

View File

@@ -1,10 +0,0 @@
const loadImage = require('../utils/load-image');
function loader() {
const callback = this.async();
loadImage(this.resourcePath)
.then(sprite => callback(null, `export default ${JSON.stringify(sprite)}`))
.catch(error => callback(error));
}
module.exports = loader;

View File

@@ -1,30 +0,0 @@
const {promises: fs} = require('fs');
const sizeOf = require('image-size');
const nodeCanvas = require('canvas');
const SIZE = 1024;
const context = nodeCanvas.createCanvas(SIZE, SIZE).getContext('2d');
const image = new nodeCanvas.Image();
module.exports = async function (fileName) {
const buffer = await fs.readFile(fileName);
const dimensions = sizeOf(buffer);
image.src = `data:image/png;base64,${buffer.toString('base64')}`;
context.clearRect(0, 0, SIZE, SIZE);
context.drawImage(image, 0, 0, dimensions.width, dimensions.height);
const imageData = context.getImageData(
0,
0,
dimensions.width,
dimensions.height,
);
return {
fileName,
src: image.src,
data: Array.from(imageData.data),
width: dimensions.width,
height: dimensions.height,
};
};

566
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -36,7 +36,6 @@
"@types/prismjs": "^1.26.0",
"@types/three": "^0.141.0",
"@types/webpack-env": "^1.17.0",
"canvas": "^2.9.1",
"html-webpack-plugin": "^5.5.0",
"image-size": "^1.0.1",
"konva": "^8.3.9",

View File

@@ -4,7 +4,7 @@ import type {SceneRunner} from './Scene';
import {Project, ProjectSize} from './Project';
import {Player} from './player';
import {hot} from './hot';
import {AudioManager} from './audio';
import {AudioManager} from './media';
interface BootstrapConfig {
name: string;

View File

@@ -1,14 +1,15 @@
import {getset, KonvaNode} from '../decorators';
import {Sprite, SpriteData} from './Sprite';
import {Sprite} from './Sprite';
import {GetSet} from 'konva/lib/types';
import {LinearLayout} from './LinearLayout';
import {Center} from '../types';
import {Surface, SurfaceConfig} from './Surface';
import {getStyle, Style} from '../styles';
import {ImageDataSource} from '../media';
export interface AnimationClipConfig extends SurfaceConfig {
animation: SpriteData[];
skin?: SpriteData;
animation: ImageDataSource[];
skin?: ImageDataSource;
frame?: number;
style?: Partial<Style>;
}

View File

@@ -1,88 +1,97 @@
import {Context} from 'konva/lib/Context';
import {Util} from 'konva/lib/Util';
import {GetSet, Vector2d} from 'konva/lib/types';
import {waitFor} from '../flow';
import {getset, KonvaNode, threadable} from '../decorators';
import {cached, getset, KonvaNode, threadable} from '../decorators';
import {GeneratorHelper} from '../helpers';
import {InterpolationFunction, map, tween} from '../tweening';
import {cancel, ThreadGenerator} from '../threading';
import {parseColor} from 'mix-color';
import {Shape, ShapeConfig} from 'konva/lib/Shape';
import {getImageData, ImageDataSource} from '../media';
export interface SpriteData {
fileName: string;
src: string;
data: number[];
width: number;
height: number;
}
export const SPRITE_CHANGE_EVENT = 'spriteChange';
export interface SpriteConfig extends ShapeConfig {
animation: SpriteData[];
skin?: SpriteData;
mask?: SpriteData;
animation: ImageDataSource[];
skin?: ImageDataSource;
mask?: ImageDataSource;
playing?: boolean;
fps?: number;
maskBlend?: number;
frame?: number;
}
export const SPRITE_CHANGE_EVENT = 'spriteChange';
const COMPUTE_CANVAS_SIZE = 64;
/**
* A class for animated sprites.
*
* Allows to use custom alpha masks and skins.
*/
@KonvaNode()
export class Sprite extends Shape {
@getset(null, Sprite.prototype.recalculate)
@getset(null, Sprite.prototype.updateAnimation)
public animation: GetSet<SpriteConfig['animation'], this>;
@getset(null, Sprite.prototype.recalculate)
public skin: GetSet<SpriteConfig['skin'], this>;
@getset(null, Sprite.prototype.recalculate, Sprite.prototype.maskTween)
public mask: GetSet<SpriteConfig['mask'], this>;
@getset(0, Sprite.prototype.updateAnimation)
public frame: GetSet<SpriteConfig['frame'], this>;
@getset(false)
public playing: GetSet<SpriteConfig['playing'], this>;
@getset(10)
public fps: GetSet<SpriteConfig['fps'], this>;
@getset(0, Sprite.prototype.recalculate)
public maskBlend: GetSet<SpriteConfig['maskBlend'], this>;
@getset(0, Sprite.prototype.recalculate)
public frame: GetSet<SpriteConfig['frame'], this>;
private spriteData: SpriteData = {
height: 0,
width: 0,
src: '',
data: [],
fileName: '',
};
/**
* An image used for 2D UV mapping.
*/
@getset(null, Sprite.prototype.updateSkin)
public skin: GetSet<SpriteConfig['skin'], this>;
/**
* An image that defines which parts of the sprite should be visible.
*
* The red channel is used for sampling.
*/
@getset(null, Sprite.prototype.updateMask, Sprite.prototype.maskTween)
public mask: GetSet<SpriteConfig['mask'], this>;
/**
* The blend between the original opacity and the opacity calculated using the
* mask.
*/
@getset(0, Sprite.prototype.updateFrame)
public maskBlend: GetSet<SpriteConfig['maskBlend'], this>;
private task: ThreadGenerator | null = null;
private imageData: ImageData;
private baseMask: SpriteData;
private baseMask: ImageDataSource;
private baseMaskBlend = 0;
private synced = false;
private readonly computeCanvas: HTMLCanvasElement;
private readonly context: CanvasRenderingContext2D;
public constructor(config?: SpriteConfig) {
super(config);
this.computeCanvas = Util.createCanvasElement();
this.computeCanvas.width = COMPUTE_CANVAS_SIZE;
this.computeCanvas.height = COMPUTE_CANVAS_SIZE;
this.computeCanvas = document.createElement('canvas');
this.context = this.computeCanvas.getContext('2d');
this.recalculate();
}
public _sceneFunc(context: Context) {
protected _sceneFunc(context: Context) {
const size = this.getSize();
let source: ImageDataSource = this.computeCanvas;
if (this.requiresProcessing()) {
// Make sure the compute canvas is up-to-date
this.getFrameData();
} else {
const animation = this.animation();
const frame = this.frame();
source = animation[frame % animation.length];
}
context.save();
context._context.imageSmoothingEnabled = false;
context.drawImage(
this.computeCanvas,
source,
0,
0,
this.spriteData.width,
this.spriteData.height,
source.width,
source.height,
size.width / -2,
size.height / -2,
size.width,
@@ -91,63 +100,124 @@ export class Sprite extends Shape {
context.restore();
}
private recalculate() {
private updateSkin() {
this._clearCache(this.getSkinData);
this.updateFrame();
}
private updateAnimation() {
this._clearCache(this.getRawFrameData);
this.updateFrame();
}
private updateMask() {
this._clearCache(this.getMaskData);
this.updateFrame();
}
private updateFrame() {
this._clearCache(this.getFrameData);
}
@cached('Sprite.skinData')
private getSkinData() {
const skin = this.skin();
return skin ? getImageData(skin) : null;
}
@cached('Sprite.maskData')
private getMaskData() {
const mask = this.mask();
return mask ? getImageData(mask) : null;
}
@cached('Sprite.baseMaskData')
private getBaseMaskData() {
return this.baseMask ? getImageData(this.baseMask) : null;
}
@cached('Sprite.rawFrameData')
private getRawFrameData() {
const animation = this.animation();
const frameId = this.frame() % animation.length;
return getImageData(animation[frameId]);
}
@cached('Sprite.frameData')
private getFrameData() {
if (!this.requiresProcessing()) {
return this.getRawFrameData();
}
const skin = this.skin();
const mask = this.mask();
const blend = this.maskBlend();
if (!this.context || !animation || animation.length === 0) return;
const rawFrameData = this.getRawFrameData();
const frameData = this.context.createImageData(rawFrameData);
const frameId = this.frame() % animation.length;
this.spriteData = animation[frameId];
this.offset(this.getOriginOffset());
this.imageData = this.context.createImageData(
this.spriteData.width,
this.spriteData.height,
);
this.computeCanvas.width = rawFrameData.width;
this.computeCanvas.height = rawFrameData.height;
if (skin) {
for (let y = 0; y < this.spriteData.height; y++) {
for (let x = 0; x < this.spriteData.width; x++) {
const skinData = this.getSkinData();
for (let y = 0; y < rawFrameData.height; y++) {
for (let x = 0; x < rawFrameData.width; x++) {
const id = this.positionToId({x, y});
const skinX = this.spriteData.data[id];
const skinY = this.spriteData.data[id + 1];
const skinX = rawFrameData.data[id];
const skinY = rawFrameData.data[id + 1];
const skinId = ((skin.height - 1 - skinY) * skin.width + skinX) * 4;
this.imageData.data[id] = skin.data[skinId];
this.imageData.data[id + 1] = skin.data[skinId + 1];
this.imageData.data[id + 2] = skin.data[skinId + 2];
this.imageData.data[id + 3] = Math.round(
(this.spriteData.data[id + 3] / 255) *
(skin.data[skinId + 3] / 255) *
frameData.data[id] = skinData.data[skinId];
frameData.data[id + 1] = skinData.data[skinId + 1];
frameData.data[id + 2] = skinData.data[skinId + 2];
frameData.data[id + 3] = Math.round(
(rawFrameData.data[id + 3] / 255) *
(skinData.data[skinId + 3] / 255) *
255,
);
}
}
} else {
this.imageData.data.set(this.spriteData.data);
frameData.data.set(rawFrameData.data);
}
if (mask || this.baseMask) {
for (let y = 0; y < this.spriteData.height; y++) {
for (let x = 0; x < this.spriteData.width; x++) {
const maskData = this.getMaskData();
const baseMaskData = this.getBaseMaskData();
for (let y = 0; y < rawFrameData.height; y++) {
for (let x = 0; x < rawFrameData.width; x++) {
const id = this.positionToId({x, y});
const maskValue = map(
mask?.data[id] ?? 255,
this.baseMask?.data[id] ?? 255,
maskData?.data[id] ?? 255,
baseMaskData?.data[id] ?? 255,
this.baseMaskBlend,
);
this.imageData.data[id + 3] *= map(1, maskValue / 255, blend);
frameData.data[id + 3] *= map(1, maskValue / 255, blend);
}
}
}
this.context.putImageData(this.imageData, 0, 0);
this.context.putImageData(frameData, 0, 0);
this.fire(SPRITE_CHANGE_EVENT);
this.markDirty();
return frameData;
}
private requiresProcessing(): boolean {
return !!(this.skin() || this.mask() || this.baseMask);
}
/**
* A generator that runs this sprite's animation.
*
* For the animation to actually play, the {@link Sprite.playing} value has to
* be set to `true`.
*
* Should be run concurrently:
* ```ts
* yield sprite.play();
* ```
*/
public play(): ThreadGenerator {
const runTask = this.playRunner();
if (this.task) {
@@ -164,10 +234,17 @@ export class Sprite extends Shape {
return this.task;
}
/**
* Play the given animation once.
*
* @param animation The animation to play.
* @param next An optional animation that should be switched to next. If not
* present the sprite will go back to the previous animation.
*/
@threadable()
public *playOnce(
animation: SpriteData[],
next: SpriteData[] = null,
animation: ImageDataSource[],
next: ImageDataSource[] = null,
): ThreadGenerator {
next ??= this.animation();
this.animation(animation);
@@ -178,6 +255,11 @@ export class Sprite extends Shape {
this.animation(next);
}
/**
* Cancel the current {@link Sprite.play()} generator.
*
* Should be used instead of manually canceling the generator.
*/
@threadable()
public *stop() {
if (this.task) {
@@ -186,8 +268,6 @@ export class Sprite extends Shape {
}
}
private synced = false;
@threadable('spriteAnimationRunner')
private *playRunner(): ThreadGenerator {
this.frame(0);
@@ -202,6 +282,11 @@ export class Sprite extends Shape {
}
}
/**
* Wait until the given frame is shown.
*
* @param frame
*/
@threadable()
public *waitForFrame(frame: number): ThreadGenerator {
let limit = 1000;
@@ -221,45 +306,52 @@ export class Sprite extends Shape {
@threadable()
private *maskTween(
from: SpriteData,
to: SpriteData,
from: ImageDataSource,
to: ImageDataSource,
time: number,
interpolation: InterpolationFunction,
onEnd: () => void,
): ThreadGenerator {
this.baseMask = from;
this._clearCache(this.getBaseMaskData);
this.baseMaskBlend = 1;
this.mask(to);
yield* tween(time, value => {
this.baseMaskBlend = interpolation(1 - value);
this.recalculate();
this.updateFrame();
});
this.baseMask = null;
this.baseMaskBlend = 0;
onEnd();
}
public getColorAt(position: Vector2d): string {
const id = this.positionToId(position);
return `rgba(${this.imageData.data[id]
const frameData = this.getFrameData();
return `rgba(${frameData.data[id]
.toString()
.padStart(3, ' ')}, ${this.imageData.data[id + 1]
.padStart(3, ' ')}, ${frameData.data[id + 1]
.toString()
.padStart(3, ' ')}, ${this.imageData.data[id + 2]
.padStart(3, ' ')}, ${frameData.data[id + 2]
.toString()
.padStart(3, ' ')}, ${this.imageData.data[id + 3] / 255})`;
.padStart(3, ' ')}, ${frameData.data[id + 3] / 255})`;
}
public getParsedColorAt(position: Vector2d): ReturnType<typeof parseColor> {
const id = this.positionToId(position);
const frameData = this.getFrameData();
return {
r: this.imageData.data[id],
g: this.imageData.data[id + 1],
b: this.imageData.data[id + 2],
a: this.imageData.data[id + 3],
r: frameData.data[id],
g: frameData.data[id + 1],
b: frameData.data[id + 2],
a: frameData.data[id + 3],
};
}
public positionToId(position: Vector2d): number {
return (position.y * this.imageData.width + position.x) * 4;
const frameData = this.getRawFrameData();
return (position.y * frameData.width + position.x) * 4;
}
}

View File

@@ -1,22 +1,47 @@
/* eslint-disable @typescript-eslint/no-unused-vars,@typescript-eslint/no-namespace */
declare module '*.png' {
const value: import('./components/Sprite').SpriteData;
export = value;
}
declare module '*.glsl' {
const value: string;
export = value;
}
declare module '*.label' {
const value: Record<string, number>;
declare module '*.png?img' {
const value: Promise<HTMLImageElement>;
export = value;
}
declare module '*.anim' {
const value: import('./components/Sprite').SpriteData[];
declare module '*.png?anim' {
const value: Promise<HTMLImageElement[]>;
export = value;
}
declare module '*.jpg' {
const value: string;
export = value;
}
declare module '*.jpg?img' {
const value: Promise<HTMLImageElement>;
export = value;
}
declare module '*.jpg?anim' {
const value: Promise<HTMLImageElement[]>;
export = value;
}
declare module '*.jpeg' {
const value: string;
export = value;
}
declare module '*.jpeg?img' {
const value: Promise<HTMLImageElement>;
export = value;
}
declare module '*.jpeg?anim' {
const value: Promise<HTMLImageElement[]>;
export = value;
}
@@ -25,12 +50,27 @@ declare module '*.wav' {
export = value;
}
declare module '*.mp3' {
const value: string;
export = value;
}
declare module '*.ogg' {
const value: string;
export = value;
}
declare module '*.mp4' {
const value: string;
export = value;
}
declare module '*.csv' {
const value: unknown;
export = value;
}
declare module '*.mp4' {
declare module '*.glsl' {
const value: string;
export = value;
}

View File

@@ -1,2 +1,4 @@
export * from './AudioData';
export * from './AudioManager';
export * from './loadImage';
export * from './loadVideo';

31
src/media/loadImage.ts Normal file
View File

@@ -0,0 +1,31 @@
import {Size} from '../types';
const imageLookup: Record<string, HTMLImageElement> = {};
const canvas = document.createElement('canvas');
const context = canvas.getContext('2d');
export type ImageDataSource = CanvasImageSource & Size;
export async function loadImage(source: string): Promise<HTMLImageElement> {
if (!imageLookup[source]) {
const image = new Image();
imageLookup[source] = image;
image.src = source;
await new Promise(resolve => (image.onload = resolve));
}
return imageLookup[source];
}
export function loadAnimation(sources: string[]): Promise<HTMLImageElement[]> {
return Promise.all(sources.map(loadImage));
}
export function getImageData(image: ImageDataSource) {
canvas.width = image.width;
canvas.height = image.height;
context.clearRect(0, 0, image.width, image.height);
context.drawImage(image, 0, 0);
return context.getImageData(0, 0, image.width, image.height);
}

View File

@@ -9,24 +9,129 @@ import {useScene} from '../utils';
declare module 'konva/lib/Node' {
export interface Node {
/**
* @internal
*/
_centroid: boolean;
style?: GetSet<Partial<Style>, this>;
/**
* The empty space between the borders of this node and its content.
*
* Analogous to CSS padding.
*/
padd: GetSet<PossibleSpacing, this>;
/**
* The empty space between the borders of this node and surrounding nodes.
*
* Analogous to CSS margin.
*/
margin: GetSet<PossibleSpacing, this>;
/**
* The origin of this node.
*
* By default, each node has its origin in the middle.
*
* Analogous to CSS margin.
*/
origin: GetSet<Origin, this>;
drawOrigin: GetSet<Origin, this>;
/**
* @param value
* @ignore
*/
setX(value: number): this;
/**
* @param value
* @ignore
*/
setY(value: number): this;
/**
* @param width
* @ignore
*/
setWidth(width: number): void;
/**
* @param height
* @ignore
*/
setHeight(height: number): void;
/**
* @param value
* @ignore
*/
setPadd(value: PossibleSpacing): this;
/**
* @param value
* @ignore
*/
setMargin(value: PossibleSpacing): this;
/**
* @param value
* @ignore
*/
setOrigin(value: Origin): this;
/**
* @ignore
*/
getPadd(): Spacing;
/**
* @ignore
*/
getMargin(): Spacing;
/**
* @ignore
*/
getOrigin(): Origin;
/**
* Get the size of this node used for layout calculations.
*
* The returned size should include the padding.
* A node can use the size of its children to derive its own dimensions.
*
* @param custom Custom node configuration to use during the calculations.
* When present, the method will return the layout size that
* the node would have, if it had these options configured.
*/
getLayoutSize(custom?: NodeConfig): Size;
/**
* Get the vector from the local origin of this node to its current origin.
*
* The local origin is the center of coordinates of the canvas when drawing
* the node. Centroid nodes will have their local origin at the center.
* Other shapes will have it in the top left corner.
*
* The current origin is configured via {@link Node.origin}.
*
* @param custom Custom node configuration to use during the calculations.
* When present, the method will return the origin offset that
* the node would have, if it had these options configured.
*/
getOriginOffset(custom?: NodeConfig): Vector2d;
/**
* Get the vector from the current origin of this node to the `newOrigin`.
*
* @param newOrigin The origin to which the delta should be calculated.
*
* @param custom Custom node configuration to use during the calculations.
* When present, the method will return the origin offset that
* the node would have, if it had these options configured.
*/
getOriginDelta(newOrigin: Origin, custom?: NodeConfig): Vector2d;
/**

View File

@@ -3,7 +3,7 @@ import {
SimpleEventDispatcher,
} from 'strongly-typed-events';
import {AudioManager} from '../audio';
import {AudioManager} from '../media';
import type {Project} from '../Project';
const MAX_AUDIO_DESYNC = 1 / 50;

View File

@@ -1 +0,0 @@
export * from './loadVideo';

View File

@@ -1,5 +1,6 @@
{
"out": "api",
"excludeExternals": true,
"entryPoints": [
"src",
"src/animations",