feat: waveform data

src/animations/surfaceTransition.ts

@@ -41,8 +41,8 @@ export function surfaceTransition(fromSurfaceOriginal: Surface, clone = true) {
   }
   const from = fromSurfaceOriginal.getMask();
 
-  decorate(surfaceTransitionExecutor, threadable());
-  function* surfaceTransitionExecutor(
+  decorate(surfaceTransitionRunner, threadable());
+  function* surfaceTransitionRunner(
     target: Surface,
     config: SurfaceTransitionConfig = {},
   ): ThreadGenerator {
@@ -150,5 +150,5 @@ export function surfaceTransition(fromSurfaceOriginal: Surface, clone = true) {
     target.show();
   }
 
-  return surfaceTransitionExecutor;
+  return surfaceTransitionRunner;
 }

src/components/Sprite.ts

@@ -188,7 +188,7 @@ export class Sprite extends LayoutShape {
 
   private synced = false;
 
-  @threadable()
+  @threadable('spriteAnimationRunner')
   private *playRunner(): ThreadGenerator {
     this.frame(0);
     while (this.task !== null) {

src/global.d.ts

@@ -17,3 +17,13 @@ declare module '*.anim' {
   const value: import('./components/Sprite').SpriteData[];
   export = value;
 }
+
+declare module '*.wav' {
+  const value: string;
+  export = value;
+}
+
+declare module '*.wav?meta' {
+  const value: import('./types/Waveform').Waveform;
+  export = value;
+}

src/player/Player.ts

@@ -1,4 +1,5 @@
 import type {Project} from '../Project';
+import type {Waveform} from '../types';
 import {
   PromiseSimpleEventDispatcher,
   SimpleEventDispatcher,
@@ -58,7 +59,7 @@ export class Player {
   private readonly renderChanged =
     new PromiseSimpleEventDispatcher<PlayerRenderEvent>();
 
-  private readonly audio: HTMLAudioElement = null;
+  private readonly audioElement: HTMLAudioElement = null;
   private startTime: number;
   private renderTime: number = 0;
   private requestId: number = null;
@@ -138,7 +139,10 @@ export class Player {
 
   public constructor(
     public readonly project: Project,
-    audioSrc?: string,
+    public readonly audio?: {
+      src: string,
+      meta: Waveform,
+    },
     public readonly labels?: Record<string, number>,
   ) {
     this.startTime = performance.now();
@@ -154,8 +158,8 @@ export class Player {
       this.state.muted = state.muted;
     }
 
-    if (audioSrc) {
-      this.audio = new Audio(audioSrc);
+    if (audio) {
+      this.audioElement = new Audio(audio.src);
     }
 
     this.request();
@@ -233,12 +237,12 @@ export class Player {
 
     // Pause / play audio.
     const audioPaused = state.paused || state.finished || state.render;
-    if (this.audio && this.audio.paused !== audioPaused) {
+    if (this.audioElement && this.audioElement.paused !== audioPaused) {
       if (audioPaused) {
-        this.audio.pause();
+        this.audioElement.pause();
       } else {
         try {
-          await this.audio.play();
+          await this.audioElement.play();
           this.syncAudio(-3);
           this.audioError = false;
         } catch (e) {
@@ -249,8 +253,8 @@ export class Player {
         }
       }
     }
-    if (this.audio) {
-      this.audio.muted = state.muted;
+    if (this.audioElement) {
+      this.audioElement.muted = state.muted;
     }
 
     // Rendering
@@ -286,7 +290,7 @@ export class Player {
       state.paused ||
       (state.speed === 1 &&
         this.hasAudio() &&
-        this.audio.currentTime < this.project.time)
+        this.audioElement.currentTime < this.project.time)
     ) {
       this.request();
       return;
@@ -295,9 +299,9 @@ export class Player {
     else if (
       this.hasAudio() &&
       state.speed === 1 &&
-      this.project.time < this.audio.currentTime - MAX_AUDIO_DESYNC
+      this.project.time < this.audioElement.currentTime - MAX_AUDIO_DESYNC
     ) {
-      const seekFrame = this.project.secondsToFrames(this.audio.currentTime);
+      const seekFrame = this.project.secondsToFrames(this.audioElement.currentTime);
       state.finished = await this.project.seek(seekFrame, state.speed);
     }
     // Simply move forward one frame
@@ -341,13 +345,13 @@ export class Player {
   }
 
   private hasAudio(): boolean {
-    return this.audio && !this.audioError;
+    return this.audioElement && !this.audioError;
   }
 
   private syncAudio(frameOffset: number = 0) {
-    if (!this.audio) return;
+    if (!this.audioElement) return;
 
-    this.audio.currentTime = Math.max(
+    this.audioElement.currentTime = Math.max(
       0,
       this.project.framesToSeconds(this.project.frame + frameOffset),
     );

src/types/Waveform.ts

@@ -0,0 +1,9 @@
+export interface Waveform {
+  version: number;
+  channels: number;
+  sample_rate: number;
+  samples_per_pixel: number;
+  bits: 8 | 16;
+  length: number;
+  data: number[];
+}

src/types/index.ts

@@ -1,3 +1,4 @@
 export * from './Size';
 export * from './Origin';
-export * from './Spacing';
+export * from './Spacing';
+export * from './Waveform';

tools/loaders/wav-loader.js

@@ -0,0 +1,27 @@
+const {exec} = require('child_process');
+const {promises: fs} = require('fs');
+const loaderUtils = require('loader-utils');
+const path = require('path');
+
+function loader() {
+  const source = this.resourcePath;
+  const destination = source.slice(0, -4) + '.json';
+
+  const callback = this.async();
+  exec(
+    `..\\bin\\audiowaveform.exe -i ${source} -o ${destination}`,
+    {cwd: __dirname},
+    async (error, stdout) => {
+      if (error) {
+        callback(error);
+        return;
+      }
+
+      fs.readFile(destination, 'utf8')
+        .then(data => callback(null, `export default ${data};`))
+        .catch(callback);
+    },
+  );
+}
+
+module.exports = loader;

tools/serve.mjs

@@ -75,6 +75,23 @@ const compiler = webpack({
           },
         ],
       },
+      {
+        oneOf: [
+          {
+            test: /\.wav$/i,
+            resourceQuery: /meta/,
+            use: [
+              {
+                loader: 'wav-loader',
+              },
+            ],
+          },
+          {
+            test: /\.wav$/i,
+            type: 'asset',
+          },
+        ],
+      },
     ],
   },
   resolveLoader: {