feat: three.js integration

package-lock.json

@@ -13,6 +13,7 @@
         "image-size": "^1.0.1",
         "konva": "^8.3.2",
         "mix-color": "^1.1.2",
+        "three": "^0.138.3",
         "ts-loader": "^9.2.6",
         "typescript": "^4.5.5",
         "url-loader": "^4.1.1",
@@ -25,6 +26,7 @@
         "mc-serve": "tools/serve.mjs"
       },
       "devDependencies": {
+        "@types/three": "^0.138.0",
         "node-loader": "^2.0.0",
         "prettier": "^2.5.1",
         "webpack-cli": "^4.9.2"
@@ -231,6 +233,12 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/three": {
+      "version": "0.138.0",
+      "resolved": "https://registry.npmjs.org/@types/three/-/three-0.138.0.tgz",
+      "integrity": "sha512-D8AoV7h2kbCfrv/DcebHOFh1WDwyus3HdooBkAwcBikXArdqnsQ38PQ85JCunnvun160oA9jz53GszF3zch3tg==",
+      "dev": true
+    },
     "node_modules/@types/ws": {
       "version": "8.2.2",
       "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.2.2.tgz",
@@ -3623,6 +3631,11 @@
         "node": ">= 8"
       }
     },
+    "node_modules/three": {
+      "version": "0.138.3",
+      "resolved": "https://registry.npmjs.org/three/-/three-0.138.3.tgz",
+      "integrity": "sha512-4t1cKC8gimNyJChJbaklg8W/qj3PpsLJUIFm5LIuAy/hVxxNm1ru2FGTSfbTSsuHmC/7ipsyuGKqrSAKLNtkzg=="
+    },
     "node_modules/thunky": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz",
@@ -4381,6 +4394,12 @@
         "@types/node": "*"
       }
     },
+    "@types/three": {
+      "version": "0.138.0",
+      "resolved": "https://registry.npmjs.org/@types/three/-/three-0.138.0.tgz",
+      "integrity": "sha512-D8AoV7h2kbCfrv/DcebHOFh1WDwyus3HdooBkAwcBikXArdqnsQ38PQ85JCunnvun160oA9jz53GszF3zch3tg==",
+      "dev": true
+    },
     "@types/ws": {
       "version": "8.2.2",
       "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.2.2.tgz",
@@ -6870,6 +6889,11 @@
         "terser": "^5.7.2"
       }
     },
+    "three": {
+      "version": "0.138.3",
+      "resolved": "https://registry.npmjs.org/three/-/three-0.138.3.tgz",
+      "integrity": "sha512-4t1cKC8gimNyJChJbaklg8W/qj3PpsLJUIFm5LIuAy/hVxxNm1ru2FGTSfbTSsuHmC/7ipsyuGKqrSAKLNtkzg=="
+    },
     "thunky": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz",

package.json

@@ -17,6 +17,7 @@
     "image-size": "^1.0.1",
     "konva": "^8.3.2",
     "mix-color": "^1.1.2",
+    "three": "^0.138.3",
     "ts-loader": "^9.2.6",
     "typescript": "^4.5.5",
     "url-loader": "^4.1.1",
@@ -34,6 +35,7 @@
     "tsconfig.project.json"
   ],
   "devDependencies": {
+    "@types/three": "^0.138.0",
     "node-loader": "^2.0.0",
     "prettier": "^2.5.1",
     "webpack-cli": "^4.9.2"

src/components/ThreeView.ts

@@ -0,0 +1,181 @@
+import {LayoutShape, LayoutShapeConfig} from './LayoutShape';
+import {Size} from '../types';
+import {Util} from 'konva/lib/Util';
+import {Context} from 'konva/lib/Context';
+
+import * as THREE from 'three';
+import {GetSet} from 'konva/lib/types';
+import {Factory} from 'konva/lib/Factory';
+
+export interface ThreeViewConfig extends LayoutShapeConfig {
+  canvasSize: Size;
+  cameraScale?: number;
+  quality?: number;
+  skipFrames?: number;
+}
+
+interface Pool<T> {
+  borrow(): T;
+  dispose(object: T): void;
+}
+
+class CanvasPool implements Pool<HTMLCanvasElement> {
+  private pool: HTMLCanvasElement[] = [];
+
+  public borrow(): HTMLCanvasElement {
+    if (this.pool.length) {
+      return this.pool.pop();
+    } else {
+      return Util.createCanvasElement();
+    }
+  }
+
+  public dispose(canvas: HTMLCanvasElement) {
+    this.pool.push(canvas);
+  }
+}
+
+const canvasPool2D = new CanvasPool();
+const canvasPool3D = new CanvasPool();
+
+export class ThreeView extends LayoutShape {
+  public canvasSize: GetSet<Size, this>;
+  public cameraScale: GetSet<number, this>;
+  public quality: GetSet<number, this>;
+  public skipFrames: GetSet<number, this>;
+  public readonly scene: THREE.Scene;
+  public readonly camera: THREE.OrthographicCamera;
+
+  private readonly threeCanvas: HTMLCanvasElement;
+  private readonly copyCanvas: HTMLCanvasElement;
+  private readonly renderer: THREE.WebGLRenderer;
+  private readonly context: WebGLRenderingContext;
+  private readonly copyContext: CanvasRenderingContext2D;
+
+  private copyData: ImageData;
+  private pixels: Uint8ClampedArray;
+  private renderedFrames: number = 0;
+
+  public constructor(config?: ThreeViewConfig) {
+    super(config);
+    this.threeCanvas = canvasPool3D.borrow();
+    this.copyCanvas = canvasPool2D.borrow();
+    this.copyContext = this.copyCanvas.getContext('2d');
+
+    this.scene = new THREE.Scene();
+    this.scene.background = new THREE.Color(0x568585);
+    this.camera = new THREE.OrthographicCamera(-1, 1, 1, -1, 1, 1000);
+    this.renderer = new THREE.WebGLRenderer({
+      canvas: this.threeCanvas,
+      antialias: true,
+    });
+    this.context = this.renderer.getContext();
+    this.camera.position.z = 3;
+    this.camera.position.y = 2;
+
+    this.handleCanvasSizeChange();
+  }
+
+  destroy(): this {
+    this.renderer.dispose();
+    canvasPool2D.dispose(this.copyCanvas);
+    canvasPool3D.dispose(this.threeCanvas);
+
+    return super.destroy();
+  }
+
+  private handleCanvasSizeChange() {
+    if (!this.renderer) return;
+
+    const size = {...this.canvasSize()};
+
+    const ratio = size.width / size.height;
+    const scale = this.cameraScale() / 2;
+    this.camera.left = -ratio * scale;
+    this.camera.right = ratio * scale;
+    this.camera.bottom = -scale;
+    this.camera.top = scale;
+
+    size.width *= this.quality();
+    size.height *= this.quality();
+    this.renderer.setSize(size.width, size.height);
+    this.camera.updateProjectionMatrix();
+    this.copyCanvas.width = size.width;
+    this.copyCanvas.height = size.height;
+    this.copyData = this.copyContext.createImageData(size.width, size.height);
+    this.pixels = new Uint8ClampedArray(size.width * size.height * 4);
+  }
+
+  getLayoutSize(): Size {
+    return this.canvasSize();
+  }
+
+  _sceneFunc(context: Context) {
+    const scale = this.quality();
+    const size = {...this.canvasSize()};
+    size.width *= scale;
+    size.height *= scale;
+
+    if (this.renderedFrames < 1) {
+      this.renderedFrames = this.skipFrames();
+      this.renderer.render(this.scene, this.camera);
+      this.context.readPixels(
+        0,
+        0,
+        size.width,
+        size.height,
+        this.context.RGBA,
+        this.context.UNSIGNED_BYTE,
+        this.pixels,
+      );
+
+      this.copyData.data.set(this.pixels);
+      this.copyContext.putImageData(this.copyData, 0, 0);
+    } else {
+      this.renderedFrames--;
+    }
+
+    context.save();
+    context._context.imageSmoothingEnabled = false;
+    context.scale(1 / scale, 1 / -scale);
+    context.drawImage(
+      this.copyCanvas,
+      // this.copyImage,
+      0,
+      0,
+      size.width,
+      size.height,
+      size.width / -2,
+      size.height / -2,
+      size.width,
+      size.height,
+    );
+    context.restore();
+  }
+}
+
+Factory.addGetterSetter(
+  ThreeView,
+  'canvasSize',
+  {width: 1, height: 1},
+  undefined,
+  //@ts-ignore
+  ThreeView.handleCanvasSizeChange,
+);
+Factory.addGetterSetter(
+  ThreeView,
+  'cameraScale',
+  1,
+  undefined,
+  //@ts-ignore
+  ThreeView.handleCanvasSizeChange,
+);
+Factory.addGetterSetter(
+  ThreeView,
+  'quality',
+  1,
+  undefined,
+  //@ts-ignore
+  ThreeView.handleCanvasSizeChange,
+);
+Factory.addGetterSetter(ThreeView, 'skipFrames', 0);

src/components/index.ts

@@ -1,3 +1,8 @@
 export * from './Arrow';
 export * from './Connection';
-export * from './TextContent';
+export * from './Debug';
+export * from './Layout';
+export * from './Sprite';
+export * from './Surface';
+export * from './TextContent';
+export * from './ThreeView';

src/global.d.ts

@@ -1,4 +1,9 @@
 declare module "*.png" {
   const value: any;
   export = value;
+}
+
+declare module "*.glsl" {
+  const value: string;
+  export = value;
 }

src/materials/MeshBoneMaterial.ts

@@ -0,0 +1,86 @@
+import {
+  MaterialParameters,
+  ShaderLib,
+  ShaderMaterial,
+  Texture,
+  Vector4,
+} from 'three';
+
+import fragmentShader from 'MC/shaders/bone_highlight_fragment.glsl';
+import vertexShader from 'MC/shaders/bone_highlight_vertex.glsl';
+
+export interface MeshBoneMaterialParameters extends MaterialParameters {
+  highlight?: number;
+  map?: Texture;
+  wireframe?: boolean;
+  highlightColor?: Vector4;
+}
+
+export class MeshBoneMaterial extends ShaderMaterial {
+  public set map(value: Texture) {
+    if (value) {
+      this.uniforms.map = {value};
+    } else {
+      delete this.uniforms.map;
+    }
+  }
+
+  public get map(): Texture {
+    return this.uniforms.map?.value ?? null;
+  }
+
+  public set highlight(value: number | null) {
+    this.uniforms.highlight = {value: value ?? -1};
+  }
+
+  public get highlight(): number | null {
+    return this.uniforms.highlight?.value === -1
+      ? null
+      : this.uniforms.highlight?.value;
+  }
+
+  public set highlightColor(value: Vector4) {
+    this.uniforms.highlightDiffuse = {value};
+  }
+
+  public get highlightColor(): Vector4 {
+    return this.uniforms.highlightDiffuse?.value;
+  }
+
+  public set transparency(value: number) {
+    this.uniforms.opacity.value = value;
+  }
+
+  public get transparency(): number {
+    return this.uniforms.opacity.value;
+  }
+
+  public set wireframeOverlay(value: boolean) {
+    if (value) {
+      this.defines.USE_WIREFRAME = {value: true};
+    } else {
+      delete this.defines.USE_WIREFRAME;
+    }
+  }
+
+  public get wireframeOverlay(): boolean {
+    return !!this.defines.USE_WIREFRAME;
+  }
+
+  public constructor(parameters: MeshBoneMaterialParameters = {}) {
+    const {map, highlight, highlightColor, opacity, wireframe, ...rest} =
+      parameters;
+    super({
+      ...rest,
+      fragmentShader,
+      vertexShader,
+    });
+
+    this.uniforms = {...ShaderLib.basic.uniforms};
+    this.map = map;
+    this.highlight = highlight;
+    this.highlightColor = highlightColor ?? new Vector4(0, 0, 0, 0);
+    this.transparency = opacity;
+    this.wireframeOverlay = wireframe;
+  }
+}

src/shaders/bone_highlight_fragment.glsl

@@ -0,0 +1,81 @@
+uniform vec3 diffuse;
+uniform float opacity;
+uniform vec4 highlightDiffuse;
+
+varying float vHighlight;
+varying vec3 vCenter;
+#ifndef FLAT_SHADED
+    varying vec3 vNormal;
+#endif
+
+
+#include <common>
+#include <dithering_pars_fragment>
+#include <color_pars_fragment>
+#include <uv_pars_fragment>
+#include <uv2_pars_fragment>
+#include <map_pars_fragment>
+#include <alphamap_pars_fragment>
+#include <alphatest_pars_fragment>
+#include <aomap_pars_fragment>
+#include <lightmap_pars_fragment>
+#include <envmap_common_pars_fragment>
+#include <envmap_pars_fragment>
+#include <cube_uv_reflection_fragment>
+#include <fog_pars_fragment>
+#include <specularmap_pars_fragment>
+#include <logdepthbuf_pars_fragment>
+#include <clipping_planes_pars_fragment>
+
+void main() {
+    #include <clipping_planes_fragment>
+    vec4 diffuseColor = vec4( diffuse, opacity );
+    #include <logdepthbuf_fragment>
+    #include <map_fragment>
+    #include <color_fragment>
+    #include <alphamap_fragment>
+    #include <alphatest_fragment>
+    #include <specularmap_fragment>
+    ReflectedLight reflectedLight = ReflectedLight( vec3( 0.0 ), vec3( 0.0 ), vec3( 0.0 ), vec3( 0.0 ) );
+    #ifdef USE_LIGHTMAP
+        vec4 lightMapTexel= texture2D( lightMap, vUv2 );
+        reflectedLight.indirectDiffuse += lightMapTexel.rgb * lightMapIntensity;
+    #else
+        reflectedLight.indirectDiffuse += vec3( 1.0 );
+    #endif
+    #include <aomap_fragment>
+    reflectedLight.indirectDiffuse *= diffuseColor.rgb;
+    vec3 outgoingLight = reflectedLight.indirectDiffuse;
+    #include <envmap_fragment>
+    #include <output_fragment>
+    #include <tonemapping_fragment>
+    #include <encodings_fragment>
+    #include <fog_fragment>
+    #include <premultiplied_alpha_fragment>
+    #include <dithering_fragment>
+
+    #ifdef USE_WIREFRAME
+        float thickness = 1.0;
+        vec3 afwidth = fwidth( vCenter.xyz );
+        vec3 edge3 = smoothstep( ( thickness - 1.0 ) * afwidth, thickness * afwidth, vCenter.xyz );
+        float edge = 1.0 - min( min( edge3.x, edge3.y ), edge3.z );
+
+        gl_FragColor = mix(
+            gl_FragColor,
+            mix(vec4(0.0, 0.0, 0.0, 1.0), highlightDiffuse, vHighlight),
+            edge
+        );
+    #else
+        gl_FragColor.rgb = mix(
+            gl_FragColor.rgb,
+            highlightDiffuse.rgb * mix((gl_FragColor.r + gl_FragColor.b + gl_FragColor.g) / 3.0, 1.0, 0.7),
+            vHighlight
+        );
+
+        gl_FragColor.a = mix(
+            mix(gl_FragColor.a, highlightDiffuse.a, vHighlight),
+            gl_FragColor.a,
+            step(gl_FragColor.a, 0.3)
+        );
+    #endif
+}

src/shaders/bone_highlight_vertex.glsl

@@ -0,0 +1,47 @@
+#include <common>
+#include <uv_pars_vertex>
+#include <uv2_pars_vertex>
+#include <envmap_pars_vertex>
+#include <color_pars_vertex>
+#include <fog_pars_vertex>
+#include <morphtarget_pars_vertex>
+#include <skinning_pars_vertex>
+#include <logdepthbuf_pars_vertex>
+#include <clipping_planes_pars_vertex>
+
+uniform float highlight;
+attribute vec3 center;
+
+varying float vHighlight;
+varying vec3 vCenter;
+
+void main() {
+    #include <uv_vertex>
+    #include <uv2_vertex>
+    #include <color_vertex>
+    #include <morphcolor_vertex>
+    #if defined ( USE_ENVMAP ) || defined ( USE_SKINNING )
+        #include <beginnormal_vertex>
+        #include <morphnormal_vertex>
+        #include <skinbase_vertex>
+        #include <skinnormal_vertex>
+        #include <defaultnormal_vertex>
+    #endif
+    #include <begin_vertex>
+    #include <morphtarget_vertex>
+    #include <skinning_vertex>
+    #include <project_vertex>
+    #include <logdepthbuf_vertex>
+    #include <clipping_planes_vertex>
+    #include <worldpos_vertex>
+    #include <envmap_vertex>
+    #include <fog_vertex>
+
+    #ifdef USE_SKINNING
+        vec4 highlights = skinWeight * step(skinIndex, vec4(highlight)) * step(vec4(highlight), skinIndex);
+        vHighlight = highlights.x + highlights.y + highlights.z + highlights.w;
+    #else
+        vHighlight = 0.0;
+    #endif
+    vCenter = center;
+}

tools/serve.mjs

@@ -34,6 +34,10 @@ const compiler = webpack({
           },
         ],
       },
+      {
+        test: /\.glsl$/i,
+        type: 'asset/source',
+      },
     ],
   },
   resolve: {