feat(ui): updated cursor position tracking

- Record both absolute and relative positions
- Use simpler method to get relative position
- Generalize getColorUnderCursor to be getColorAtCoordinate

invokeai/frontend/web/src/features/controlLayers/konva/CanvasTool/CanvasToolBrush.ts

@@ -124,7 +124,7 @@ export class CanvasToolBrush extends CanvasModuleBase {
 
     const settings = this.manager.stateApi.getSettings();
     const brushPreviewFill = this.manager.stateApi.getBrushPreviewColor();
-    const alignedCursorPos = alignCoordForTool(cursorPos, settings.brushWidth);
+    const alignedCursorPos = alignCoordForTool(cursorPos.relative, settings.brushWidth);
     const radius = settings.brushWidth / 2;
 
     // The circle is scaled
@@ -141,14 +141,14 @@ export class CanvasToolBrush extends CanvasModuleBase {
     const twoPixels = this.manager.stage.unscale(2);
 
     this.konva.innerBorder.setAttrs({
-      x: cursorPos.x,
-      y: cursorPos.y,
+      x: cursorPos.relative.x,
+      y: cursorPos.relative.y,
       innerRadius: radius,
       outerRadius: radius + onePixel,
     });
     this.konva.outerBorder.setAttrs({
-      x: cursorPos.x,
-      y: cursorPos.y,
+      x: cursorPos.relative.x,
+      y: cursorPos.relative.y,
       innerRadius: radius + onePixel,
       outerRadius: radius + twoPixels,
     });

invokeai/frontend/web/src/features/controlLayers/konva/CanvasTool/CanvasToolColorPicker.ts

@@ -207,6 +207,8 @@ export class CanvasToolColorPicker extends CanvasModuleBase {
 
     this.setVisibility(true);
 
+    const { x, y } = cursorPos.relative;
+
     const settings = this.manager.stateApi.getSettings();
     const colorUnderCursor = this.parent.$colorUnderCursor.get();
     const colorPickerInnerRadius = this.manager.stage.unscale(this.config.RING_INNER_RADIUS);
@@ -215,28 +217,28 @@ export class CanvasToolColorPicker extends CanvasModuleBase {
     const twoPixels = this.manager.stage.unscale(2);
 
     this.konva.ringCandidateColor.setAttrs({
-      x: cursorPos.x,
-      y: cursorPos.y,
+      x,
+      y,
       fill: rgbColorToString(colorUnderCursor),
       innerRadius: colorPickerInnerRadius,
       outerRadius: colorPickerOuterRadius,
     });
     this.konva.ringCurrentColor.setAttrs({
-      x: cursorPos.x,
-      y: cursorPos.y,
+      x,
+      y,
       fill: rgbColorToString(settings.color),
       innerRadius: colorPickerInnerRadius,
       outerRadius: colorPickerOuterRadius,
     });
     this.konva.ringInnerBorder.setAttrs({
-      x: cursorPos.x,
-      y: cursorPos.y,
+      x,
+      y,
       innerRadius: colorPickerOuterRadius,
       outerRadius: colorPickerOuterRadius + onePixel,
     });
     this.konva.ringOuterBorder.setAttrs({
-      x: cursorPos.x,
-      y: cursorPos.y,
+      x,
+      y,
       innerRadius: colorPickerOuterRadius + onePixel,
       outerRadius: colorPickerOuterRadius + twoPixels,
     });
@@ -249,35 +251,35 @@ export class CanvasToolColorPicker extends CanvasModuleBase {
     );
     this.konva.crosshairNorthOuter.setAttrs({
       strokeWidth: outerThickness,
-      points: [cursorPos.x, cursorPos.y - size, cursorPos.x, cursorPos.y - space],
+      points: [x, y - size, x, y - space],
     });
     this.konva.crosshairNorthInner.setAttrs({
       strokeWidth: innerThickness,
-      points: [cursorPos.x, cursorPos.y - size, cursorPos.x, cursorPos.y - space],
+      points: [x, y - size, x, y - space],
     });
     this.konva.crosshairEastOuter.setAttrs({
       strokeWidth: outerThickness,
-      points: [cursorPos.x + space, cursorPos.y, cursorPos.x + size, cursorPos.y],
+      points: [x + space, y, x + size, y],
     });
     this.konva.crosshairEastInner.setAttrs({
       strokeWidth: innerThickness,
-      points: [cursorPos.x + space, cursorPos.y, cursorPos.x + size, cursorPos.y],
+      points: [x + space, y, x + size, y],
     });
     this.konva.crosshairSouthOuter.setAttrs({
       strokeWidth: outerThickness,
-      points: [cursorPos.x, cursorPos.y + space, cursorPos.x, cursorPos.y + size],
+      points: [x, y + space, x, y + size],
     });
     this.konva.crosshairSouthInner.setAttrs({
       strokeWidth: innerThickness,
-      points: [cursorPos.x, cursorPos.y + space, cursorPos.x, cursorPos.y + size],
+      points: [x, y + space, x, y + size],
     });
     this.konva.crosshairWestOuter.setAttrs({
       strokeWidth: outerThickness,
-      points: [cursorPos.x - space, cursorPos.y, cursorPos.x - size, cursorPos.y],
+      points: [x - space, y, x - size, y],
     });
     this.konva.crosshairWestInner.setAttrs({
       strokeWidth: innerThickness,
-      points: [cursorPos.x - space, cursorPos.y, cursorPos.x - size, cursorPos.y],
+      points: [x - space, y, x - size, y],
     });
   };
 

invokeai/frontend/web/src/features/controlLayers/konva/CanvasTool/CanvasToolEraser.ts

@@ -104,7 +104,7 @@ export class CanvasToolEraser extends CanvasModuleBase {
     this.setVisibility(true);
 
     const settings = this.manager.stateApi.getSettings();
-    const alignedCursorPos = alignCoordForTool(cursorPos, settings.eraserWidth);
+    const alignedCursorPos = alignCoordForTool(cursorPos.relative, settings.eraserWidth);
     const radius = settings.eraserWidth / 2;
 
     // The circle is scaled
@@ -119,14 +119,14 @@ export class CanvasToolEraser extends CanvasModuleBase {
     const twoPixels = this.manager.stage.unscale(2);
 
     this.konva.innerBorder.setAttrs({
-      x: cursorPos.x,
-      y: cursorPos.y,
+      x: cursorPos.relative.x,
+      y: cursorPos.relative.y,
       innerRadius: radius,
       outerRadius: radius + onePixel,
     });
     this.konva.outerBorder.setAttrs({
-      x: cursorPos.x,
-      y: cursorPos.y,
+      x: cursorPos.relative.x,
+      y: cursorPos.relative.y,
       innerRadius: radius + onePixel,
       outerRadius: radius + twoPixels,
     });

invokeai/frontend/web/src/features/controlLayers/konva/CanvasTool/CanvasToolModule.ts

@@ -7,13 +7,12 @@ import {
   alignCoordForTool,
   calculateNewBrushSizeFromWheelDelta,
   floorCoord,
-  getColorUnderCursor,
+  getColorAtCoordinate,
   getIsPrimaryMouseDown,
   getLastPointOfLastLine,
   getLastPointOfLastLineWithPressure,
   getLastPointOfLine,
   getPrefixedId,
-  getScaledCursorPosition,
   isDistanceMoreThanMin,
   offsetCoord,
 } from 'features/controlLayers/konva/util';
@@ -83,7 +82,7 @@ export class CanvasToolModule extends CanvasModuleBase {
   /**
    * The last cursor position.
    */
-  $cursorPos = atom<Coordinate | null>(null);
+  $cursorPos = atom<{ relative: Coordinate; absolute: Coordinate } | null>(null);
   /**
    * The color currently under the cursor. Only has a value when the color picker tool is active.
    */
@@ -212,10 +211,15 @@ export class CanvasToolModule extends CanvasModuleBase {
     }
   };
 
-  syncLastCursorPos = (): Coordinate | null => {
-    const pos = getScaledCursorPosition(this.konva.stage);
-    this.$cursorPos.set(pos);
-    return pos;
+  syncCursorPositions = () => {
+    const relative = this.konva.stage.getRelativePointerPosition();
+    const absolute = this.konva.stage.getPointerPosition();
+
+    if (!relative || !absolute) {
+      return;
+    }
+
+    this.$cursorPos.set({ relative, absolute });
   };
 
   getClip = (
@@ -302,11 +306,14 @@ export class CanvasToolModule extends CanvasModuleBase {
   onStagePointerEnter = async (e: KonvaEventObject<PointerEvent>) => {
     try {
       this.$lastPointerType.set(e.evt.pointerType);
+
       if (!this.getCanDraw()) {
         return;
       }
 
-      const cursorPos = this.syncLastCursorPos();
+      this.syncCursorPositions();
+      const cursorPos = this.$cursorPos.get();
+
       const isMouseDown = this.$isMouseDown.get();
       const settings = this.manager.stateApi.getSettings();
       const tool = this.$tool.get();
@@ -322,7 +329,7 @@ export class CanvasToolModule extends CanvasModuleBase {
       }
 
       if (tool === 'brush') {
-        const normalizedPoint = offsetCoord(cursorPos, selectedEntity.state.position);
+        const normalizedPoint = offsetCoord(cursorPos.relative, selectedEntity.state.position);
         const alignedPoint = alignCoordForTool(normalizedPoint, settings.brushWidth);
         if (e.evt.pointerType === 'pen' && settings.pressureSensitivity) {
           await selectedEntity.bufferRenderer.setBuffer({
@@ -347,7 +354,7 @@ export class CanvasToolModule extends CanvasModuleBase {
       }
 
       if (tool === 'eraser') {
-        const normalizedPoint = offsetCoord(cursorPos, selectedEntity.state.position);
+        const normalizedPoint = offsetCoord(cursorPos.relative, selectedEntity.state.position);
         const alignedPoint = alignCoordForTool(normalizedPoint, settings.brushWidth);
         if (selectedEntity.bufferRenderer.state && selectedEntity.bufferRenderer.hasBuffer()) {
           selectedEntity.bufferRenderer.commitBuffer();
@@ -386,7 +393,8 @@ export class CanvasToolModule extends CanvasModuleBase {
 
       const isMouseDown = getIsPrimaryMouseDown(e);
       this.$isMouseDown.set(isMouseDown);
-      const cursorPos = this.syncLastCursorPos();
+
+      const cursorPos = this.$cursorPos.get();
       const tool = this.$tool.get();
       const settings = this.manager.stateApi.getSettings();
       const selectedEntity = this.manager.stateApi.getSelectedEntityAdapter();
@@ -395,7 +403,7 @@ export class CanvasToolModule extends CanvasModuleBase {
         return;
       }
 
-      const normalizedPoint = offsetCoord(cursorPos, selectedEntity.state.position);
+      const normalizedPoint = offsetCoord(cursorPos.relative, selectedEntity.state.position);
 
       if (tool === 'brush') {
         if (e.evt.pointerType === 'pen' && settings.pressureSensitivity) {
@@ -541,7 +549,7 @@ export class CanvasToolModule extends CanvasModuleBase {
       const settings = this.manager.stateApi.getSettings();
 
       if (tool === 'colorPicker') {
-        const color = getColorUnderCursor(this.konva.stage);
+        const color = this.$colorUnderCursor.get();
         if (color) {
           this.manager.stateApi.setColor({ ...settings.color, ...color });
         }
@@ -597,11 +605,17 @@ export class CanvasToolModule extends CanvasModuleBase {
         return;
       }
 
+      this.syncCursorPositions();
+      const cursorPos = this.$cursorPos.get();
+
+      if (!cursorPos) {
+        return;
+      }
+
       const tool = this.$tool.get();
-      const cursorPos = this.syncLastCursorPos();
 
       if (tool === 'colorPicker') {
-        const color = getColorUnderCursor(this.konva.stage);
+        const color = getColorAtCoordinate(this.konva.stage, cursorPos.absolute);
         if (color) {
           this.$colorUnderCursor.set(color);
         }
@@ -610,7 +624,8 @@ export class CanvasToolModule extends CanvasModuleBase {
 
       const isMouseDown = this.$isMouseDown.get();
       const selectedEntity = this.manager.stateApi.getSelectedEntityAdapter();
-      if (!cursorPos || !isMouseDown || !selectedEntity?.$isInteractable.get()) {
+
+      if (!isMouseDown || !selectedEntity?.$isInteractable.get()) {
         return;
       }
 
@@ -625,11 +640,11 @@ export class CanvasToolModule extends CanvasModuleBase {
       if (tool === 'brush' && (bufferState.type === 'brush_line' || bufferState.type === 'brush_line_with_pressure')) {
         const lastPoint = getLastPointOfLine(bufferState.points);
         const minDistance = settings.brushWidth * this.config.BRUSH_SPACING_TARGET_SCALE;
-        if (!lastPoint || !isDistanceMoreThanMin(cursorPos, lastPoint, minDistance)) {
+        if (!lastPoint || !isDistanceMoreThanMin(cursorPos.relative, lastPoint, minDistance)) {
           return;
         }
 
-        const normalizedPoint = offsetCoord(cursorPos, selectedEntity.state.position);
+        const normalizedPoint = offsetCoord(cursorPos.relative, selectedEntity.state.position);
         const alignedPoint = alignCoordForTool(normalizedPoint, settings.brushWidth);
 
         if (lastPoint.x === alignedPoint.x && lastPoint.y === alignedPoint.y) {
@@ -650,11 +665,11 @@ export class CanvasToolModule extends CanvasModuleBase {
       ) {
         const lastPoint = getLastPointOfLine(bufferState.points);
         const minDistance = settings.eraserWidth * this.config.BRUSH_SPACING_TARGET_SCALE;
-        if (!lastPoint || !isDistanceMoreThanMin(cursorPos, lastPoint, minDistance)) {
+        if (!lastPoint || !isDistanceMoreThanMin(cursorPos.relative, lastPoint, minDistance)) {
           return;
         }
 
-        const normalizedPoint = offsetCoord(cursorPos, selectedEntity.state.position);
+        const normalizedPoint = offsetCoord(cursorPos.relative, selectedEntity.state.position);
         const alignedPoint = alignCoordForTool(normalizedPoint, settings.eraserWidth);
 
         if (lastPoint.x === alignedPoint.x && lastPoint.y === alignedPoint.y) {
@@ -670,7 +685,7 @@ export class CanvasToolModule extends CanvasModuleBase {
 
         await selectedEntity.bufferRenderer.setBuffer(bufferState);
       } else if (tool === 'rect' && bufferState.type === 'rect') {
-        const normalizedPoint = offsetCoord(cursorPos, selectedEntity.state.position);
+        const normalizedPoint = offsetCoord(cursorPos.relative, selectedEntity.state.position);
         const alignedPoint = floorCoord(normalizedPoint);
         bufferState.rect.width = Math.round(alignedPoint.x - bufferState.rect.x);
         bufferState.rect.height = Math.round(alignedPoint.y - bufferState.rect.y);

invokeai/frontend/web/src/features/controlLayers/konva/util.ts

@@ -640,13 +640,9 @@ export const getPointerType = (e: KonvaEventObject<PointerEvent>): 'mouse' | 'pe
  * @param stage The konva stage
  * @returns The color under the cursor, or null if the cursor is not over the stage
  */
-export const getColorUnderCursor = (stage: Konva.Stage): RgbColor | null => {
-  const pos = stage.getPointerPosition();
-  if (!pos) {
-    return null;
-  }
+export const getColorAtCoordinate = (stage: Konva.Stage, coord: Coordinate): RgbColor | null => {
   const ctx = stage
-    .toCanvas({ x: pos.x, y: pos.y, width: 1, height: 1, imageSmoothingEnabled: false })
+    .toCanvas({ x: coord.x, y: coord.y, width: 1, height: 1, imageSmoothingEnabled: false })
     .getContext('2d');
 
   if (!ctx) {