feat(image): Implement template matching #99

* Initial template matching logic using openCV
* Test using pokemon cheat logo
* Test using an "exact" selection from a stock image https://unsplash.com/photos/J2RQT7kJSMM

Dockerfile

@@ -81,8 +81,9 @@ RUN apk add --no-cache \
 #
 
 # vips required to run sharp library for image comparison
+# opencv required for other image processing
 RUN echo "http://dl-4.alpinelinux.org/alpine/v3.14/community" >> /etc/apk/repositories \
-    && apk --no-cache add vips
+    && apk --no-cache add vips opencv opencv-dev
 
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
@@ -115,6 +116,15 @@ RUN npm install --production \
     && rm -rf node_modules/ts-node \
     && rm -rf node_modules/typescript
 
+# build bindings for opencv
+RUN apk add --no-cache --virtual .build-deps \
+    make \
+    g++ \
+    gcc \
+    libgcc \
+    && npm run cv-install-docker-prebuild \
+    && apk del .build-deps
+
 ENV NPM_CONFIG_LOGLEVEL debug
 
 # can set database to use more performant better-sqlite3 since we control everything

docs/operator/installation.md

@@ -60,6 +60,65 @@ An example of running CM using the [minimum configuration](/docs/operator/config
 node src/index.js run
 ```
 
+### Dependencies
+
+Note: All below dependencies are automatically included in the [Docker](#docker-recommended) image.
+
+#### Sharp
+
+For basic [Image Comparisons](/docs/imageComparison.md) and image data CM uses [sharp](https://sharp.pixelplumbing.com/) which depends on [libvips](https://www.libvips.org/)
+
+Binaries for Sharp and libvips ship with the `sharp` npm package for all major operating systems and should require no additional steps to use -- installing CM with the above script should be sufficient.
+
+See more about Sharp dependencies in the [image comparison prerequisites.](/docs/imageComparison.md#prerequisites)
+
+
+#### OpenCV
+
+For advanced image comparison CM uses [OpenCV.](https://opencv.org/) OpenCV is an **optional** dependency that is only utilized if CM is configured to run these advanced image operations so if you are NOT doing any image-related operations you can safely ignore this section/dependency.
+
+**NOTE:** Depending on the image being compared (resolution) and operations being performed this can be a **CPU heavy resource.** TODO: Add rules that are cpu heavy...
+
+##### Installation
+
+Installation is not an automatic process. The below instructions are a summary of "easy" paths for installation but are not exhaustive. DO reference the detailed instructions (including additional details for windows installs) at [opencv4nodejs How to Install](https://github.com/UrielCh/opencv4nodejs#how-to-install).
+
+###### Build From Source
+
+This may take **some time** since openCV will be built from scratch.
+
+On windows you must first install build tools: `npm install --global windows-build-tools`
+
+Otherwise, run one of the following commands from the CM project directory:
+
+* For CUDA (Nvidia GPU acceleration): `npm run cv-autoinstall-cuda`
+* Normal: `npm run cv-autoinstall`
+
+###### Build from Prebuilt
+
+In this use-case you already have openCV built OR are using a distro prebuilt package. This method is much faster than building from source as only bindings need to be built.
+
+[More information on prebuild installation](https://github.com/UrielCh/opencv4nodejs#installing-opencv-manually)
+
+Prerequisites:
+
+* Windows `choco install OpenCV -y -version 4.1.0`
+* MacOS `brew install opencv@4; brew link --force opencv@4`
+* Linux -- varies, check your package manager for `opencv` and `opencv-dev`
+
+A script for building on **Ubuntu** is already included in CM:
+
+* `sudo apt install opencv-dev`
+* `npm run cv-install-ubuntu-prebuild`
+
+Otherwise, you will need to modify `scripts` in CM's `package.json`, use the script `cv-install-ubuntu-prebuild` as an example. Your command must include:
+
+* `--incDir [path/to/opencv/dev-files]` (on linux, usually `/usr/include/opencv4/`)
+* `--libDir [path/to/opencv/shared-files]` (on linux usually `/lib/x86_64-linux-gnu/` or `/usr/lib/`)
+* `--binDir=[path/to/openv/binaries]` (on linux usually `/usr/bin/`)
+
+After you have modified/added a script for your operating system run it with `npm run yourScriptName`
+
 ## [Heroku Quick Deploy](https://heroku.com/about)
 
 **NOTE:** This is still experimental and requires more testing.

package.json

@@ -21,7 +21,11 @@
     "circular-graph": "madge --image graph.svg --circular --extensions ts src/index.ts",
     "postinstall": "patch-package",
     "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js",
-    "initMigration": "npm run typeorm -- migration:generate -t 1642180264563 -d ormconfig.js \"src/Common/Migrations/Database/init\""
+    "initMigration": "npm run typeorm -- migration:generate -t 1642180264563 -d ormconfig.js \"src/Common/Migrations/Database/init\"",
+    "cv-install-ubuntu-prebuild": "build-opencv --incDir /usr/include/opencv4/ --libDir /lib/x86_64-linux-gnu/ --binDir=/usr/bin/ --nobuild rebuild",
+    "cv-install-docker-prebuild": "build-opencv --incDir /usr/include/opencv4/ --libDir /usr/lib/ --binDir=/usr/bin/ --nobuild rebuild",
+    "cv-autoinstall-cuda": "build-opencv --version 4.5.5 --flags=\"-DWITH_CUDA=ON -DWITH_CUDNN=ON -DOPENCV_DNN_CUDA=ON -DCUDA_FAST_MATH=ON\" build",
+    "cv-autoinstall": "build-opencv build"
   },
   "engines": {
     "node": ">=16"
@@ -42,6 +46,7 @@
     "@nlpjs/language": "^4.22.7",
     "@nlpjs/nlp": "^4.23.5",
     "@stdlib/regexp-regexp": "^0.0.6",
+    "@u4/opencv4nodejs": "^6.2.1",
     "ajv": "^7.2.4",
     "ansi-regex": ">=5.0.1",
     "async": "^3.2.0",
@@ -160,6 +165,7 @@
     "better-sqlite3": "^7.5.0",
     "mongo": "^3.6.0",
     "mysql": "^2.18.1",
+    "opencv-build": "^0.1.9",
     "pg": "^8.7.1",
     "sharp": "^0.29.1"
   }

src/Common/OpenCVService.ts

@@ -0,0 +1,242 @@
+import winston, {Logger} from "winston";
+import {CMError} from "../Utils/Errors";
+import {formatNumber, mergeArr, resolvePath} from "../util";
+import * as cvTypes from '@u4/opencv4nodejs'
+import ImageData from "./ImageData";
+import {pathToFileURL} from "url";
+
+let cv: any;
+
+export const getCV = async (): Promise<typeof cvTypes.cv> => {
+    if (cv === undefined) {
+        try {
+            const cvImport = await import('@u4/opencv4nodejs');
+            if (cvImport === undefined) {
+                throw new CMError('Could not initialize openCV because opencv4nodejs is not installed');
+            }
+            cv = cvImport.default;
+        } catch (e: any) {
+            throw new CMError('Could not initialize openCV', {cause: e});
+        }
+    }
+    return cv as typeof cvTypes.cv;
+}
+
+export class OpenCVService {
+
+    logger: Logger;
+
+    constructor(logger?: Logger) {
+        const parentLogger = logger ?? winston.loggers.get('app');
+        this.logger = parentLogger.child({labels: ['OpenCV']}, mergeArr)
+    }
+
+    async cv() {
+        if (cv === undefined) {
+            try {
+                const cvImport = await import('@u4/opencv4nodejs');
+                if (cvImport === undefined) {
+                    throw new CMError('Could not initialize openCV because opencv4nodejs is not installed');
+                }
+                cv = cvImport.default;
+            } catch (e: any) {
+                throw new CMError('Could not initialize openCV', {cause: e});
+            }
+        }
+        return cv as typeof cvTypes.cv;
+    }
+}
+
+interface CurrentMaxData {
+    confidence: number,
+    loc: cvTypes.Point2,
+    ratio?: number
+}
+
+export interface MatchResult {matchRec?: cvTypes.Rect, matchedConfidence?: number}
+
+
+/**
+ * Use openCV matchTemplate() to find images within images
+ *
+ * The majority of these code concepts are based on https://pyimagesearch.com/2015/01/26/multi-scale-template-matching-using-python-opencv/
+ * and examples/usage of opencv.js is from https://github.com/UrielCh/opencv4nodejs/tree/master/examples/src/templateMatch
+ *
+ * */
+export class TemplateCompare {
+    cv: typeof cvTypes.cv;
+    logger: Logger;
+
+    template?: cvTypes.Mat;
+    downscaledTemplates: cvTypes.Mat[] = [];
+
+    constructor(cv: typeof cvTypes.cv, logger: Logger) {
+        this.cv = cv;
+        this.logger = logger.child({labels: ['OpenCV', 'Template Match']}, mergeArr)
+    }
+
+    protected async normalizeImage(image: ImageData) {
+        return this.cv.imdecode(await ((await image.sharp()).clone().greyscale().toBuffer()));
+    }
+
+    async setTemplate(image: ImageData) {
+        this.template = await this.normalizeImage(image);
+    }
+
+    protected getTemplate() {
+        if (this.template === undefined) {
+            throw new Error('Template is not defined, use setTemplate() first');
+        }
+        return this.template.copy().canny(50, 200);
+    }
+
+    downscaleTemplates() {
+        if (this.template === undefined) {
+            throw new Error('Template is not defined, use setTemplate() first');
+        }
+
+        const [tH, tW] = this.template.sizes;
+
+        for (let i = 10; i <= 80; i += 10) {
+            const templateRatio = (100 - i) / 100;
+
+            // for debugging
+            // const scaled = this.template.copy().resize(new cv.Size(Math.floor(templateRatio * tW), Math.floor(templateRatio * tH))).canny(50, 200);
+            // const path = pathToFileURL(resolvePath(`./tests/assets/star/starTemplateScaled-${Math.floor(templateRatio * 100)}.jpg`, './')).pathname;
+            // cv.imwrite(path, scaled);
+            this.downscaledTemplates.push(this.template.copy().resize(new cv.Size(Math.floor(templateRatio * tW), Math.floor(templateRatio * tH))).canny(50, 200))
+        }
+    }
+
+    async matchImage(sourceImageData: ImageData, downscaleWhich: 'template' | 'image', confidence = 0.5): Promise<[boolean, MatchResult]> {
+        if (this.template === undefined) {
+            throw new Error('Template is not defined, use setTemplate() first');
+        }
+
+        let currMax: CurrentMaxData | undefined;
+
+        let matchRec: cvTypes.Rect | undefined;
+        let matchedConfidence: number | undefined;
+
+        if (downscaleWhich === 'template') {
+            // in this scenario we assume our template is a significant fraction of the size of the source
+            // so we want to scale down the template size incrementally
+            // because we are assuming the template in the image is smaller than our source template
+
+            // generate scaled templates and save for later use!
+            // its likely this class is in use in Recent/Repeat rules which means we will probably be comparing this template against many images
+            if (this.downscaledTemplates.length === 0) {
+                this.downscaleTemplates();
+            }
+
+            let currMaxTemplateSize: number[] | undefined;
+
+            const src = (await this.normalizeImage(sourceImageData)).canny(50, 200);
+
+            const edgedTemplate = await this.getTemplate();
+
+            for (const scaledTemplate of [edgedTemplate].concat(this.downscaledTemplates)) {
+
+                // more information on methods...
+                // https://docs.opencv.org/4.x/d4/dc6/tutorial_py_template_matching.html
+                // https://stackoverflow.com/questions/58158129/understanding-and-evaluating-template-matching-methods
+                // https://stackoverflow.com/questions/48799711/explain-difference-between-opencvs-template-matching-methods-in-non-mathematica
+                // https://datahacker.rs/014-template-matching-using-opencv-in-python/
+                // ...may want to try with TM_SQDIFF but will need to use minimum values instead of max
+                const result = src.matchTemplate(scaledTemplate, cv.TM_CCOEFF_NORMED);
+
+                const minMax = result.minMaxLoc();
+                const {maxVal, maxLoc} = minMax;
+
+                if (currMax === undefined || maxVal > currMax.confidence) {
+                    currMaxTemplateSize = scaledTemplate.sizes;
+                    currMax = {confidence: maxVal, loc: maxLoc};
+                    console.log(`New Best Max Confidence: ${formatNumber(maxVal, {toFixed: 4})}`)
+                }
+                if (maxVal >= confidence) {
+                    this.logger.verbose(`Match with confidence ${formatNumber(maxVal, {toFixed: 4})} met threshold of ${confidence}`);
+                    break;
+                }
+            }
+
+            if (currMax !== undefined) {
+                matchedConfidence = currMax.confidence;
+
+                if (currMaxTemplateSize !== undefined) {
+                    const startX = currMax.loc.x;
+                    const startY = currMax.loc.y;
+
+                    matchRec = new cv.Rect(startX, startY, currMaxTemplateSize[1], currMaxTemplateSize[0]);
+                }
+            }
+
+
+        } else {
+            // in this scenario we assume our template is small, compared to the source image
+            // and the template found in the source is likely larger than the template
+            // so we scale down the source incrementally to try to get them to match
+
+            const normalSrc = (await this.normalizeImage(sourceImageData));
+            let src = normalSrc.copy();
+            const [width, height] = src.sizes;
+
+            const edgedTemplate = await this.getTemplate();
+            const [tH, tW] = edgedTemplate.sizes;
+
+            let ratio = 1;
+
+            for (let i = 0; i <= 80; i += 5) {
+                ratio = (100 - i) / 100;
+
+                if (i !== 100) {
+                    const resizedWidth = Math.floor(width * ratio);
+                    const resizedHeight = Math.floor(height * ratio);
+                    src = src.resize(new cv.Size(resizedWidth, resizedHeight));
+                }
+
+                const [sH, sW] = src.sizes;
+                if (sH < tH || sW < tW) {
+                    // scaled source is smaller than template
+                    this.logger.debug(`Template matching ended early due to downscaled image being smaller than template`);
+                    break;
+                }
+
+                const edged = src.canny(50, 200);
+                const result = edged.matchTemplate(edgedTemplate, cv.TM_CCOEFF_NORMED);
+
+                const minMax = result.minMaxLoc();
+                const {maxVal, maxLoc} = minMax;
+
+                if (currMax === undefined || maxVal > currMax.confidence) {
+                    currMax = {confidence: maxVal, loc: maxLoc, ratio};
+                    console.log(`New Best Confidence: ${formatNumber(maxVal, {toFixed: 4})}`)
+                }
+                if (maxVal >= confidence) {
+                    this.logger.verbose(`Match with confidence ${formatNumber(maxVal, {toFixed: 4})} met threshold of ${confidence}`);
+                    break;
+                }
+            }
+
+            if (currMax === undefined) {
+                // template was larger than source
+                this.logger.debug('No local max found');
+            } else {
+                const maxRatio = currMax.ratio as number;
+
+                const startX = currMax.loc.x * (1 / maxRatio);
+                const startY = currMax.loc.y * (1 / maxRatio);
+
+                const endWidth = tW * (1 / maxRatio);
+                const endHeight = tH * (1 / maxRatio);
+
+                matchRec = new cv.Rect(startX, startY, endWidth, endHeight);
+                matchedConfidence = currMax.confidence;
+            }
+        }
+
+        if (currMax !== undefined) {
+            return [currMax.confidence >= confidence, {matchRec, matchedConfidence}]
+        }
+        return [false, {matchRec, matchedConfidence}]
+    }
+}

tests/opencv.test.ts

@@ -0,0 +1,75 @@
+import {describe, it} from 'mocha';
+import chai from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import express, {Request, Response} from "express";
+import {formatNumber, resolvePath, sleep} from "../src/util";
+import {pathToFileURL, URL} from "url";
+import ImageData from "../src/Common/ImageData";
+import * as cvTypes from '@u4/opencv4nodejs'
+import {getCV, TemplateCompare} from "../src/Common/OpenCVService";
+import winston from 'winston';
+
+chai.use(chaiAsPromised);
+
+const assert = chai.assert;
+
+const star = pathToFileURL(resolvePath('./tests/assets/star-transparent.png', './'));
+const starInside = pathToFileURL(resolvePath('./tests/assets/star-inside.png', './'));
+const tran = pathToFileURL(resolvePath('./tests/assets/tran.jpg', './'));
+const tranSel = pathToFileURL(resolvePath('./tests/assets/tran-selection.jpg', './'));
+
+describe('Template Matching', function () {
+
+    let cv: typeof cvTypes.cv;
+
+    before(async () => {
+        cv = await getCV();
+    });
+
+    it('matches a standard example', async function () {
+
+        const templateMatch = new TemplateCompare(cv, winston.loggers.get('app'));
+
+        await templateMatch.setTemplate(new ImageData({path: tranSel}));
+
+        const [passed, results] = await templateMatch.matchImage(new ImageData({
+            path: tran
+        }), 'template');
+
+        if(results.matchRec !== undefined) {
+            const src = cv.imread(tran.pathname);
+            src.drawRectangle(
+                results.matchRec,
+                new cv.Vec3(0, 255, 0),
+                2,
+                cv.LINE_8
+            );
+            // TODO mask is not drawn correctly (its above?)
+            cv.imwrite(pathToFileURL(resolvePath(`./tests/assets/tran-masked.jpg`, './')).pathname, src);
+        }
+
+        assert.isTrue(passed);
+    });
+
+    it('matches a template using service', async function () {
+
+        const templateMatch = new TemplateCompare(cv, winston.loggers.get('app'));
+
+        await templateMatch.setTemplate(new ImageData({path: star}));
+
+        const [passed, results] = await templateMatch.matchImage(new ImageData({
+            path: starInside
+        }), 'template', 0.2);
+
+        if(results.matchRec !== undefined) {
+            const src = cv.imread(starInside.pathname);
+            src.drawRectangle(
+                results.matchRec,
+                new cv.Vec3(0, 255, 0),
+                2,
+                cv.LINE_8
+            );
+            cv.imwrite(pathToFileURL(resolvePath(`./tests/assets/star-masked.jpg`, './')).pathname, src);
+        }
+    });
+});