refactor: Introduce staggered startup for bots/polling to decrease load on host/reddit and improve image comparison performance

* Implement staggered startup for bots (reddit accounts, top-level) * Implement staggered startup for managers (subreddits) and subreddit polling * Introduce random -1/+1 second to polling interval for every stream to ensure none are synced so there is no instantaneous spike in cpu/traffic/memory on host/reddit * Add user-configurable stagger interval for shared mod polling * Implement second image comparison approach with pixelmatch for reduced memory usage when image dimensions are exactly the same * Use sharp to resize images to 400 width max when using resemblejs to reduce memory usage
2026-04-19 03:00:07 -04:00 · 2021-10-07 17:13:27 -04:00
parent f8fc63991f
commit 8cf30b6b7d
12 changed files with 833 additions and 55 deletions
--- a/package-lock.json
+++ b/package-lock.json
@@ -5,6 +5,7 @@
  "requires": true,
  "packages": {
    "": {
+      "name": "redditcontextbot",
      "version": "0.5.1",
      "license": "ISC",
      "dependencies": {
@@ -45,12 +46,16 @@
        "normalize-url": "^6.1.0",
        "object-hash": "^2.2.0",
        "p-event": "^4.2.0",
+        "p-map": "^4.0.0",
        "passport": "^0.4.1",
        "passport-custom": "^1.1.1",
        "passport-jwt": "^4.0.0",
+        "pixelmatch": "^5.2.1",
        "pretty-print-json": "^1.0.3",
        "resemblejs": "^4.0.0",
        "safe-stable-stringify": "^1.1.1",
+        "set-random-interval": "^1.1.0",
+        "sharp": "^0.29.1",
        "snoostorm": "^1.5.2",
        "snoowrap": "^1.23.0",
        "socket.io": "^4.1.3",
@@ -88,6 +93,7 @@
        "@types/passport-jwt": "^3.0.6",
        "@types/pixelmatch": "^5.2.4",
        "@types/resemblejs": "^3.2.1",
+        "@types/sharp": "^0.29.2",
        "@types/tcp-port-used": "^1.0.0",
        "@types/triple-beam": "^1.3.2",
        "ts-auto-guard": "*",
@@ -98,7 +104,8 @@
        "node": ">=15"
      },
      "optionalDependencies": {
-        "node-canvas": "^2.7.0"
+        "node-canvas": "^2.7.0",
+        "sharp": "^0.29.1"
      }
    },
    "node_modules/@awaitjs/express": {
@@ -696,6 +703,15 @@
        "@types/node": "*"
      }
    },
+    "node_modules/@types/sharp": {
+      "version": "0.29.2",
+      "resolved": "https://registry.npmjs.org/@types/sharp/-/sharp-0.29.2.tgz",
+      "integrity": "sha512-tIbMvtPa8kMyFMKNhpsPT1HO3CgXLuiCAA8bxHAGAZLyALpYvYc4hUu3pu0+3oExQA5LwvHrWp+OilgXCYVQgg==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
    "node_modules/@types/socket.io": {
      "version": "2.1.13",
      "resolved": "https://registry.npmjs.org/@types/socket.io/-/socket.io-2.1.13.tgz",
@@ -794,6 +810,18 @@
      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
      "optional": true
    },
+    "node_modules/aggregate-error": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
+      "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
+      "dependencies": {
+        "clean-stack": "^2.0.0",
+        "indent-string": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
    "node_modules/ajv": {
      "version": "7.2.4",
      "resolved": "https://registry.npmjs.org/ajv/-/ajv-7.2.4.tgz",
@@ -1021,6 +1049,23 @@
        "tweetnacl": "^0.14.3"
      }
    },
+    "node_modules/bl": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+      "optional": true,
+      "dependencies": {
+        "buffer": "^5.5.0",
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.4.0"
+      }
+    },
+    "node_modules/bl/node_modules/inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+      "optional": true
+    },
    "node_modules/bluebird": {
      "version": "3.7.2",
      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
@@ -1197,6 +1242,14 @@
        "node": ">=10"
      }
    },
+    "node_modules/clean-stack": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
+      "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
    "node_modules/cliui": {
      "version": "7.0.4",
      "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
@@ -1511,7 +1564,7 @@
      "version": "0.6.0",
      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
-      "dev": true,
+      "devOptional": true,
      "engines": {
        "node": ">=4.0.0"
      }
@@ -1804,6 +1857,15 @@
      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
      "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="
    },
+    "node_modules/expand-template": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
+      "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
+      "optional": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
    "node_modules/express": {
      "version": "4.17.1",
      "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
@@ -2106,6 +2168,12 @@
        "node": ">= 0.6"
      }
    },
+    "node_modules/fs-constants": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+      "optional": true
+    },
    "node_modules/fs-minipass": {
      "version": "2.1.0",
      "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
@@ -2226,6 +2294,12 @@
        "assert-plus": "^1.0.0"
      }
    },
+    "node_modules/github-from-package": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
+      "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=",
+      "optional": true
+    },
    "node_modules/glob": {
      "version": "7.1.7",
      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz",
@@ -2489,6 +2563,14 @@
        "node": ">=12.0.0"
      }
    },
+    "node_modules/indent-string": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
+      "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
    "node_modules/inflight": {
      "version": "1.0.6",
      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@@ -2504,6 +2586,12 @@
      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
    },
+    "node_modules/ini": {
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
+      "optional": true
+    },
    "node_modules/ip-regex": {
      "version": "4.3.0",
      "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-4.3.0.tgz",
@@ -3078,6 +3166,12 @@
        "node": ">=10"
      }
    },
+    "node_modules/mkdirp-classic": {
+      "version": "0.5.3",
+      "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
+      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
+      "optional": true
+    },
    "node_modules/moment": {
      "version": "2.29.1",
      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz",
@@ -3129,6 +3223,12 @@
      "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==",
      "optional": true
    },
+    "node_modules/napi-build-utils": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
+      "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==",
+      "optional": true
+    },
    "node_modules/negotiator": {
      "version": "0.6.2",
      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
@@ -3137,6 +3237,21 @@
        "node": ">= 0.6"
      }
    },
+    "node_modules/node-abi": {
+      "version": "2.30.1",
+      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.1.tgz",
+      "integrity": "sha512-/2D0wOQPgaUWzVSVgRMx+trKJRC2UG4SUc4oCJoXx9Uxjtp0Vy3/kt7zcbxHF8+Z/pK3UloLWzBISg72brfy1w==",
+      "optional": true,
+      "dependencies": {
+        "semver": "^5.4.1"
+      }
+    },
+    "node_modules/node-addon-api": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.2.0.tgz",
+      "integrity": "sha512-eazsqzwG2lskuzBqCGPi7Ac2UgOoMz8JVOXVhTvvPDYhthvNpefx8jWD8Np7Gv+2Sz0FlPWZk0nJV0z598Wn8Q==",
+      "optional": true
+    },
    "node_modules/node-canvas": {
      "version": "2.7.0",
      "resolved": "https://registry.npmjs.org/node-canvas/-/node-canvas-2.7.0.tgz",
@@ -3299,6 +3414,20 @@
        "node": ">=4"
      }
    },
+    "node_modules/p-map": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
+      "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
+      "dependencies": {
+        "aggregate-error": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
    "node_modules/p-timeout": {
      "version": "3.2.0",
      "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
@@ -3402,6 +3531,52 @@
        "url": "https://github.com/sponsors/jonschlinkert"
      }
    },
+    "node_modules/pixelmatch": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.2.1.tgz",
+      "integrity": "sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ==",
+      "dependencies": {
+        "pngjs": "^4.0.1"
+      },
+      "bin": {
+        "pixelmatch": "bin/pixelmatch"
+      }
+    },
+    "node_modules/pngjs": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-4.0.1.tgz",
+      "integrity": "sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg==",
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
+    "node_modules/prebuild-install": {
+      "version": "6.1.4",
+      "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-6.1.4.tgz",
+      "integrity": "sha512-Z4vpywnK1lBg+zdPCVCsKq0xO66eEV9rWo2zrROGGiRS4JtueBOdlB1FnY8lcy7JsUud/Q3ijUxyWN26Ika0vQ==",
+      "optional": true,
+      "dependencies": {
+        "detect-libc": "^1.0.3",
+        "expand-template": "^2.0.3",
+        "github-from-package": "0.0.0",
+        "minimist": "^1.2.3",
+        "mkdirp-classic": "^0.5.3",
+        "napi-build-utils": "^1.0.1",
+        "node-abi": "^2.21.0",
+        "npmlog": "^4.0.1",
+        "pump": "^3.0.0",
+        "rc": "^1.2.7",
+        "simple-get": "^3.0.3",
+        "tar-fs": "^2.0.0",
+        "tunnel-agent": "^0.6.0"
+      },
+      "bin": {
+        "prebuild-install": "bin.js"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
    "node_modules/pretty-print-json": {
      "version": "1.0.3",
      "resolved": "https://registry.npmjs.org/pretty-print-json/-/pretty-print-json-1.0.3.tgz",
@@ -3531,6 +3706,21 @@
        "node": ">= 0.8"
      }
    },
+    "node_modules/rc": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
+      "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+      "optional": true,
+      "dependencies": {
+        "deep-extend": "^0.6.0",
+        "ini": "~1.3.0",
+        "minimist": "^1.2.0",
+        "strip-json-comments": "~2.0.1"
+      },
+      "bin": {
+        "rc": "cli.js"
+      }
+    },
    "node_modules/readable-stream": {
      "version": "3.6.0",
      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -3841,11 +4031,97 @@
      "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=",
      "optional": true
    },
+    "node_modules/set-random-interval": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/set-random-interval/-/set-random-interval-1.1.0.tgz",
+      "integrity": "sha512-WHBpz4W+TmPQlBRiGxNKH2pczEV8LV9yNlA4dX237oCehJtimEKaq7TZK5vxfyu+YVYPZ9Lj/0ioOWriWSJ4/Q==",
+      "dependencies": {
+        "typescript": "^3.6.3"
+      }
+    },
+    "node_modules/set-random-interval/node_modules/typescript": {
+      "version": "3.9.10",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz",
+      "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==",
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=4.2.0"
+      }
+    },
    "node_modules/setprototypeof": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
      "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw=="
    },
+    "node_modules/sharp": {
+      "version": "0.29.1",
+      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.29.1.tgz",
+      "integrity": "sha512-DpgdAny9TuS+oWCQ7MRS8XyY9x6q1+yW3a5wNx0J3HrGuB/Jot/8WcT+lElHY9iJu2pwtegSGxqMaqFiMhs4rQ==",
+      "hasInstallScript": true,
+      "optional": true,
+      "dependencies": {
+        "color": "^4.0.1",
+        "detect-libc": "^1.0.3",
+        "node-addon-api": "^4.1.0",
+        "prebuild-install": "^6.1.4",
+        "semver": "^7.3.5",
+        "simple-get": "^3.1.0",
+        "tar-fs": "^2.1.1",
+        "tunnel-agent": "^0.6.0"
+      },
+      "engines": {
+        "node": ">=12.13.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/sharp/node_modules/color": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/color/-/color-4.0.1.tgz",
+      "integrity": "sha512-rpZjOKN5O7naJxkH2Rx1sZzzBgaiWECc6BYXjeCE6kF0kcASJYbUq02u7JqIHwCb/j3NhV+QhRL2683aICeGZA==",
+      "optional": true,
+      "dependencies": {
+        "color-convert": "^2.0.1",
+        "color-string": "^1.6.0"
+      }
+    },
+    "node_modules/sharp/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "optional": true,
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/sharp/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "optional": true
+    },
+    "node_modules/sharp/node_modules/semver": {
+      "version": "7.3.5",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
+      "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
+      "optional": true,
+      "dependencies": {
+        "lru-cache": "^6.0.0"
+      },
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
    "node_modules/shebang-command": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@@ -4171,7 +4447,7 @@
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
      "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=",
-      "dev": true,
+      "devOptional": true,
      "engines": {
        "node": ">=0.10.0"
      }
@@ -4237,6 +4513,40 @@
        "node": ">= 10"
      }
    },
+    "node_modules/tar-fs": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
+      "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
+      "optional": true,
+      "dependencies": {
+        "chownr": "^1.1.1",
+        "mkdirp-classic": "^0.5.2",
+        "pump": "^3.0.0",
+        "tar-stream": "^2.1.4"
+      }
+    },
+    "node_modules/tar-fs/node_modules/chownr": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
+      "optional": true
+    },
+    "node_modules/tar-stream": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
+      "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
+      "optional": true,
+      "dependencies": {
+        "bl": "^4.0.3",
+        "end-of-stream": "^1.4.1",
+        "fs-constants": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^3.1.1"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
    "node_modules/tcp-port-used": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/tcp-port-used/-/tcp-port-used-1.0.2.tgz",
@@ -5386,6 +5696,15 @@
        "@types/node": "*"
      }
    },
+    "@types/sharp": {
+      "version": "0.29.2",
+      "resolved": "https://registry.npmjs.org/@types/sharp/-/sharp-0.29.2.tgz",
+      "integrity": "sha512-tIbMvtPa8kMyFMKNhpsPT1HO3CgXLuiCAA8bxHAGAZLyALpYvYc4hUu3pu0+3oExQA5LwvHrWp+OilgXCYVQgg==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
    "@types/socket.io": {
      "version": "2.1.13",
      "resolved": "https://registry.npmjs.org/@types/socket.io/-/socket.io-2.1.13.tgz",
@@ -5471,6 +5790,15 @@
        }
      }
    },
+    "aggregate-error": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
+      "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
+      "requires": {
+        "clean-stack": "^2.0.0",
+        "indent-string": "^4.0.0"
+      }
+    },
    "ajv": {
      "version": "7.2.4",
      "resolved": "https://registry.npmjs.org/ajv/-/ajv-7.2.4.tgz",
@@ -5652,6 +5980,25 @@
        "tweetnacl": "^0.14.3"
      }
    },
+    "bl": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+      "optional": true,
+      "requires": {
+        "buffer": "^5.5.0",
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.4.0"
+      },
+      "dependencies": {
+        "inherits": {
+          "version": "2.0.4",
+          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+          "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+          "optional": true
+        }
+      }
+    },
    "bluebird": {
      "version": "3.7.2",
      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
@@ -5786,6 +6133,11 @@
      "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==",
      "optional": true
    },
+    "clean-stack": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
+      "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A=="
+    },
    "cliui": {
      "version": "7.0.4",
      "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
@@ -6046,7 +6398,7 @@
      "version": "0.6.0",
      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
-      "dev": true
+      "devOptional": true
    },
    "deep-is": {
      "version": "0.1.3",
@@ -6255,6 +6607,12 @@
      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
      "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="
    },
+    "expand-template": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
+      "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
+      "optional": true
+    },
    "express": {
      "version": "4.17.1",
      "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
@@ -6493,6 +6851,12 @@
      "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
      "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
    },
+    "fs-constants": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+      "optional": true
+    },
    "fs-minipass": {
      "version": "2.1.0",
      "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
@@ -6588,6 +6952,12 @@
        "assert-plus": "^1.0.0"
      }
    },
+    "github-from-package": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
+      "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=",
+      "optional": true
+    },
    "glob": {
      "version": "7.1.7",
      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz",
@@ -6776,6 +7146,11 @@
        "queue": "6.0.2"
      }
    },
+    "indent-string": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
+      "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg=="
+    },
    "inflight": {
      "version": "1.0.6",
      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@@ -6791,6 +7166,12 @@
      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
    },
+    "ini": {
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
+      "optional": true
+    },
    "ip-regex": {
      "version": "4.3.0",
      "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-4.3.0.tgz",
@@ -7252,6 +7633,12 @@
      "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
      "devOptional": true
    },
+    "mkdirp-classic": {
+      "version": "0.5.3",
+      "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
+      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
+      "optional": true
+    },
    "moment": {
      "version": "2.29.1",
      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz",
@@ -7291,11 +7678,32 @@
      "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==",
      "optional": true
    },
+    "napi-build-utils": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
+      "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==",
+      "optional": true
+    },
    "negotiator": {
      "version": "0.6.2",
      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
      "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
    },
+    "node-abi": {
+      "version": "2.30.1",
+      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.1.tgz",
+      "integrity": "sha512-/2D0wOQPgaUWzVSVgRMx+trKJRC2UG4SUc4oCJoXx9Uxjtp0Vy3/kt7zcbxHF8+Z/pK3UloLWzBISg72brfy1w==",
+      "optional": true,
+      "requires": {
+        "semver": "^5.4.1"
+      }
+    },
+    "node-addon-api": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.2.0.tgz",
+      "integrity": "sha512-eazsqzwG2lskuzBqCGPi7Ac2UgOoMz8JVOXVhTvvPDYhthvNpefx8jWD8Np7Gv+2Sz0FlPWZk0nJV0z598Wn8Q==",
+      "optional": true
+    },
    "node-canvas": {
      "version": "2.7.0",
      "resolved": "https://registry.npmjs.org/node-canvas/-/node-canvas-2.7.0.tgz",
@@ -7409,6 +7817,14 @@
      "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
      "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4="
    },
+    "p-map": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
+      "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
+      "requires": {
+        "aggregate-error": "^3.0.0"
+      }
+    },
    "p-timeout": {
      "version": "3.2.0",
      "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
@@ -7485,6 +7901,40 @@
      "integrity": "sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw==",
      "dev": true
    },
+    "pixelmatch": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.2.1.tgz",
+      "integrity": "sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ==",
+      "requires": {
+        "pngjs": "^4.0.1"
+      }
+    },
+    "pngjs": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-4.0.1.tgz",
+      "integrity": "sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg=="
+    },
+    "prebuild-install": {
+      "version": "6.1.4",
+      "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-6.1.4.tgz",
+      "integrity": "sha512-Z4vpywnK1lBg+zdPCVCsKq0xO66eEV9rWo2zrROGGiRS4JtueBOdlB1FnY8lcy7JsUud/Q3ijUxyWN26Ika0vQ==",
+      "optional": true,
+      "requires": {
+        "detect-libc": "^1.0.3",
+        "expand-template": "^2.0.3",
+        "github-from-package": "0.0.0",
+        "minimist": "^1.2.3",
+        "mkdirp-classic": "^0.5.3",
+        "napi-build-utils": "^1.0.1",
+        "node-abi": "^2.21.0",
+        "npmlog": "^4.0.1",
+        "pump": "^3.0.0",
+        "rc": "^1.2.7",
+        "simple-get": "^3.0.3",
+        "tar-fs": "^2.0.0",
+        "tunnel-agent": "^0.6.0"
+      }
+    },
    "pretty-print-json": {
      "version": "1.0.3",
      "resolved": "https://registry.npmjs.org/pretty-print-json/-/pretty-print-json-1.0.3.tgz",
@@ -7576,6 +8026,18 @@
        "unpipe": "1.0.0"
      }
    },
+    "rc": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
+      "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
+      "optional": true,
+      "requires": {
+        "deep-extend": "^0.6.0",
+        "ini": "~1.3.0",
+        "minimist": "^1.2.0",
+        "strip-json-comments": "~2.0.1"
+      }
+    },
    "readable-stream": {
      "version": "3.6.0",
      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -7809,11 +8271,78 @@
      "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=",
      "optional": true
    },
+    "set-random-interval": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/set-random-interval/-/set-random-interval-1.1.0.tgz",
+      "integrity": "sha512-WHBpz4W+TmPQlBRiGxNKH2pczEV8LV9yNlA4dX237oCehJtimEKaq7TZK5vxfyu+YVYPZ9Lj/0ioOWriWSJ4/Q==",
+      "requires": {
+        "typescript": "^3.6.3"
+      },
+      "dependencies": {
+        "typescript": {
+          "version": "3.9.10",
+          "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz",
+          "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q=="
+        }
+      }
+    },
    "setprototypeof": {
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
      "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw=="
    },
+    "sharp": {
+      "version": "0.29.1",
+      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.29.1.tgz",
+      "integrity": "sha512-DpgdAny9TuS+oWCQ7MRS8XyY9x6q1+yW3a5wNx0J3HrGuB/Jot/8WcT+lElHY9iJu2pwtegSGxqMaqFiMhs4rQ==",
+      "optional": true,
+      "requires": {
+        "color": "^4.0.1",
+        "detect-libc": "^1.0.3",
+        "node-addon-api": "^4.1.0",
+        "prebuild-install": "^6.1.4",
+        "semver": "^7.3.5",
+        "simple-get": "^3.1.0",
+        "tar-fs": "^2.1.1",
+        "tunnel-agent": "^0.6.0"
+      },
+      "dependencies": {
+        "color": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/color/-/color-4.0.1.tgz",
+          "integrity": "sha512-rpZjOKN5O7naJxkH2Rx1sZzzBgaiWECc6BYXjeCE6kF0kcASJYbUq02u7JqIHwCb/j3NhV+QhRL2683aICeGZA==",
+          "optional": true,
+          "requires": {
+            "color-convert": "^2.0.1",
+            "color-string": "^1.6.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "optional": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "optional": true
+        },
+        "semver": {
+          "version": "7.3.5",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
+          "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
+          "optional": true,
+          "requires": {
+            "lru-cache": "^6.0.0"
+          }
+        }
+      }
+    },
    "shebang-command": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@@ -8054,7 +8583,7 @@
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
      "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=",
-      "dev": true
+      "devOptional": true
    },
    "supports-color": {
      "version": "5.5.0",
@@ -8104,6 +8633,39 @@
        "yallist": "^4.0.0"
      }
    },
+    "tar-fs": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
+      "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
+      "optional": true,
+      "requires": {
+        "chownr": "^1.1.1",
+        "mkdirp-classic": "^0.5.2",
+        "pump": "^3.0.0",
+        "tar-stream": "^2.1.4"
+      },
+      "dependencies": {
+        "chownr": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+          "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
+          "optional": true
+        }
+      }
+    },
+    "tar-stream": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
+      "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
+      "optional": true,
+      "requires": {
+        "bl": "^4.0.3",
+        "end-of-stream": "^1.4.1",
+        "fs-constants": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^3.1.1"
+      }
+    },
    "tcp-port-used": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/tcp-port-used/-/tcp-port-used-1.0.2.tgz",
--- a/package.json
+++ b/package.json
@@ -62,12 +62,15 @@
    "normalize-url": "^6.1.0",
    "object-hash": "^2.2.0",
    "p-event": "^4.2.0",
+    "p-map": "^4.0.0",
    "passport": "^0.4.1",
    "passport-custom": "^1.1.1",
    "passport-jwt": "^4.0.0",
+    "pixelmatch": "^5.2.1",
    "pretty-print-json": "^1.0.3",
    "resemblejs": "^4.0.0",
    "safe-stable-stringify": "^1.1.1",
+    "set-random-interval": "^1.1.0",
    "snoostorm": "^1.5.2",
    "snoowrap": "^1.23.0",
    "socket.io": "^4.1.3",
@@ -105,6 +108,7 @@
    "@types/passport-jwt": "^3.0.6",
    "@types/pixelmatch": "^5.2.4",
    "@types/resemblejs": "^3.2.1",
+    "@types/sharp": "^0.29.2",
    "@types/tcp-port-used": "^1.0.0",
    "@types/triple-beam": "^1.3.2",
    "ts-auto-guard": "*",
@@ -112,6 +116,7 @@
    "typescript-json-schema": "^0.50.1"
  },
  "optionalDependencies": {
-    "node-canvas": "^2.7.0"
+    "node-canvas": "^2.7.0",
+    "sharp": "^0.29.1"
  }
 }
--- a/src/App.ts
+++ b/src/App.ts
@@ -4,6 +4,7 @@ import {getLogger} from "./Utils/loggerFactory";
 import {Invokee, OperatorConfig} from "./Common/interfaces";
 import Bot from "./Bot";
 import LoggedError from "./Utils/LoggedError";
+import {sleep} from "./util";

 export class App {

@@ -66,6 +67,7 @@ export class App {
                try {
                    await b.testClient();
                    await b.buildManagers();
+                    await sleep(2000);
                    b.runManagers(causedBy).catch((err) => {
                        this.logger.error(`Unexpected error occurred while running Bot ${b.botName}. Bot must be re-built to restart`);
                        if (!err.logged || !(err instanceof LoggedError)) {
--- a/src/Bot/index.ts
+++ b/src/Bot/index.ts
@@ -1,4 +1,4 @@
-import Snoowrap, {Subreddit} from "snoowrap";
+import Snoowrap, {Comment, Submission, Subreddit} from "snoowrap";
 import {Logger} from "winston";
 import dayjs, {Dayjs} from "dayjs";
 import {Duration} from "dayjs/plugin/duration";
@@ -49,6 +49,7 @@ class Bot {
    maxWorkers: number;
    startedAt: Dayjs = dayjs();
    sharedModqueue: boolean = false;
+    streamListedOnce: string[] = [];

    apiSample: number[] = [];
    apiRollingAvg: number = 0;
@@ -91,6 +92,7 @@ class Bot {
            },
            polling: {
                sharedMod,
+                stagger,
            },
            queue: {
                maxWorkers,
@@ -199,12 +201,32 @@ class Bot {
            }
        }

+        const modStreamListingListener = (name: string) => async (listing: (Comment|Submission)[]) => {
+            // dole out in order they were received
+            if(!this.streamListedOnce.includes(name)) {
+                this.streamListedOnce.push(name);
+                return;
+            }
+            for(const i of listing) {
+                const foundManager = this.subManagers.find(x => x.subreddit.display_name === i.subreddit.display_name && x.modStreamCallbacks.get(name) !== undefined);
+                if(foundManager !== undefined) {
+                    foundManager.modStreamCallbacks.get(name)(i);
+                    if(stagger !== undefined) {
+                        await sleep(stagger);
+                    }
+                }
+            }
+
+        }
+
        const defaultUnmoderatedStream = new UnmoderatedStream(this.client, {subreddit: 'mod', limit: 100, clearProcessed: { size: 100, retain: 100 }});
        // @ts-ignore
        defaultUnmoderatedStream.on('error', modStreamErrorListener('unmoderated'));
+        defaultUnmoderatedStream.on('listing', modStreamListingListener('unmoderated'));
        const defaultModqueueStream = new ModQueueStream(this.client, {subreddit: 'mod', limit: 100, clearProcessed: { size: 100, retain: 100 }});
        // @ts-ignore
        defaultModqueueStream.on('error', modStreamErrorListener('modqueue'));
+        defaultModqueueStream.on('listing', modStreamListingListener('modqueue'));
        this.cacheManager.modStreams.set('unmoderated', defaultUnmoderatedStream);
        this.cacheManager.modStreams.set('modqueue', defaultModqueueStream);

@@ -337,7 +359,7 @@ class Bot {

    async runModStreams(notify = false) {
        for(const [k,v] of this.cacheManager.modStreams) {
-            if(!v.running && v.listeners('item').length > 0) {
+            if(!v.running && this.subManagers.some(x => x.modStreamCallbacks.get(k) !== undefined)) {
                v.startInterval();
                this.logger.info(`Starting default ${k.toUpperCase()} mod stream`);
                if(notify) {
@@ -347,6 +369,7 @@ class Bot {
                        }
                    }
                }
+                await sleep(2000);
            }
        }
    }
@@ -359,6 +382,7 @@ class Bot {
        for (const manager of this.subManagers) {
            if (manager.validConfigLoaded && manager.botState.state !== RUNNING) {
                await manager.start(causedBy, {reason: 'Caused by application startup'});
+                await sleep(2000);
            }
        }

--- a/src/Common/ImageData.ts
+++ b/src/Common/ImageData.ts
@@ -1,9 +1,10 @@
 import fetch from "node-fetch";
 import {Submission} from "snoowrap/dist/objects";
 import {URL} from "url";
-import {absPercentDifference, isValidImageURL} from "../util";
+import {absPercentDifference, getSharpAsync, isValidImageURL} from "../util";
 import sizeOf from "image-size";
 import SimpleError from "../Utils/SimpleError";
+import {Sharp} from "sharp";

 export interface ImageDataOptions {
    width?: number,
@@ -19,7 +20,8 @@ class ImageData {
    url: URL
    variants: ImageData[] = []
    preferredResolution?: [number, number]
-    private buff?: Buffer;
+    sharpImg!: Sharp
+    actualResolution?: [number, number]

    constructor(data: ImageDataOptions, aggressive = false) {
        this.width = data.width;
@@ -31,41 +33,64 @@ class ImageData {
        this.variants = data.variants || [];
    }

-    get data(): Promise<Buffer> {
-        return (async () => {
-            if (this.buff === undefined) {
-                try {
-                    const response = await fetch(this.url.toString())
-                    if (response.ok) {
-                        const ct = response.headers.get('Content-Type');
-                        if (ct !== null && ct.includes('image')) {
-                            this.buff = await response.buffer();
-                            if (this.width === undefined || this.height === undefined) {
-                                const dimensions = sizeOf(this.buff);
-                                this.width = dimensions.width;
-                                this.height = dimensions.height;
-                            }
-                        } else {
-                            throw new SimpleError(`Content-Type for fetched URL ${this.url} did not contain "image"`);
+    async data(format?: string): Promise<Buffer> {
+        await this.sharp();
+        switch(format) {
+            case 'jpg':
+                return this.sharpImg.jpeg().toBuffer();
+            case 'png':
+                return this.sharpImg.png().toBuffer();
+            default:
+                return this.sharpImg.raw().toBuffer();
+        }
+        //return this.buff;
+    }
+
+    async sharp(): Promise<Sharp> {
+        if (this.sharpImg === undefined) {
+            try {
+                const response = await fetch(this.url.toString())
+                if (response.ok) {
+                    const ct = response.headers.get('Content-Type');
+                    if (ct !== null && ct.includes('image')) {
+                        const sFunc = await getSharpAsync();
+                        //const imgInfo = await sFunc(await response.buffer()).ensureAlpha().raw().toBuffer({resolveWithObject: true});
+                        this.sharpImg = await sFunc(await response.buffer()).ensureAlpha();
+                        const meta = await this.sharpImg.metadata();
+                        //this.buff =  imgInfo.data;
+                        //this.buff = await response.buffer();
+                        if (this.width === undefined || this.height === undefined) {
+                            // this.width = imgInfo.info.width;
+                            // this.height = imgInfo.info.height;
+                            this.width = meta.width;
+                            this.height = meta.height;
                        }
+                        //this.actualResolution = [imgInfo.info.width, imgInfo.info.height];
+                        this.actualResolution = [meta.width as number, meta.height as number];
+                        //this.sharpImg = sharpImg;
                    } else {
-                        throw new SimpleError(`URL response was not OK: (${response.status})${response.statusText}`);
+                        throw new SimpleError(`Content-Type for fetched URL ${this.url} did not contain "image"`);
                    }
+                } else {
+                    throw new SimpleError(`URL response was not OK: (${response.status})${response.statusText}`);
+                }


-                } catch (err) {
-                    if(!(err instanceof SimpleError)) {
-                        throw new Error(`Error occurred while fetching response from URL: ${err.message}`);
-                    } else {
-                      throw err;
-                    }
+            } catch (err) {
+                if(!(err instanceof SimpleError)) {
+                    throw new Error(`Error occurred while fetching response from URL: ${err.message}`);
+                } else {
+                    throw err;
                }
            }
-            return this.buff;
-        })();
+        }
+        return this.sharpImg;
    }

    get pixels() {
+        if (this.actualResolution !== undefined) {
+            return this.actualResolution[0] * this.actualResolution[1];
+        }
        if (this.width === undefined || this.height === undefined) {
            return undefined;
        }
@@ -102,6 +127,13 @@ class ImageData {
        });
    }

+    isSameDimensions(otherImage: ImageData) {
+        if (!this.hasDimensions || !otherImage.hasDimensions) {
+            return false;
+        }
+        return this.width === otherImage.width && this.height === otherImage.height;
+    }
+
    static fromSubmission(sub: Submission, aggressive = false): ImageData {
        const url = new URL(sub.url);
        const data: any = {
--- a/src/Common/interfaces.ts
+++ b/src/Common/interfaces.ts
@@ -277,6 +277,16 @@ export interface ImageDetection {
 //     variants?: ImageData[]
 // }

+export interface ImageComparisonResult {
+    isSameDimensions: boolean
+    dimensionDifference: {
+        width: number;
+        height: number;
+    };
+    misMatchPercentage: number;
+    analysisTime: number;
+}
+
 export interface ResembleResult extends ResembleSingleCallbackComparisonResult {
    rawMisMatchPercentage: number
 }
@@ -1253,6 +1263,13 @@ export interface BotInstanceJsonConfig {
         * @default false
         * */
        sharedMod?: boolean,
+
+        /**
+         * If sharing a mod stream stagger pushing relevant Activities to individual subreddits.
+         *
+         * Useful when running many subreddits and rules are potentially cpu/memory/traffic heavy -- allows spreading out load
+         * */
+        stagger?: number,
    },
    /**
     * Settings related to default configurations for queue behavior for subreddits
@@ -1539,6 +1556,7 @@ export interface BotInstanceConfig extends BotInstanceJsonConfig {
    },
    polling: {
        sharedMod: boolean,
+        stagger?: number,
        limit: number,
        interval: number,
    },
--- a/src/ConfigBuilder.ts
+++ b/src/ConfigBuilder.ts
@@ -619,6 +619,7 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
            name: botName,
            polling: {
                sharedMod = false,
+                stagger,
                limit = 100,
                interval = 30,
            } = {},
@@ -723,6 +724,7 @@ export const buildOperatorConfigWithDefaults = (data: OperatorJsonConfig): Opera
            caching: botCache,
            polling: {
                sharedMod,
+                stagger,
                limit,
                interval,
            },
--- a/src/Rule/RecentActivityRule.ts
+++ b/src/Rule/RecentActivityRule.ts
@@ -2,6 +2,7 @@ import {Rule, RuleJSONConfig, RuleOptions, RulePremise, RuleResult} from "./inde
 import {Comment, VoteableContent} from "snoowrap";
 import Submission from "snoowrap/dist/objects/Submission";
 import as from 'async';
+import pMap from 'p-map';
 // @ts-ignore
 import subImageMatch from 'matches-subimage';
 import {
@@ -19,7 +20,7 @@ import {
    parseStringToRegex,
    parseSubredditName,
    parseUsableLinkIdentifier,
-    PASS,
+    PASS, sleep,
    toStrongSubredditState
 } from "../util";
 import {
@@ -112,7 +113,11 @@ export class RecentActivityRule extends Rule {
                if(this.imageDetection.enable) {
                    try {
                        referenceImage = ImageData.fromSubmission(item);
+                        await referenceImage.sharp();
                        referenceImage.setPreferredResolutionByWidth(1000);
+                        if(referenceImage.preferredResolution !== undefined) {
+                            await (referenceImage.getSimilarResolutionVariant(...referenceImage.preferredResolution) as ImageData).sharp();
+                        }
                    } catch (err) {
                        this.logger.verbose(err.message);
                    }
@@ -159,7 +164,7 @@ export class RecentActivityRule extends Rule {
                }
                // parallel all the things
                this.logger.profile('asyncCompare');
-                const results = await Promise.all(viableActivity.map(x => ci(x)));
+                const results = await pMap(viableActivity, ci, {concurrency: 2});
                this.logger.profile('asyncCompare', {level: 'debug', message: 'Total time for image download and compare'});
                const totalAnalysisTime = analysisTimes.reduce((acc, x) => acc + x,0);
                this.logger.debug(`Reference image compared ${analysisTimes.length} times. Timings: Avg ${formatNumber(totalAnalysisTime / analysisTimes.length, {toFixed: 0})}ms | Max: ${Math.max(...analysisTimes)}ms | Min: ${Math.min(...analysisTimes)}ms | Total: ${totalAnalysisTime}ms (${formatNumber(totalAnalysisTime/1000)}s)`);
--- a/src/Schema/OperatorConfig.json
+++ b/src/Schema/OperatorConfig.json
@@ -79,6 +79,10 @@
                                    "default": false,
                                    "description": "If set to `true` all subreddits polling unmoderated/modqueue with default polling settings will share a request to \"r/mod\"\notherwise each subreddit will poll its own mod view\n\n* ENV => `SHARE_MOD`\n* ARG => `--shareMod`",
                                    "type": "boolean"
+                                },
+                                "stagger": {
+                                    "description": "If sharing a mod stream stagger pushing relevant Activities to individual subreddits.\n\nUseful when running many subreddits and rules are potentially cpu/memory/traffic heavy -- allows spreading out load",
+                                    "type": "number"
                                }
                            },
                            "type": "object"
--- a/src/Subreddit/Manager.ts
+++ b/src/Subreddit/Manager.ts
@@ -759,11 +759,10 @@ export class Manager {
                }
            };

-            stream.on('item', onItem);
-
            if (modStreamType !== undefined) {
                this.modStreamCallbacks.set(pollOn, onItem);
            } else {
+                stream.on('item', onItem);
                // @ts-ignore
                stream.on('error', async (err: any) => {

@@ -943,10 +942,11 @@ export class Manager {
                s.end();
            }
            this.streams = [];
-            for (const [k, v] of this.modStreamCallbacks) {
-                const stream = this.cacheManager.modStreams.get(k) as Poll<Snoowrap.Submission | Snoowrap.Comment>;
-                stream.removeListener('item', v);
-            }
+            // for (const [k, v] of this.modStreamCallbacks) {
+            //     const stream = this.cacheManager.modStreams.get(k) as Poll<Snoowrap.Submission | Snoowrap.Comment>;
+            //     stream.removeListener('item', v);
+            // }
+            this.modStreamCallbacks = new Map();
            this.startedAt = undefined;
            this.logger.info(`Events STOPPED by ${causedBy}`);
            this.eventsState = {
--- a/src/Subreddit/Streams.ts
+++ b/src/Subreddit/Streams.ts
@@ -5,7 +5,8 @@ import {PollConfiguration} from "snoostorm/out/util/Poll";
 import {ClearProcessedOptions, DEFAULT_POLLING_INTERVAL} from "../Common/interfaces";
 import dayjs, {Dayjs} from "dayjs";
 import { Duration } from "dayjs/plugin/duration";
-import {parseDuration} from "../util";
+import {parseDuration, sleep} from "../util";
+import setRandomInterval from 'set-random-interval';

 type Awaitable<T> = Promise<T> | T;

@@ -27,6 +28,7 @@ export class SPoll<T extends object> extends Poll<T> {
    clearProcessedSize?: number;
    clearProcessedAfter?: Dayjs;
    retainProcessed: number = 0;
+    randInterval?: { clear: () => void };

    constructor(options: RCBPollConfiguration<T>) {
        super(options);
@@ -51,7 +53,7 @@ export class SPoll<T extends object> extends Poll<T> {

    startInterval = () => {
        this.running = true;
-        this.interval = setInterval((function (self) {
+        this.randInterval = setRandomInterval((function (self) {
            return async () => {
                try {
                    // clear the tracked, processed activity ids after a set period or number of activities have been processed
@@ -93,11 +95,14 @@ export class SPoll<T extends object> extends Poll<T> {
                    self.end();
                }
            }
-        })(this), this.frequency);
+        })(this), this.frequency - 1, this.frequency + 1);
    }

    end = () => {
        this.running = false;
+        if(this.randInterval !== undefined) {
+            this.randInterval.clear();
+        }
        super.end();
    }
 }
--- a/src/util.ts
+++ b/src/util.ts
@@ -9,7 +9,7 @@ import {InvalidOptionArgumentError} from "commander";
 import Submission from "snoowrap/dist/objects/Submission";
 import {Comment} from "snoowrap";
 import {inflateSync, deflateSync} from "zlib";
-import sizeOf from 'image-size';
+import pixelmatch from 'pixelmatch';
 import {
    ActivityWindowCriteria,
    CacheOptions,
@@ -17,7 +17,7 @@ import {
    DurationComparison,
    GenericComparison,
    HistoricalStats,
-    HistoricalStatsDisplay,
+    HistoricalStatsDisplay, ImageComparisonResult,
    //ImageData,
    ImageDetection,
    //ImageDownloadOptions,
@@ -51,6 +51,7 @@ import reRegExp from '@stdlib/regexp-regexp';
 import fetch, {Response} from "node-fetch";
 import { URL } from "url";
 import ImageData from "./Common/ImageData";
+import {Sharp, SharpOptions} from "sharp";

 const ReReg = reRegExp();

@@ -1194,6 +1195,19 @@ export const isValidImageURL = (str: string): boolean => {
 }

 let resembleCIFunc: Function;
+type SharpCreate = (input?:
+| Buffer
+| Uint8Array
+| Uint8ClampedArray
+| Int8Array
+| Uint16Array
+| Int16Array
+| Uint32Array
+| Int32Array
+| Float32Array
+| Float64Array
+| string,) => Sharp;
+let sharpImg: SharpCreate;

 const getCIFunc = async () => {
    if (resembleCIFunc === undefined) {
@@ -1207,7 +1221,103 @@ const getCIFunc = async () => {
    return resembleCIFunc;
 }

-export const compareImages = async (data1: ImageData, data2: ImageData, threshold?: number, variantDimensionDiff = 0): Promise<[ResembleResult, boolean?]> => {
+export const getSharpAsync = async (): Promise<SharpCreate> => {
+    if (sharpImg === undefined) {
+        const sharpModule = await import('sharp');
+        if (sharpModule === undefined) {
+            throw new Error('Could not import sharp');
+        }
+        // @ts-ignore
+        sharpImg = sharpModule.default;
+    }
+    return sharpImg;
+}
+
+export const compareImages = async (data1: ImageData, data2: ImageData, threshold: number, variantDimensionDiff = 0): Promise<[ImageComparisonResult, boolean, string[]]> => {
+    let results: ImageComparisonResult | undefined;
+    const errors = [];
+    try {
+        results = await pixelImageCompare(data1, data2);
+    } catch (err) {
+        if(!(err instanceof SimpleError)) {
+            errors.push(err.message);
+        }
+        // swallow this and continue with resemble
+    }
+    if (results === undefined) {
+        results = await resembleImageCompare(data1, data2, threshold, variantDimensionDiff);
+    }
+
+    return [results, results.misMatchPercentage < threshold, errors];
+}
+
+export const pixelImageCompare = async (data1: ImageData, data2: ImageData): Promise<ImageComparisonResult> => {
+
+    let pixelDiff: number | undefined = undefined;
+
+    let sharpFunc: SharpCreate;
+
+    try {
+        sharpFunc = await getSharpAsync();
+    } catch (err) {
+        err.message = `Unable to do image comparison due to an issue importing the comparison library. It is likely sharp is not installed (see ContextMod docs). Error Message: ${err.message}`;
+        throw err;
+    }
+
+    if(data1.preferredResolution !== undefined) {
+        const [prefWidth, prefHeight] = data1.preferredResolution;
+        const prefImgData = data2.getSimilarResolutionVariant(prefWidth, prefHeight);
+        if(prefImgData !== undefined) {
+            const refThumbnail = data1.getSimilarResolutionVariant(prefWidth, prefHeight) as ImageData;
+            // go ahead and fetch comparing image data so analysis time doesn't include download time
+            await prefImgData.data();
+            const [actualWidth, actualHeight] = refThumbnail.actualResolution as [number, number];
+            //const normalRefData = await sharpFunc(await refThumbnail.data).normalise().ensureAlpha().raw().toBuffer({resolveWithObject: true});
+            const time = Date.now();
+            //pixelDiff = pixelmatch(normalRefData.data, await sharpFunc(await prefImgData.data).ensureAlpha().raw().toBuffer(), null, normalRefData.info.width, normalRefData.info.height);
+            const refInfo = await (await refThumbnail.sharp()).resize(400, null, {fit: 'outside'}).raw().toBuffer({resolveWithObject: true});
+            pixelDiff = pixelmatch(refInfo.data, await (await prefImgData.sharp()).resize(400, null, {fit: 'outside'}).raw().toBuffer(), null, refInfo.info.width, refInfo.info.height);
+            return {
+                isSameDimensions: true,
+                dimensionDifference: {
+                    height: 0,
+                    width: 0,
+                },
+                misMatchPercentage: pixelDiff / (refInfo.info.width * refInfo.info.height),
+                analysisTime: Date.now() - time,
+            }
+        }
+    }
+    // try to determine by provided dimensions (if any) before downloading
+    if(data1.hasDimensions && data2.hasDimensions && !data1.isSameDimensions(data2)) {
+        throw new SimpleError('No images have same dimensions');
+    }
+    // download anyway because resemblejs uses a lot of memory (15-30MB per image) vs. 2 downloads which will be ~2-5MB
+    if(!data1.hasDimensions) {
+        await data1.data;
+    }
+    if(!data2.hasDimensions) {
+        await data2.data;
+    }
+    // should have all dimensions now
+    if(!data1.isSameDimensions(data2)) {
+        throw new SimpleError('No images have same dimensions');
+    }
+    // ok so now are sure everything is same dimensions
+    const time = Date.now();
+    pixelDiff = pixelmatch(await data1.data(), await data2.data(), null, data1.width as number, data2.height as number);
+    return {
+        isSameDimensions: true,
+        dimensionDifference: {
+            height: 0,
+            width: 0,
+        },
+        misMatchPercentage: pixelDiff / (data1.pixels as number),
+        analysisTime: Date.now() - time,
+    }
+}
+
+export const resembleImageCompare = async (data1: ImageData, data2: ImageData, threshold?: number, variantDimensionDiff = 0): Promise<ImageComparisonResult> => {
    let ci: Function;

    try {
@@ -1217,7 +1327,9 @@ export const compareImages = async (data1: ImageData, data2: ImageData, threshol
        throw err;
    }

-    let results: ResembleResult | undefined = undefined;
+    let results: ImageComparisonResult | undefined = undefined;
+    // @ts-ignore
+    let resResult: ResembleResult = undefined;

    //const [minWidth, minHeight] = getMinimumDimensions(data1, data2);
    const compareOptions = {
@@ -1244,15 +1356,22 @@ export const compareImages = async (data1: ImageData, data2: ImageData, threshol
        const [prefWidth, prefHeight] = data1.preferredResolution;
        const prefImgData = data2.getSimilarResolutionVariant(prefWidth, prefHeight, variantDimensionDiff);
        if(prefImgData !== undefined) {
-            results = await ci(await data1.getSimilarResolutionVariant(prefWidth, prefHeight)?.data, await prefImgData.data, compareOptions) as ResembleResult;
+            resResult = await ci(await (await (data1.getSimilarResolutionVariant(prefWidth, prefHeight) as ImageData).sharp()).resize(400, null, {fit: 'outside'}).jpeg().toBuffer()
+                , await (await prefImgData.sharp()).resize(400, null, {fit: 'outside'}).jpeg().toBuffer()
+                , compareOptions) as ResembleResult;
        }
    }
-    if(results === undefined) {
-        results = await ci(await data1.data, await data2.data, compareOptions) as ResembleResult;
+    if(resResult === undefined) {
+        resResult = await ci(await (await data1.sharp()).resize(400, null, {fit: 'outside'}).jpeg().toBuffer(),
+            await (await data2.sharp()).resize(400, null, {fit: 'outside'}).jpeg().toBuffer(), compareOptions) as ResembleResult;
    }

-    const sameImage = threshold === undefined ? undefined : results.rawMisMatchPercentage < threshold;
-    return [results, sameImage];
+    return {
+        isSameDimensions: resResult.isSameDimensions,
+        dimensionDifference: resResult.dimensionDifference,
+        misMatchPercentage: resResult.rawMisMatchPercentage,
+        analysisTime: resResult.analysisTime
+    };
 }

 export const createHistoricalStatsDisplay = (data: HistoricalStats): HistoricalStatsDisplay => {