fix!(support): migrate to opencv-bindings and away from opencv4nodejs
MODIFY
packages/doctor/lib/general.js
packages/doctor/lib/general.js
@@ -48,23 +48,6 @@ class NodeVersionCheck extends DoctorCheck {
}
checks.push(new NodeVersionCheck());
-class OptionalOpencv4nodejsCommandCheck extends DoctorCheck {
- async diagnose () {
- const packageName = '@u4/opencv4nodejs';
- const packageInfo = await getNpmPackageInfo(packageName);
-
- if (packageInfo) {
- return okOptional(`${packageName} is installed at: ${packageInfo.path}. Installed version is: ${packageInfo.version}`);
- }
- return nokOptional(`${packageName} cannot be found.`);
- }
-
- async fix () { // eslint-disable-line require-await
- return `Why ${'opencv4nodejs'.bold} is needed and how to install it: https://github.com/appium/appium/blob/master/docs/en/writing-running-appium/image-comparison.md`;
- }
-}
-checks.push(new OptionalOpencv4nodejsCommandCheck());
-
class OptionalFfmpegCommandCheck extends DoctorCheck {
async diagnose () {
const ffmpegPath = await resolveExecutablePath('ffmpeg');
@@ -98,6 +81,6 @@ class OptionalMjpegConsumerCommandCheck extends DoctorCheck {
checks.push(new OptionalMjpegConsumerCommandCheck());
-export { NodeBinaryCheck, NodeVersionCheck,
- OptionalOpencv4nodejsCommandCheck, OptionalFfmpegCommandCheck, OptionalMjpegConsumerCommandCheck };
+export { NodeBinaryCheck, NodeVersionCheck, OptionalFfmpegCommandCheck,
+ OptionalMjpegConsumerCommandCheck };
export default checks;

MODIFY
packages/doctor/test/general-specs.js
packages/doctor/test/general-specs.js
// transpile:mocha
import { NodeBinaryCheck, NodeVersionCheck,
- OptionalOpencv4nodejsCommandCheck, OptionalFfmpegCommandCheck, OptionalMjpegConsumerCommandCheck } from '../lib/general';
+ OptionalFfmpegCommandCheck, OptionalMjpegConsumerCommandCheck } from '../lib/general';
import * as tp from 'teen_process';
import * as utils from '../lib/utils';
import NodeDetector from '../lib/node-detector';
@@ -79,46 +79,6 @@ describe('general', function () {
});
}));
- describe('OptionalOpencv4nodejsCommandCheck', withMocks({tp}, (mocks) => {
- let check = new OptionalOpencv4nodejsCommandCheck();
- it('autofix', function () {
- check.autofix.should.not.be.ok;
- });
- it('diagnose - success', async function () {
- mocks.tp.expects('exec').once().returns({stdout: `
- {
- "dependencies": {
- "@u4/opencv4nodejs": {
- "version": "4.13.0",
- "from": "@u4/opencv4nodejs",
- "resolved": "https://registry.npmjs.org/@u4/opencv4nodejs/-/opencv4nodejs-4.14.1.tgz"
- }
- },
- "path": "/path/to/node/node/v11.4.0/lib"
- }
- `, stderr: ''});
- (await check.diagnose()).should.deep.equal({
- ok: true,
- optional: true,
- message: '@u4/opencv4nodejs is installed at: /path/to/node/node/v11.4.0/lib. Installed version is: 4.13.0'
- });
- mocks.verify();
- });
- it('diagnose - failure', async function () {
- mocks.tp.expects('exec').once().returns({stdout: '{}', stderr: ''});
- (await check.diagnose()).should.deep.equal({
- ok: false,
- optional: true,
- message: '@u4/opencv4nodejs cannot be found.'
- });
- mocks.verify();
- });
- it('fix', async function () {
- removeColors(await check.fix()).should.
- equal('Why opencv4nodejs is needed and how to install it: https://github.com/appium/appium/blob/master/docs/en/writing-running-appium/image-comparison.md');
- });
- }));
-
describe('OptionalFfmpegCommandCheck', withMocks({tp, utils}, (mocks) => {
let check = new OptionalFfmpegCommandCheck();
it('autofix', function () {

MODIFY
packages/support/lib/image-util.js
packages/support/lib/image-util.js
@@ -5,15 +5,10 @@ import { PNG } from 'pngjs';
import B from 'bluebird';
import { hasValue } from './util';
import log from './logger';
-import { requirePackage } from './node';
-
-
-const OCV_OLD = 'opencv4nodejs';
-const OCV_NEW = '@u4/opencv4nodejs';
+let cv;
const { MIME_JPEG, MIME_PNG, MIME_BMP } = Jimp;
-let cv = null;
/**
* @typedef {Object} Region
@@ -42,26 +37,25 @@ const SCANLINE_FILTER_METHOD = 4;
const DEFAULT_MATCH_THRESHOLD = 0.5;
const MATCH_NEIGHBOUR_THRESHOLD = 10;
-const AVAILABLE_DETECTORS = [
- 'AKAZE',
- 'AGAST',
- 'BRISK',
- 'FAST',
- 'GFTT',
- 'KAZE',
- 'MSER',
- 'SIFT',
- 'ORB',
-];
+const AVAILABLE_DETECTORS = {
+ AKAZE: 'AKAZE',
+ AGAST: 'AgastFeatureDetector',
+ BRISK: 'BRISK',
+ FAST: 'FastFeatureDetector',
+ GFTT: 'GFTTDetector',
+ KAZE: 'KAZE',
+ MSER: 'MSER',
+ ORB: 'ORB',
+};
-const AVAILABLE_MATCHING_FUNCTIONS = [
- 'FlannBased',
- 'BruteForce',
- 'BruteForceL1',
- 'BruteForceHamming',
- 'BruteForceHammingLut',
- 'BruteForceSL2',
-];
+const AVAILABLE_MATCHING_FUNCTIONS = {
+ FlannBased: 'FlannBased',
+ BruteForce: 'BruteForce',
+ BruteForceL1: 'BruteForce-L1',
+ BruteForceHamming: 'BruteForce-Hamming',
+ BruteForceHammingLut: 'BruteForce-HammingLUT',
+ BruteForceSL2: 'BruteForce-SL2',
+};
const MATCHING_METHODS = [
'TM_CCOEFF',
@@ -123,34 +117,28 @@ async function getJimpImage (data) {
}
/**
- * @throws {Error} If opencv4nodejs module is not installed or cannot be loaded
+ * Spins until the opencv-bindings module is fully loaded
*/
-async function initOpenCV () {
- if (cv) {
- return;
+async function initOpenCv () {
+ cv = require('opencv-bindings');
+ let waited = false;
+ while (!cv.getBuildInformation) {
+ log.info(`OpenCV module not fully loaded, waiting...`);
+ await B.delay(500);
+ waited = true;
}
-
- log.debug(`Initializing opencv`);
- for (const ocvPackage of [OCV_OLD, OCV_NEW]) {
- try {
- log.debug(`Attempting to load '${ocvPackage}'`);
- cv = await requirePackage(ocvPackage);
- break;
- } catch (err) {
- log.warn(`Unable to load '${ocvPackage}': ${err.message}`);
+ // opencv-bindings sets a global unhandledRejection handler of an abort, which we don't want, so
+ // undo it here. https://github.com/opencv/opencv/issues/21481
+ process.removeAllListeners('unhandledRejection');
+ if (waited) {
+ log.info(`OpenCV module successfully loaded`);
}
}
- if (!cv) {
- throw new Error(`An opencv node module is required to use OpenCV features. ` +
- `Please install one first (e.g., 'npm i -g ${OCV_NEW}') and restart Appium. ` +
- 'Read https://github.com/UrielCh/opencv4nodejs#how-to-install for more details on this topic.');
- }
-}
/**
* @typedef {Object} MatchComputationResult
- * @property {cv.DescriptorMatch} desciptor - OpenCV match descriptor
+ * @property {cv.DescriptorMatch} descriptor - OpenCV match descriptor
* @property {Array<cv.KeyPoint>} keyPoints - The array of key points
*/
@@ -165,9 +153,11 @@ async function initOpenCV () {
*
* @returns {MatchComputationResult}
*/
-async function detectAndCompute (img, detector) {
- const keyPoints = await detector.detectAsync(img);
- const descriptor = await detector.computeAsync(img, keyPoints);
+function detectAndCompute (img, detector) {
+ const keyPoints = new cv.KeyPointVector();
+ const descriptor = new cv.Mat();
+ detector.detect(img, keyPoints);
+ detector.compute(img, keyPoints, descriptor);
return {
keyPoints,
descriptor
@@ -227,21 +217,23 @@ function highlightRegion (mat, region) {
}
// highlight in red
- const color = new cv.Vec(0, 0, 255);
+ const color = new cv.Scalar(255, 0, 0, 255);
const thickness = 2;
- mat.drawRectangle(new cv.Rect(region.x, region.y, region.width, region.height), color, thickness, cv.LINE_8);
+ const topLeft = new cv.Point(region.x, region.y);
+ const botRight = new cv.Point(region.x + region.width, region.y + region.height);
+ cv.rectangle(mat, topLeft, botRight, color, thickness, cv.LINE_8, 0);
return mat;
}
/**
* @typedef {Object} MatchingOptions
* @property {?string} detectorName ['ORB'] One of possible OpenCV feature detector names
- * from `AVAILABLE_DETECTORS` array.
+ * from keys of the `AVAILABLE_DETECTORS` object.
* Some of these methods (FAST, AGAST, GFTT, FAST, SIFT and MSER) are not available
* in the default OpenCV installation and have to be enabled manually before
* library compilation.
* @property {?string} matchFunc ['BruteForce'] The name of the matching function.
- * Should be one of `AVAILABLE_MATCHING_FUNCTIONS` array.
+ * Should be one of the keys of the `AVAILABLE_MATCHING_FUNCTIONS` object.
* @property {?number|Function} goodMatchesFactor The maximum count of "good" matches
* (e. g. with minimal distances) or a function, which accepts 3 arguments: the current distance,
* minimal distance, maximum distance and returns true or false to include or exclude the match.
@@ -280,36 +272,40 @@ function highlightRegion (mat, region) {
* @throws {Error} If `detectorName` value is unknown.
*/
async function getImagesMatches (img1Data, img2Data, options = {}) {
- await initOpenCV();
+ await initOpenCv();
+ let img1, img2, detector, result1, result2, matcher, matchesVec;
+ try {
const {detectorName = 'ORB', visualize = false,
goodMatchesFactor, matchFunc = 'BruteForce'} = options;
- if (!_.includes(AVAILABLE_DETECTORS, detectorName)) {
+ if (!_.includes(_.keys(AVAILABLE_DETECTORS), detectorName)) {
throw new Error(`'${detectorName}' detector is unknown. ` +
- `Only ${JSON.stringify(AVAILABLE_DETECTORS)} detectors are supported.`);
+ `Only ${JSON.stringify(_.keys(AVAILABLE_DETECTORS))} detectors are supported.`);
}
- if (!_.includes(AVAILABLE_MATCHING_FUNCTIONS, matchFunc)) {
+ if (!_.includes(_.keys(AVAILABLE_MATCHING_FUNCTIONS), matchFunc)) {
throw new Error(`'${matchFunc}' matching function is unknown. ` +
- `Only ${JSON.stringify(AVAILABLE_MATCHING_FUNCTIONS)} matching functions are supported.`);
- }
-
- const detector = new cv[`${detectorName}Detector`]();
- const [img1, img2] = await B.all([
- cv.imdecodeAsync(img1Data),
- cv.imdecodeAsync(img2Data)
- ]);
- const [result1, result2] = await B.all([
- detectAndCompute(img1, detector),
- detectAndCompute(img2, detector)
- ]);
+ `Only ${JSON.stringify(_.keys(AVAILABLE_MATCHING_FUNCTIONS))} matching functions are supported.`);
+ }
+
+ detector = new cv[AVAILABLE_DETECTORS[detectorName]]();
+ ([img1, img2] = await B.all([
+ cvMatFromImage(img1Data),
+ cvMatFromImage(img2Data)
+ ]));
+ result1 = detectAndCompute(img1, detector);
+ result2 = detectAndCompute(img2, detector);
+ matcher = new cv.DescriptorMatcher(AVAILABLE_MATCHING_FUNCTIONS[matchFunc]);
+ matchesVec = new cv.DMatchVector();
let matches = [];
- try {
- matches = await cv[`match${matchFunc}Async`](result1.descriptor, result2.descriptor);
- } catch (e) {
- throw new Error(`Cannot find any matches between the given images. Try another detection algorithm. ` +
- ` Original error: ${e}`);
+ matcher.match(result1.descriptor, result2.descriptor, matchesVec);
+ const totalCount = matchesVec.size();
+ if (totalCount < 1) {
+ throw new Error(`Could not find any matches between images. Double-check orientation, ` +
+ `resolution, or use another detector or matching function.`);
+ }
+ for (let i = 0; i < totalCount; i++) {
+ matches.push(matchesVec.get(i));
}
- const totalCount = matches.length;
if (hasValue(goodMatchesFactor)) {
if (_.isFunction(goodMatchesFactor)) {
const distances = matches.map((match) => match.distance);
@@ -327,7 +323,7 @@ async function getImagesMatches (img1Data, img2Data, options = {}) {
}
const extractPoint = (keyPoints, indexPropertyName) => (match) => {
- const {pt, point} = keyPoints[match[indexPropertyName]];
+ const {pt, point} = keyPoints.get(match[indexPropertyName]);
// https://github.com/justadudewhohacks/opencv4nodejs/issues/584
return (pt || point);
};
@@ -345,7 +341,13 @@ async function getImagesMatches (img1Data, img2Data, options = {}) {
count: matches.length,
};
if (visualize) {
- const visualization = cv.drawMatches(img1, img2, result1.keyPoints, result2.keyPoints, matches);
+ const goodMatchesVec = new cv.DMatchVector();
+ for (let i = 0; i < matches.length; i++) {
+ goodMatchesVec.push_back(matches[i]);
+ }
+ const visualization = new cv.Mat();
+ const color = new cv.Scalar(0, 255, 0, 255);
+ cv.drawMatches(img1, result1.keyPoints, img2, result2.keyPoints, goodMatchesVec, visualization, color);
highlightRegion(visualization, rect1);
highlightRegion(visualization, {
x: img1.cols + rect2.x,
@@ -353,9 +355,16 @@ async function getImagesMatches (img1Data, img2Data, options = {}) {
width: rect2.width,
height: rect2.height
});
- result.visualization = await cv.imencodeAsync('.png', visualization);
+ result.visualization = await jimpImgFromCvMat(visualization).getBufferAsync(Jimp.MIME_PNG);
}
+
return result;
+ } finally {
+ try {
+ img1.delete(); img2.delete(); detector.delete(); result1.keyPoints.delete(); result1.descriptor.delete();
+ result2.keyPoints.delete(); result2.descriptor.delete(); matcher.delete(); matchesVec.delete();
+ } catch (ign) {}
+ }
}
/**
@@ -395,54 +404,53 @@ async function getImagesMatches (img1Data, img2Data, options = {}) {
* @throws {Error} If the given images have different resolution.
*/
async function getImagesSimilarity (img1Data, img2Data, options = {}) {
- await initOpenCV();
+ await initOpenCv();
const {
method = DEFAULT_MATCHING_METHOD,
visualize = false,
} = options;
- let [template, reference] = await B.all([
- cv.imdecodeAsync(img1Data),
- cv.imdecodeAsync(img2Data)
- ]);
+
+ let template, reference, matched;
+ try {
+ ([template, reference] = await B.all([
+ cvMatFromImage(img1Data),
+ cvMatFromImage(img2Data)
+ ]));
if (template.rows !== reference.rows || template.cols !== reference.cols) {
throw new Error('Both images are expected to have the same size in order to ' +
'calculate the similarity score.');
}
- [template, reference] = await B.all([
- template.convertToAsync(cv.CV_8UC3),
- reference.convertToAsync(cv.CV_8UC3)
- ]);
+ template.convertTo(template, cv.CV_8UC3);
+ reference.convertTo(reference, cv.CV_8UC3);
- let matched;
- try {
- matched = await reference.matchTemplateAsync(template, toMatchingMethod(method));
- } catch (e) {
- throw new Error(`The reference image did not match to the template one. Original error: ${e.message}`);
- }
- const minMax = await matched.minMaxLocAsync();
+ matched = new cv.Mat();
+ cv.matchTemplate(reference, template, matched, toMatchingMethod(method));
+ const minMax = cv.minMaxLoc(matched);
const result = {
score: minMax.maxVal
};
+
if (visualize) {
- const resultMat = new cv.Mat(template.rows, template.cols * 2, cv.CV_8UC3);
- await B.all([
- reference.copyToAsync(
- resultMat.getRegion(new cv.Rect(0, 0, reference.cols, reference.rows))),
- template.copyToAsync(
- resultMat.getRegion(new cv.Rect(reference.cols, 0, template.cols, template.rows)))
- ]);
- let mask = reference.absdiff(template);
- mask = await mask.cvtColorAsync(cv.COLOR_BGR2GRAY);
- let contours = [];
+ let bothImages, resultMat, mask, contours, hierarchy;
try {
- mask = await mask.thresholdAsync(128, 255, cv.THRESH_BINARY | cv.THRESH_OTSU);
- contours = await mask.findContoursAsync(cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE);
- } catch (ign) {
- // No contours can be found, which means, most likely, that images are equal
- }
- for (const contour of contours) {
- const boundingRect = contour.boundingRect();
+ resultMat = new cv.Mat(template.rows, template.cols * 2, cv.CV_8UC3);
+ bothImages = new cv.MatVector();
+ bothImages.push_back(reference);
+ bothImages.push_back(template);
+ cv.hconcat(bothImages, resultMat);
+
+ mask = new cv.Mat();
+ cv.absdiff(reference, template, mask);
+ cv.cvtColor(mask, mask, cv.COLOR_BGR2GRAY, 0);
+
+ cv.threshold(mask, mask, 128, 255, cv.THRESH_BINARY | cv.THRESH_OTSU);
+ contours = new cv.MatVector();
+ hierarchy = new cv.Mat();
+ cv.findContours(mask, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE);
+
+ for (let i = 0; i < contours.size(); i++) {
+ const boundingRect = cv.boundingRect(contours.get(i));
highlightRegion(resultMat, boundingRect);
highlightRegion(resultMat, {
x: reference.cols + boundingRect.x,
@@ -451,9 +459,19 @@ async function getImagesSimilarity (img1Data, img2Data, options = {}) {
height: boundingRect.height
});
}
- result.visualization = await cv.imencodeAsync('.png', resultMat);
+ result.visualization = await jimpImgFromCvMat(resultMat).getBufferAsync(Jimp.MIME_PNG);
+ } finally {
+ try {
+ bothImages.delete(); resultMat.delete(); mask.delete(); contours.delete(); hierarchy.delete();
+ } catch (ign) {}
+ }
}
return result;
+ } finally {
+ try {
+ template.delete(); reference.delete(); matched.delete();
+ } catch (ign) {}
+ }
}
/**
@@ -503,7 +521,7 @@ async function getImagesSimilarity (img1Data, img2Data, options = {}) {
* @throws {Error} If no occurrences of the partial image can be found in the full image
*/
async function getImageOccurrence (fullImgData, partialImgData, options = {}) {
- await initOpenCV();
+ await initOpenCv();
const {
visualize = false,
@@ -513,26 +531,37 @@ async function getImageOccurrence (fullImgData, partialImgData, options = {}) {
method = DEFAULT_MATCHING_METHOD,
} = options;
- const [fullImg, partialImg] = await B.all([
- cv.imdecodeAsync(fullImgData),
- cv.imdecodeAsync(partialImgData)
- ]);
+ let fullImg, partialImg, matched;
+
+ try {
+ ([fullImg, partialImg] = await B.all([
+ cvMatFromImage(fullImgData),
+ cvMatFromImage(partialImgData)
+ ]));
+ matched = new cv.Mat();
const results = [];
let visualization = null;
try {
- const matched = await fullImg.matchTemplateAsync(partialImg, toMatchingMethod(method));
- const minMax = await matched.minMaxLocAsync();
+ cv.matchTemplate(fullImg, partialImg, matched, toMatchingMethod(method));
+ const minMax = cv.minMaxLoc(matched);
if (multiple) {
- const nonZeroMatchResults = matched.threshold(threshold, 1, cv.THRESH_BINARY)
- .convertTo(cv.CV_8U)
- .findNonZero();
- const matches = filterNearMatches(nonZeroMatchResults, matchNeighbourThreshold);
+ const matches = [];
+ for (let row = 0; row < matched.rows; row++) {
+ for (let col = 0; col < matched.cols; col++) {
+ const score = matched.floatAt(row, col);
+ if (score >= threshold) {
+ matches.push({score, x: col, y: row});
+ }
+ }
+ }
+
+ const nearMatches = filterNearMatches(matches, matchNeighbourThreshold);
- for (const {x, y} of matches) {
+ for (const {x, y, score} of nearMatches) {
results.push({
- score: matched.at(y, x),
+ score,
rect: {
x, y,
width: partialImg.cols,
@@ -564,24 +593,41 @@ async function getImageOccurrence (fullImgData, partialImgData, options = {}) {
}
if (visualize) {
- const fullHighlightedImage = fullImg.copy();
+ const fullHighlightedImage = fullImg.clone();
for (const result of results) {
- const singleHighlightedImage = fullImg.copy();
+ const singleHighlightedImage = fullImg.clone();
highlightRegion(singleHighlightedImage, result.rect);
highlightRegion(fullHighlightedImage, result.rect);
- result.visualization = await cv.imencodeAsync('.png', singleHighlightedImage);
+ result.visualization = await jimpImgFromCvMat(singleHighlightedImage).getBufferAsync(Jimp.MIME_PNG);
}
- visualization = await cv.imencodeAsync('.png', fullHighlightedImage);
+ visualization = await jimpImgFromCvMat(fullHighlightedImage).getBufferAsync(Jimp.MIME_PNG);
}
-
return {
rect: results[0].rect,
score: results[0].score,
visualization,
multiple: results
};
+ } finally {
+ try {
+ fullImg.delete(); partialImg.delete(); matched.delete();
+ } catch (ign) {}
+ }
+}
+
+function jimpImgFromCvMat (mat) {
+ return new Jimp({
+ width: mat.cols,
+ height: mat.rows,
+ data: Buffer.from(mat.data)
+ });
+}
+
+async function cvMatFromImage (img) {
+ const jimpImg = await Jimp.read(img);
+ return cv.matFromImageData(jimpImg.bitmap);
}
/**
@@ -711,5 +757,5 @@ function getRectIntersection (rect, imageSize) {
export {
cropBase64Image, base64ToImage, imageToBase64, cropImage, getImagesMatches,
getImagesSimilarity, getImageOccurrence, getJimpImage, MIME_JPEG, MIME_PNG,
- MIME_BMP,
+ MIME_BMP
};

MODIFY
packages/support/package.json
packages/support/package.json
"mv": "2.1.1",
"ncp": "2.0.0",
"npmlog": "5.0.1",
+ "opencv-bindings": "^4.5.5",
"plist": "3.0.4",
"pluralize": "8.0.0",
"pngjs": "6.0.0",

MODIFY
packages/support/test/image-util-e2e-specs.js
packages/support/test/image-util-e2e-specs.js
@@ -16,10 +16,6 @@ async function getImage (name) {
}
describe('image-util', function () {
- before(function () {
- // TODO: remove when opencv4nodejs is fixed
- return this.skip();
- });
describe('cropBase64Image', function () {
let originalImage = null;
@@ -151,8 +147,8 @@ describe('image-util', function () {
});
it('should reject matches that fall below a threshold', async function () {
- await getImageOccurrence(originalImage, numberImage, {threshold: 1.0, multiple: true})
- .should.eventually.be.rejectedWith(/threshold/);
+ const { multiple } = await getImageOccurrence(originalImage, numberImage, {threshold: 1.0, multiple: true});
+ multiple.length.should.be.eq(1);
});
it('should visualize the partial image position in the full image', async function () {

fix!(support): migrate to opencv-bindings and away from opencv4nodejs