fix(images-plugin): fix bad merge and tests
adds/fixes a couple types; sets up type dependencies
update e2e tests to pull fake driver & plugin from local
fix sinon usage
fix some assertions
MODIFY
packages/images-plugin/lib/compare.js
packages/images-plugin/lib/compare.js
import _ from 'lodash';
import { errors } from '@appium/base-driver';
-import { imageUtil } from '@appium/support';
+import { getImagesMatches, getImagesSimilarity, getImageOccurrence } from '@appium/opencv';
const MATCH_FEATURES_MODE = 'matchFeatures';
const GET_SIMILARITY_MODE = 'getSimilarity';
@@ -20,11 +20,11 @@ const DEFAULT_MATCH_THRESHOLD = 0.4;
* @param {string} secondImage - Base64-encoded image file.
* All image formats, that OpenCV library itself accepts, are supported.
* @param {?Object} options [{}] - The content of this dictionary depends
- * on the actual `mode` value. See the documentation on `appium-support`
+ * on the actual `mode` value. See the documentation on `@appium/support`
* module for more details.
* @returns {Object} The content of the resulting dictionary depends
* on the actual `mode` and `options` values. See the documentation on
- * `appium-support` module for more details.
+ * `@appium/support` module for more details.
* @throws {Error} If required OpenCV modules are not installed or
* if `mode` value is incorrect or if there was an unexpected issue while
* matching the images.
@@ -32,27 +32,51 @@ const DEFAULT_MATCH_THRESHOLD = 0.4;
async function compareImages (mode, firstImage, secondImage, options = {}) {
const img1 = Buffer.from(firstImage, 'base64');
const img2 = Buffer.from(secondImage, 'base64');
- let result = {};
+ let result = null;
switch (_.toLower(mode)) {
case MATCH_FEATURES_MODE.toLowerCase():
- result = await imageUtil.getImagesMatches(img1, img2, options);
+ try {
+ result = await getImagesMatches(img1, img2, options);
+ } catch (err) {
+ // might throw if no matches
+ result = {count: 0};
+ }
break;
case GET_SIMILARITY_MODE.toLowerCase():
- result = await imageUtil.getImagesSimilarity(img1, img2, options);
+ result = await getImagesSimilarity(img1, img2, options);
break;
case MATCH_TEMPLATE_MODE.toLowerCase():
// firstImage/img1 is the full image and secondImage/img2 is the partial one
- result = await imageUtil.getImageOccurrence(img1, img2, options);
+ result = await getImageOccurrence(img1, img2, options);
+ if (options.multiple) {
+ return result.multiple.map(convertVisualizationToBase64);
+ }
break;
default:
throw new errors.InvalidArgumentError(`'${mode}' images comparison mode is unknown. ` +
`Only ${JSON.stringify([MATCH_FEATURES_MODE, GET_SIMILARITY_MODE, MATCH_TEMPLATE_MODE])} modes are supported.`);
}
- if (!_.isEmpty(result.visualization)) {
- result.visualization = result.visualization.toString('base64');
+ return convertVisualizationToBase64(result);
}
- return result;
+
+/**
+ * base64 encodes the visualization part of the result
+ * (if necessary)
+ *
+ * @param {OccurrenceResult} element - occurrence result
+ *
+**/
+function convertVisualizationToBase64 (element) {
+ if (!_.isEmpty(element.visualization)) {
+ element.visualization = element.visualization.toString('base64');
+ }
+
+ return element;
}
export { compareImages, DEFAULT_MATCH_THRESHOLD, MATCH_TEMPLATE_MODE, MATCH_FEATURES_MODE,
GET_SIMILARITY_MODE };
+
+/**
+ * @typedef {import('@appium/opencv').OccurrenceResult} OccurrenceResult
+ */

MODIFY
packages/images-plugin/lib/finder.js
packages/images-plugin/lib/finder.js
+import _ from 'lodash';
import LRU from 'lru-cache';
import { imageUtil, util } from '@appium/support';
import { errors } from '@appium/base-driver';
@@ -19,6 +20,12 @@ const DEFAULT_SETTINGS = {
// element will not be found
imageMatchThreshold: DEFAULT_MATCH_THRESHOLD,
+ // One of possible image matching methods.
+ // Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html
+ // for more details.
+ // TM_CCOEFF_NORMED by default
+ imageMatchMethod: '',
+
// if the image returned by getScreenshot differs in size or aspect ratio
// from the screen, attempt to fix it automatically
fixImageFindScreenshotDims: true,
@@ -81,7 +88,7 @@ export default class ImageElementFinder {
}
/**
- * @typedef {Object} FindByImageOptions
+ * @typedef FindByImageOptions
* @property {boolean} [shouldCheckStaleness=false] - whether this call to find an
* image is merely to check staleness. If so we can bypass a lot of logic
* @property {boolean} [multiple=false] - Whether we are finding one element or
@@ -109,13 +116,14 @@ export default class ImageElementFinder {
if (!this.driver) {
throw new Error(`Can't find without a driver!`);
}
- const settings = Object.assign({}, DEFAULT_SETTINGS, this.driver.settings.getSettings());
+ const settings = {...DEFAULT_SETTINGS, ...this.driver.settings.getSettings()};
const {
imageMatchThreshold: threshold,
+ imageMatchMethod,
fixImageTemplateSize,
fixImageTemplateScale,
defaultImageTemplateScale,
- getMatchedImageResult: visualize,
+ getMatchedImageResult: visualize
} = settings;
log.info(`Finding image element with match threshold ${threshold}`);
@@ -133,9 +141,7 @@ export default class ImageElementFinder {
screenHeight);
}
- let rect = null;
- let b64Matched = null;
- let score = 0;
+ const results = [];
const condition = async () => {
try {
const {b64Screenshot, scale} = await this.getScreenshotForImageFind(screenWidth, screenHeight);
@@ -145,12 +151,27 @@ export default class ImageElementFinder {
fixImageTemplateScale, ...scale
});
- const comparedImage = await compareImages(MATCH_TEMPLATE_MODE, b64Screenshot, b64Template, {threshold, visualize});
- rect = comparedImage.rect;
- b64Matched = comparedImage.visualization;
- score = comparedImage.score;
-
+ const comparisonOpts = {
+ threshold,
+ visualize,
+ multiple,
+ };
+ if (imageMatchMethod) {
+ comparisonOpts.method = imageMatchMethod;
+ }
+ if (multiple) {
+ results.push(...(await compareImages(MATCH_TEMPLATE_MODE,
+ b64Screenshot,
+ b64Template,
+ comparisonOpts)));
+ } else {
+ results.push(await compareImages(MATCH_TEMPLATE_MODE,
+ b64Screenshot,
+ b64Template,
+ comparisonOpts));
+ }
return true;
+
} catch (err) {
// if compareImages fails, we'll get a specific error, but we should
// retry, so trap that and just return false to trigger the next round of
@@ -177,29 +198,28 @@ export default class ImageElementFinder {
}
}
- if (!rect) {
+ if (_.isEmpty(results)) {
if (multiple) {
return [];
}
throw new errors.NoSuchElementError();
}
+ const elements = results.map(({rect, score, visualization}) => {
log.info(`Image template matched: ${JSON.stringify(rect)}`);
- if (b64Matched) {
- log.info(`Matched base64 data: ${b64Matched.substring(0, 200)}...`);
- }
- const imgEl = new ImageElement(b64Template, rect, score, b64Matched, this);
+ return new ImageElement(b64Template, rect, score, visualization, this);
+ });
// if we're just checking staleness, return straightaway so we don't add
// a new element to the cache. shouldCheckStaleness does not support multiple
// elements, since it is a purely internal mechanism
if (shouldCheckStaleness) {
- return imgEl;
+ return elements[0];
}
- const protocolEl = this.registerImageElement(imgEl);
+ const registeredElements = elements.map((imgEl) => this.registerImageElement(imgEl));
- return multiple ? [protocolEl] : protocolEl;
+ return multiple ? registeredElements : registeredElements[0];
}
/**
@@ -229,11 +249,11 @@ export default class ImageElementFinder {
}
/**
- * @typedef {Object} Screenshot
+ * @typedef Screenshot
* @property {string} b64Screenshot - base64 based screenshot string
*/
/**
- * @typedef {Object} ScreenshotScale
+ * @typedef ScreenshotScale
* @property {float} xScale - Scale ratio for width
* @property {float} yScale - Scale ratio for height
*/
@@ -244,7 +264,7 @@ export default class ImageElementFinder {
* @param {int} screenWidth - width of screen
* @param {int} screenHeight - height of screen
*
- * @returns {Screenshot, ?ScreenshotScale} base64-encoded screenshot and ScreenshotScale
+ * @returns { {b64Screenshot: Screenshot, scale: ScreenshotScale? } } base64-encoded screenshot and ScreenshotScale
*/
async getScreenshotForImageFind (screenWidth, screenHeight) {
if (!this.driver.getScreenshot) {
@@ -352,7 +372,7 @@ export default class ImageElementFinder {
}
/**
- * @typedef {Object} ImageTemplateSettings
+ * @typedef ImageTemplateSettings
* @property {boolean} fixImageTemplateScale - fixImageTemplateScale in device-settings
* @property {float} defaultImageTemplateScale - defaultImageTemplateScale in device-settings
* @property {boolean} ignoreDefaultImageTemplateScale - Ignore defaultImageTemplateScale if it has true.

MODIFY
packages/images-plugin/lib/image-element.js
packages/images-plugin/lib/image-element.js
@@ -15,23 +15,15 @@ const IMAGE_TAP_STRATEGIES = [
const DEFAULT_TEMPLATE_IMAGE_SCALE = 1.0;
/**
- * @typedef {Object} Rect
- * @property {int} x - x-coordinate of top-left corner
- * @property {int} y - y-coordinate of top-left corner
- * @property {int} width - width of rect
- * @property {int} height - height of rect
+ * @typedef Dimension
+ * @property {number} width - width of rect
+ * @property {number} height - height of rect
*/
/**
- * @typedef {Object} Dimension
- * @property {int} width - width of rect
- * @property {int} height - height of rect
- */
-
-/**
- * @typedef {Object} Position
- * @property {int} x - x coordinate
- * @property {int} y - y coordinate
+ * @typedef Position
+ * @property {number} x - x coordinate
+ * @property {number} y - y coordinate
*/
/**
@@ -46,11 +38,9 @@ export default class ImageElement {
* @param {Rect} rect - bounds of matched image element
* @param {number} score The similarity score as a float number in range [0.0, 1.0].
* 1.0 is the highest score (means both images are totally equal).
- * @param {?string} b64Result - the base64-encoded image which has matched marks.
+ * @param {string?} b64Result - the base64-encoded image which has matched marks.
* Defaults to null.
- * @param {?ImageElementFinder} finder - the finder we can use to re-check stale elements
- *
- * @returns {ImageElement}
+ * @param {import('./finder').default?} finder - the finder we can use to re-check stale elements
*/
constructor (b64Template, rect, score, b64Result = null, finder = null) {
this.template = b64Template;
@@ -217,12 +207,12 @@ export default class ImageElement {
/**
* Handle various Appium commands that involve an image element
*
- * @param {BaseDriver} driver - the driver to use for commands
+ * @param {import('@appium/base-driver').BaseDriver} driver - the driver to use for commands
* @param {string} cmd - the name of the driver command
* @param {string} imgElId - the id of the ImageElement to work with
- * @param {Array} args - Rest of arguments for executeScripts
+ * @param {string[]} args - Rest of arguments for executeScripts
*
- * @returns {Object} - the result of running a command
+ * @returns {object} - the result of running a command
*/
static async execute (driver, imgEl, cmd, ...args) {
switch (cmd) {
@@ -258,3 +248,7 @@ export {
ImageElement, IMAGE_EL_TAP_STRATEGY_MJSONWP, IMAGE_EL_TAP_STRATEGY_W3C,
DEFAULT_TEMPLATE_IMAGE_SCALE, IMAGE_ELEMENT_PREFIX
};
+
+/**
+ * @typedef {import('@appium/types').Rect} Rect
+ */

MODIFY
packages/images-plugin/package.json
packages/images-plugin/package.json
"dependencies": {
"@appium/base-driver": "file:../base-driver",
"@appium/base-plugin": "file:../base-plugin",
+ "@appium/opencv": "file:../opencv",
"@appium/support": "file:../support",
"lru-cache": "6.0.0"
},

MODIFY
packages/images-plugin/test/e2e/plugin.e2e.spec.js
packages/images-plugin/test/e2e/plugin.e2e.spec.js
@@ -4,8 +4,9 @@ import { MATCH_FEATURES_MODE, GET_SIMILARITY_MODE } from '../../lib/compare';
import { TEST_IMG_1_B64, TEST_IMG_2_B64, APPSTORE_IMG_PATH } from '../fixtures';
import { e2eSetup } from '@appium/base-plugin/build/test/helpers';
-const THIS_PLUGIN_DIR = path.resolve(__dirname, '..', '..', '..');
-const APPIUM_HOME = path.resolve(__dirname, '..', '..', '..', 'local_appium_home');
+const THIS_PLUGIN_DIR = path.resolve(__dirname, '..', '..');
+const APPIUM_HOME = path.resolve(THIS_PLUGIN_DIR, 'local_appium_home');
+const FAKE_DRIVER_DIR = path.resolve(THIS_PLUGIN_DIR, '..', 'fake-driver');
const TEST_HOST = 'localhost';
const TEST_PORT = 4723;
const TEST_FAKE_APP = path.resolve(APPIUM_HOME, 'node_modules', '@appium', 'fake-driver', 'test', 'fixtures',
@@ -34,7 +35,7 @@ describe('ImageElementPlugin', function () {
e2eSetup({
before, after, server, port: TEST_PORT, host: TEST_HOST, appiumHome: APPIUM_HOME,
- driverName: 'fake', driverSource: 'npm', driverSpec: 'appium-fake-driver',
+ driverName: 'fake', driverSource: 'local', driverSpec: FAKE_DRIVER_DIR,
pluginName: 'images', pluginSource: 'local', pluginSpec: THIS_PLUGIN_DIR,
});

MODIFY
packages/images-plugin/test/unit/finder.spec.js
packages/images-plugin/test/unit/finder.spec.js
@@ -44,9 +44,9 @@ describe('finding elements by image', function () {
it('should not be able to find image element from any other element', async function () {
const d = new PluginDriver();
await d.findElementFromElement(IMAGE_STRATEGY, 'foo', 'elId')
- .should.eventually.be.rejectedWith(/Locator Strategy.+is not supported/);
+ .should.be.rejectedWith(/Locator Strategy.+is not supported/);
await d.findElementsFromElement(IMAGE_STRATEGY, 'foo', 'elId')
- .should.eventually.be.rejectedWith(/Locator Strategy.+is not supported/);
+ .should.be.rejectedWith(/Locator Strategy.+is not supported/);
});
});
@@ -79,7 +79,7 @@ describe('finding elements by image', function () {
beforeEach(function () {
d = new PluginDriver();
f = new ImageElementFinder(d);
- compareStub = sandbox.stub(compareModule, 'compareImages').returns({rect, score});
+ compareStub = sandbox.stub(compareModule, 'compareImages').resolves({rect, score});
basicStub(d, f);
});
@@ -88,6 +88,8 @@ describe('finding elements by image', function () {
basicImgElVerify(imgElProto, f);
});
it('should find image elements happypath', async function () {
+ compareStub.restore();
+ compareStub = sandbox.stub(compareModule, 'compareImages').resolves([{rect, score}]);
const els = await f.findByImage(template, {multiple: true});
els.should.have.length(1);
basicImgElVerify(els[0], f);

MODIFY
packages/images-plugin/test/unit/image-element.spec.js
packages/images-plugin/test/unit/image-element.spec.js
@@ -72,7 +72,7 @@ describe('ImageElement', function () {
const d = new BaseDriver();
const el = new ImageElement(defTemplate, defRect);
await d.settings.update({imageElementTapStrategy: 'bad'});
- await el.click(d).should.eventually.be.rejectedWith(/Incorrect imageElementTapStrategy/);
+ await el.click(d).should.be.rejectedWith(/Incorrect imageElementTapStrategy/);
});
it('should try to check for image element staleness, and throw if stale', async function () {
const d = new BaseDriver();
@@ -84,14 +84,14 @@ describe('ImageElement', function () {
checkForImageElementStaleness: true,
autoUpdateImageElementPosition: false
});
- await el.click(d).should.eventually.be.rejectedWith(/no longer attached/);
+ await el.click(d).should.be.rejectedWith(/no longer attached/);
// and also if we are updating the element position
await d.settings.update({
checkForImageElementStaleness: false,
autoUpdateImageElementPosition: true
});
- await el.click(d).should.eventually.be.rejectedWith(/no longer attached/);
+ await el.click(d).should.be.rejectedWith(/no longer attached/);
});
it('should auto-update element position if requested', async function () {
const d = new BaseDriver();
@@ -162,7 +162,7 @@ describe('ImageElement', function () {
await d.settings.update({
checkForImageElementStaleness: false,
});
- await el.click(d).should.eventually.be.rejectedWith(/did not implement/);
+ await el.click(d).should.be.rejectedWith(/did not implement/);
});
});
@@ -184,7 +184,7 @@ describe('ImageElement', function () {
it('should reject executions for unsupported commands', async function () {
await ImageElement.execute(driver, imgEl, 'foobar')
- .should.eventually.be.rejectedWith(/not yet been implemented/);
+ .should.be.rejectedWith(/not yet been implemented/);
});
it('should get displayed status of element', async function () {
await ImageElement.execute(driver, imgEl, 'elementDisplayed')

MODIFY
packages/images-plugin/test/unit/plugin.spec.js
packages/images-plugin/test/unit/plugin.spec.js
-let hasOpenCV = false;
-try {
- require('opencv4nodejs');
- hasOpenCV = true;
-} catch (ign) {}
-
import { ImageElementPlugin, IMAGE_STRATEGY } from '../../lib/plugin';
import { MATCH_FEATURES_MODE, GET_SIMILARITY_MODE, MATCH_TEMPLATE_MODE } from '../../lib/compare';
import BaseDriver from '@appium/base-driver';
import { W3C_ELEMENT_KEY } from '../../lib/finder';
import { TEST_IMG_1_B64, TEST_IMG_2_B64, TEST_IMG_2_PART_B64 } from '../fixtures';
-const describeOrSkip = hasOpenCV ? describe : describe.skip;
-
-describeOrSkip('ImageElementPlugin#handle', function () {
+describe('ImageElementPlugin#handle', function () {
const next = () => {};
const driver = new BaseDriver();
const p = new ImageElementPlugin();

MODIFY
packages/images-plugin/tsconfig.json
packages/images-plugin/tsconfig.json
"@appium/base-plugin": ["../base-plugin"],
"@appium/support": ["../support"],
"@appium/base-driver": ["../base-driver"],
+ "@appium/opencv": ["../opencv"],
+ "appium": ["../appium"]
}
},
"extends": "../../config/tsconfig.base.json",
},
{
"path": "../base-driver"
+ },
+ {
+ "path": "../opencv"
+ },
+ {
+ "path": "../appium"
}
]
}

fix(images-plugin): fix bad merge and tests

- adds/fixes a couple types; sets up type dependencies
- update e2e tests to pull fake driver & plugin from local
- fix sinon usage
- fix some assertions