add getting result image by find by image (#327)

* add getMatchedImageResult and return the result if it is true

* add tests

* tweak error case

lib/basedriver/commands/find.js

@@ -210,7 +210,8 @@ helpers.findByImage = async function findByImage (b64Template, {
     imageMatchThreshold: threshold,
     fixImageTemplateSize,
     fixImageTemplateScale,
-    defaultImageTemplateScale
+    defaultImageTemplateScale,
+    getMatchedImageResult: visualize
   } = this.settings.getSettings();
 
   log.info(`Finding image element with match threshold ${threshold}`);
@@ -229,6 +230,7 @@ helpers.findByImage = async function findByImage (b64Template, {
   }
 
   let rect = null;
+  let b64Matched = null;
   const condition = async () => {
     try {
       const {b64Screenshot, scale} = await this.getScreenshotForImageFind(screenWidth, screenHeight);
@@ -238,8 +240,10 @@ helpers.findByImage = async function findByImage (b64Template, {
         fixImageTemplateScale, ...scale
       });
 
-      rect = (await this.compareImages(MATCH_TEMPLATE_MODE, b64Screenshot,
-        b64Template, {threshold})).rect;
+      const comparedImage = await this.compareImages(MATCH_TEMPLATE_MODE, b64Screenshot, b64Template, {threshold, visualize});
+      rect = comparedImage.rect;
+      b64Matched = comparedImage.visualization;
+
       return true;
     } catch (err) {
       // if compareImages fails, we'll get a specific error, but we should
@@ -275,7 +279,10 @@ helpers.findByImage = async function findByImage (b64Template, {
   }
 
   log.info(`Image template matched: ${JSON.stringify(rect)}`);
-  const imgEl = new ImageElement(b64Template, rect);
+  if (b64Matched) {
+    log.info(`Matched base64 data: ${b64Matched.substring(0, 200)}...`);
+  }
+  const imgEl = new ImageElement(b64Template, rect, b64Matched);
 
   // if we're just checking staleness, return straightaway so we don't add
   // a new element to the cache. shouldCheckStaleness does not support multiple

lib/basedriver/device-settings.js

@@ -44,6 +44,10 @@ const GLOBAL_DEFAULT_SETTINGS = {
   // which method to use for tapping by coordinate for image elements. the
   // options are 'w3c' or 'mjsonwp'
   imageElementTapStrategy: IMAGE_EL_TAP_STRATEGY_W3C,
+
+  // which method to use to save the matched image area in ImageElement class.
+  // It is used for debugging purpose.
+  getMatchedImageResult: false,
 };
 
 // declare settings that are really handled only by basedriver, so downstream
@@ -57,6 +61,7 @@ const BASEDRIVER_HANDLED_SETTINGS = [
   'checkForImageElementStaleness',
   'autoUpdateImageElementPosition',
   'imageElementTapStrategy',
+  'getMatchedImageResult',
 ];
 
 class DeviceSettings {

lib/basedriver/driver.js

@@ -302,7 +302,7 @@ class BaseDriver extends Protocol {
 
           // if one of the args is an image element, handle it separately
           return B.race([
-            imgElId ? ImageElement.execute(this, cmd, imgElId) : this[cmd](...args),
+            imgElId ? ImageElement.execute(this, cmd, imgElId, ...args) : this[cmd](...args),
             cancelPromise,
           ]);
         });

lib/basedriver/image-element.js

@@ -43,15 +43,18 @@ class ImageElement {
 
   /**
    * @param {string} b64Template - the base64-encoded image which was used to
-   * find this ImageElement
+   *                               find this ImageElement
    * @param {Rect} rect - bounds of matched image element
+   * @param {?string} b64Result - the base64-encoded image which has matched marks.
+   *                              Defaults to null.
    *
    * @returns {ImageElement}
    */
-  constructor (b64Template, rect) {
+  constructor (b64Template, rect, b64Result = null) {
     this.template = b64Template;
     this.rect = rect;
     this.id = `${IMAGE_ELEMENT_PREFIX}${UUID.create().hex}`;
+    this.b64MatchedImage = b64Result;
   }
 
   /**
@@ -78,6 +81,13 @@ class ImageElement {
     };
   }
 
+  /**
+   * @returns {?string} - the base64-encoded image which has matched marks
+   */
+  get matchedImage () {
+    return this.b64MatchedImage;
+  }
+
   /**
    * @param {string} protocolKey - the protocol-specific JSON key for
    * a WebElement
@@ -205,10 +215,11 @@ class ImageElement {
    * @param {BaseDriver} driver - the driver to use for commands
    * @param {string} cmd - the name of the driver command
    * @param {string} imgElId - the id of the ImageElement to work with
+   * @param {Array} args - Rest of arguments for executeScripts
    *
    * @returns {Object} - the result of running a command
    */
-  static async execute (driver, cmd, imgElId) {
+  static async execute (driver, cmd, imgElId, ...args) {
     if (!driver._imgElCache.has(imgElId)) {
       throw new errors.NoSuchElementError();
     }
@@ -227,6 +238,15 @@ class ImageElement {
         return imgEl.location;
       case 'getElementRect':
         return imgEl.rect;
+      case 'getAttribute':
+        // /wd/hub/session/:sessionId/element/:elementId/attribute/:name
+        // /wd/hub/session/:sessionId/element/:elementId/attribute/visual should retun the visual data
+        // e.g. ["content-desc","appium-image-element-xxxxx","xxxxx"], ["visual","appium-image-element-xxxxx","xxxxx"]
+        if (args[0] === 'visual') {
+          return imgEl.matchedImage;
+        } else {
+          throw new errors.NotYetImplementedError();
+        }
       default: throw new errors.NotYetImplementedError();
     }
   }

test/basedriver/image-element-specs.js

@@ -161,7 +161,8 @@ describe('ImageElement', function () {
   });
 
   describe('#execute', function () {
-    const imgEl = new ImageElement(defTemplate, defRect);
+    // aGFwcHkgdGVzdGluZw== is 'happy testing'
+    const imgEl = new ImageElement(defTemplate, defRect, 'aGFwcHkgdGVzdGluZw==');
     const clickStub = sinon.stub(imgEl, 'click');
 
     before(function () {
@@ -202,6 +203,20 @@ describe('ImageElement', function () {
       await ImageElement.execute(driver, 'getElementRect', imgEl.id)
         .should.eventually.eql(defRect);
     });
+    it('should get visual of element', async function () {
+      await ImageElement.execute(driver, 'getAttribute', imgEl.id, 'visual')
+        .should.eventually.eql('aGFwcHkgdGVzdGluZw==');
+    });
+    it('should get null as visual of element by default', async function () {
+      const imgElement = new ImageElement(defTemplate, defRect);
+      driver._imgElCache.set(imgElement.id, imgElement);
+      await ImageElement.execute(driver, 'getAttribute', imgElement.id, 'visual')
+        .should.eventually.eql(null);
+    });
+    it('should not get other attribute', async function () {
+      await ImageElement.execute(driver, 'getAttribute', imgEl.id, 'content-desc')
+        .should.eventually.rejectedWith('Method has not yet been implemented');
+    });
     it('should click element', async function () {
       await ImageElement.execute(driver, 'click', imgEl.id)
         .should.eventually.be.true;