Properly synchronize applications caching
MODIFY
lib/basedriver/helpers.js
lib/basedriver/helpers.js
@@ -8,12 +8,19 @@ import { tempDir, fs, util, zip } from 'appium-support';
import request from 'request';
import asyncRequest from 'request-promise';
import LRU from 'lru-cache';
+import AsyncLock from 'async-lock';
+import sanitize from 'sanitize-filename';
const ZIP_EXTS = ['.zip', '.ipa'];
-const ZIP_MIME_TYPE = 'application/zip';
-const DOWNLOADS_CACHE = new LRU({
+const ZIP_MIME_TYPES = [
+ 'application/zip',
+ 'application/x-zip-compressed',
+ 'multipart/x-zip',
+];
+const APPLICATIONS_CACHE = new LRU({
max: 100,
});
+const APPLICATIONS_CACHE_GUARD = new AsyncLock();
async function retrieveHeaders (link) {
try {
@@ -31,20 +38,27 @@ async function retrieveHeaders (link) {
}
function getCachedApplicationPath (link, currentModified) {
- if (!DOWNLOADS_CACHE.has(link) || !currentModified) {
+ if (!APPLICATIONS_CACHE.has(link) || !currentModified) {
return null;
}
- const {lastModified, fullPath} = DOWNLOADS_CACHE.get(link);
- if (currentModified.getTime() <= lastModified.getTime()) {
+ const {lastModified, fullPath} = APPLICATIONS_CACHE.get(link);
+ if (lastModified && currentModified.getTime() <= lastModified.getTime()) {
logger.debug(`Reusing already downloaded application at '${fullPath}'`);
return fullPath;
}
logger.debug(`'Last-Modified' timestamp of '${link}' has been updated. ` +
- `The updated copy of the application is going to be downloaded.`);
+ `An updated copy of the application is going to be downloaded.`);
return null;
}
+function verifyAppExtension (app, supportedAppExtensions) {
+ if (supportedAppExtensions.includes(path.extname(app))) {
+ return app;
+ }
+ throw new Error(`New app path '${app}' did not have extension(s) '${supportedAppExtensions}'`);
+}
+
async function configureApp (app, supportedAppExtensions) {
if (!_.isString(app)) {
// immediately shortcircuit if not given an app
@@ -56,10 +70,12 @@ async function configureApp (app, supportedAppExtensions) {
let newApp = app;
let shouldUnzipApp = false;
+ let archiveHash = null;
+ let currentModified = null;
const {protocol} = url.parse(newApp);
const isUrl = ['http:', 'https:'].includes(protocol);
- let currentModified = null;
+ return await APPLICATIONS_CACHE_GUARD.acquire(app, async () => {
if (isUrl) {
// Use the app from remote URL
logger.info(`Using downloadable app '${newApp}'`);
@@ -68,20 +84,20 @@ async function configureApp (app, supportedAppExtensions) {
logger.debug(`Last-Modified: ${headers['last-modified']}`);
currentModified = new Date(headers['last-modified']);
}
- const cachedPath = getCachedApplicationPath(newApp, currentModified);
+ const cachedPath = getCachedApplicationPath(app, currentModified);
if (cachedPath) {
if (await fs.exists(cachedPath)) {
logger.info(`Reusing the previously downloaded application at '${cachedPath}'`);
- return cachedPath;
+ return verifyAppExtension(cachedPath, supportedAppExtensions);
}
logger.info(`The application at '${cachedPath}' does not exist anymore. Deleting it from the cache`);
- DOWNLOADS_CACHE.del(newApp);
+ APPLICATIONS_CACHE.del(app);
}
let fileName = `appium-app${_.first(supportedAppExtensions)}`;
if (headers['content-type']) {
logger.debug(`Content-Type: ${headers['content-type']}`);
// the filetype may not be obvious for certain urls, so check the mime type too
- if (headers['content-type'] === ZIP_MIME_TYPE) {
+ if (ZIP_MIME_TYPES.includes(headers['content-type'])) {
fileName = 'appium-app.zip';
shouldUnzipApp = true;
}
@@ -89,9 +105,9 @@ async function configureApp (app, supportedAppExtensions) {
if (headers['content-disposition'] && /^attachment/i.test(headers['content-disposition'])) {
const match = /filename="([^"]+)/i.exec(headers['content-disposition']);
if (match) {
- fileName = match[1];
- logger.debug(`Parsed file name '${fileName}' from 'Content-Disposition' header`);
- shouldUnzipApp = shouldUnzipApp || _.includes(ZIP_EXTS, path.extname(fileName));
+ logger.debug(`Parsed file name '${match[1]}' from 'Content-Disposition' header`);
+ fileName = sanitize(match[1], {replacement: '-'});
+ shouldUnzipApp = shouldUnzipApp || ZIP_EXTS.includes(path.extname(fileName));
}
}
const targetPath = await tempDir.path({
@@ -102,7 +118,7 @@ async function configureApp (app, supportedAppExtensions) {
} else if (await fs.exists(newApp)) {
// Use the local app
logger.info(`Using local app '${newApp}'`);
- shouldUnzipApp = _.includes(ZIP_EXTS, path.extname(newApp));
+ shouldUnzipApp = ZIP_EXTS.includes(path.extname(newApp));
} else {
let errorMessage = `The application at '${newApp}' does not exist or is not accessible`;
// protocol value for 'C:\\temp' is 'c:', so we check the length as well
@@ -114,8 +130,21 @@ async function configureApp (app, supportedAppExtensions) {
}
if (shouldUnzipApp) {
- const tmpRoot = await tempDir.openDir();
const archivePath = newApp;
+ archiveHash = await fs.hash(archivePath);
+ if (APPLICATIONS_CACHE.has(app) && archiveHash === APPLICATIONS_CACHE.get(app).hash) {
+ const {fullPath} = APPLICATIONS_CACHE.get(app);
+ if (await fs.exists(fullPath)) {
+ if (archivePath !== app) {
+ await fs.rimraf(archivePath);
+ }
+ logger.info(`Will reuse previously cached application at '${fullPath}'`);
+ return verifyAppExtension(fullPath, supportedAppExtensions);
+ }
+ logger.info(`The application at '${fullPath}' does not exist anymore. Deleting it from the cache`);
+ APPLICATIONS_CACHE.del(app);
+ }
+ const tmpRoot = await tempDir.openDir();
try {
newApp = await unzipApp(archivePath, tmpRoot, supportedAppExtensions);
} finally {
@@ -126,21 +155,17 @@ async function configureApp (app, supportedAppExtensions) {
logger.info(`Unzipped local app to '${newApp}'`);
}
- if (!supportedAppExtensions.includes(path.extname(newApp))) {
- if (newApp !== app) {
- await fs.rimraf(newApp);
- }
- throw new Error(`New app path '${newApp}' did not have extension(s) '${supportedAppExtensions}'`);
- }
+ verifyAppExtension(newApp, supportedAppExtensions);
- if (isUrl && currentModified) {
- DOWNLOADS_CACHE.set(url, {
+ if (app !== newApp && (archiveHash || currentModified)) {
+ APPLICATIONS_CACHE.set(app, {
+ hash: archiveHash,
lastModified: currentModified,
fullPath: newApp,
});
}
-
return newApp;
+ });
}
async function downloadApp (app, targetPath) {
@@ -152,6 +177,7 @@ async function downloadApp (app, targetPath) {
}
try {
+ const started = process.hrtime();
// don't use request-promise here, we need streams
await new B((resolve, reject) => {
request(appUrl.href)
@@ -165,7 +191,15 @@ async function downloadApp (app, targetPath) {
.pipe(_fs.createWriteStream(targetPath))
.on('close', resolve);
});
- logger.debug(`'${appUrl.href}' has been downloaded to '${targetPath}'`);
+ const [seconds, ns] = process.hrtime(started);
+ const secondsElapsed = seconds + ns / 1E09;
+ const {size} = await fs.stat(targetPath);
+ logger.debug(`'${appUrl.href}' (${util.toReadableSizeString(size)}) ` +
+ `has been downloaded to '${targetPath}' in ${secondsElapsed.toFixed(3)}s`);
+ if (secondsElapsed >= 2) {
+ const bytesPerSec = Math.floor(size / secondsElapsed);
+ logger.debug(`Approximate download speed: ${util.toReadableSizeString(bytesPerSec)}/s`);
+ }
return targetPath;
} catch (err) {
throw new Error(`Problem downloading app from url ${appUrl.href}: ${err.message}`);

MODIFY
package.json
package.json
"dependencies": {
"@babel/runtime": "^7.0.0",
"appium-support": "^2.24.0",
+ "async-lock": "^1.0.0",
"asyncbox": "^2.3.1",
"bluebird": "^2.10.2",
"body-parser": "^1.18.2",
"morgan": "^1.9.0",
"request": "^2.83.0",
"request-promise": "^4.2.2",
+ "sanitize-filename": "^1.6.1",
"serve-favicon": "^2.4.5",
"source-map-support": "^0.5.5",
"teen_process": "^1.11.0",

MODIFY
test/basedriver/helpers-e2e-specs.js
test/basedriver/helpers-e2e-specs.js
@@ -53,7 +53,7 @@ describe('app download and configuration', function () {
});
it('should fail if zip file does not contain an app whose extension matches', async function () {
await configureApp(getFixture('FakeIOSApp.app.zip'), '.wrong')
- .should.be.rejectedWith(/could not find/);
+ .should.be.rejectedWith(/did not have extension/);
});
describe('should download an app from the web', function () {
const port = 8000;

MODIFY
test/express/server-e2e-specs.js
test/express/server-e2e-specs.js
@@ -70,7 +70,7 @@ describe('server', function () {
let before = Date.now();
await hwServer.close();
// expect slightly less than the request waited, since we paused above
- (Date.now() - before).should.be.above(900);
+ (Date.now() - before).should.be.above(800);
(await bodyPromise).should.equal('We have waited!');
});

Properly synchronize applications caching (#274)