diff --git a/src/drivers/npm/README.md b/src/drivers/npm/README.md index 3875fdbf9..c13078019 100644 --- a/src/drivers/npm/README.md +++ b/src/drivers/npm/README.md @@ -27,13 +27,14 @@ node index.js [url] [options] ### Options ``` - --debug=0|1 Output debug messages. - --delay=ms Wait for ms milliseconds between requests. - --max-depth=num Don't analyze pages more than num levels deep. - --max-urls=num Exit when num URLs have been analyzed. - --max-wait=ms Wait no more than ms milliseconds for page resources to load. - --recursive=0|1 Follow links on pages (crawler). - --user-agent=str Set the user agent string. + --debug=0|1 Output debug messages. + --delay=ms Wait for ms milliseconds between requests. + --max-depth=num Don't analyze pages more than num levels deep. + --max-urls=num Exit when num URLs have been analyzed. + --max-wait=ms Wait no more than ms milliseconds for page resources to load. + --recursive=0|1 Follow links on pages (crawler). + --request-timeout=ms Wait no more than ms millisecond for the page to load. + --user-agent=str Set the user agent string. ``` @@ -45,8 +46,9 @@ const options = { delay: 500, maxDepth: 3, maxUrls: 10, - maxWait: 3000, + maxWait: 1000, recursive: true, + requestTimeout: 3000, userAgent: 'Wappalyzer', }; diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 5cbf30242..d15961ac9 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -15,8 +15,9 @@ class Driver { delay: 500, maxDepth: 3, maxUrls: 10, - maxWait: 3000, + maxWait: 1000, recursive: false, + requestTimeout: 3000, userAgent: 'Mozilla/5.0 (compatible; Wappalyzer)', }, options || {}); @@ -41,6 +42,11 @@ class Driver { } analyze() { + this.time = { + start: new Date().getTime(), + last: new Date().getTime(), + } + return this.crawl(this.origPageUrl); } @@ -49,6 +55,8 @@ class Driver { } displayApps(detected) { + this.timer('displayApps'); + Object.keys(detected).forEach(appName => { const app = detected[appName]; @@ -76,6 +84,8 @@ class Driver { } fetch(pageUrl, index, depth) { + this.timer('fetch'); + return new Promise(resolve => { // Return when the URL is a duplicate or maxUrls has been reached if ( this.analyzedPageUrls.indexOf(pageUrl.href) !== -1 || this.analyzedPageUrls.length >= this.options.maxUrls ) { @@ -94,16 +104,22 @@ class Driver { this.sleep(this.options.delay * index) .then(() => { - browser.visit(pageUrl.href, error => { + this.timer('browser.visit start'); + + browser.visit(pageUrl.href, this.options.requestTimeout, error => { + this.timer('browser.visit end'); + if ( !browser.resources['0'] || !browser.resources['0'].response ) { this.wappalyzer.log('No response from server', 'browser', 'error'); return resolve(); } - browser.wait() + browser.wait(this.options.maxWait) .catch(error => this.wappalyzer.log(error.message, 'browser', 'error')) .finally(() => { + this.timer('browser.wait end'); + const headers = {}; browser.resources['0'].response.headers._headers.forEach(header => { @@ -145,6 +161,8 @@ class Driver { } crawl(pageUrl, index = 1, depth = 1) { + this.timer('crawl'); + return new Promise(resolve => { this.fetch(pageUrl, index, depth) .then(links => { @@ -158,13 +176,27 @@ class Driver { return Promise.resolve(); } }) - .then(() => resolve(this.apps)); + .then(() => { + this.timer('done'); + + resolve(this.apps) + }); }); } sleep(ms) { return ms ? new Promise(resolve => setTimeout(resolve, ms)) : Promise.resolve(); } + + timer(step) { + const time = new Date().getTime(); + const sinceStart = ( Math.round(( time - this.time.start ) / 10) / 100) + 's'; + const sinceLast = ( Math.round(( time - this.time.last ) / 10) / 100) + 's'; + + this.wappalyzer.log('[' + step + '] Time lapsed: ' + sinceLast + ' / ' + sinceStart, 'driver'); + + this.time.last = time; + } }; module.exports = Driver; diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index 4bf8607a4..7515906fa 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -2,7 +2,7 @@ "name": "wappalyzer", "description": "Uncovers the technologies used on websites", "homepage": "https://github.com/AliasIO/Wappalyzer", - "version": "5.1.6", + "version": "5.2.0", "author": "Elbert Alias", "license": "GPL-3.0", "repository": { diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json index 0abf8abdd..b46567b82 100644 --- a/src/drivers/webextension/manifest.json +++ b/src/drivers/webextension/manifest.json @@ -4,7 +4,7 @@ "author": "Elbert Alias", "homepage_url": "https://www.wappalyzer.com", "description": "Identify web technologies", - "version": "5.1.6", + "version": "5.2.0", "default_locale": "en", "manifest_version": 2, "icons": {