Merge branch 'master' of github.com:AliasIO/Wappalyzer

src/drivers/npm/README.md

@@ -27,13 +27,14 @@ node index.js [url] [options]
 ### Options
 
 ```
-  --debug=0|1        Output debug messages.
+  --debug=0|1             Output debug messages.
-  --delay=ms         Wait for ms milliseconds between requests.
+  --delay=ms              Wait for ms milliseconds between requests.
-  --max-depth=num    Don't analyze pages more than num levels deep.
+  --max-depth=num         Don't analyze pages more than num levels deep.
-  --max-urls=num     Exit when num URLs have been analyzed.
+  --max-urls=num          Exit when num URLs have been analyzed.
-  --max-wait=ms      Wait no more than ms milliseconds for page resources to load.
+  --max-wait=ms           Wait no more than ms milliseconds for page resources to load.
-  --recursive=0|1    Follow links on pages (crawler).
+  --recursive=0|1         Follow links on pages (crawler).
-  --user-agent=str   Set the user agent string.
+  --request-timeout=ms    Wait no more than ms millisecond for the page to load.
+  --user-agent=str        Set the user agent string.
 ```
 
 
@@ -45,8 +46,9 @@ const options = {
   delay: 500,
   maxDepth: 3,
   maxUrls: 10,
-  maxWait: 3000,
+  maxWait: 1000,
   recursive: true,
+  requestTimeout: 3000,
   userAgent: 'Wappalyzer',
 };
 

src/drivers/npm/driver.js

@@ -15,8 +15,9 @@ class Driver {
       delay: 500,
       maxDepth: 3,
       maxUrls: 10,
-      maxWait: 3000,
+      maxWait: 1000,
       recursive: false,
+      requestTimeout: 3000,
       userAgent: 'Mozilla/5.0 (compatible; Wappalyzer)',
     }, options || {});
 
@@ -41,6 +42,11 @@ class Driver {
   }
 
   analyze() {
+    this.time = {
+      start: new Date().getTime(),
+      last: new Date().getTime(),
+    }
+
     return this.crawl(this.origPageUrl);
   }
 
@@ -49,6 +55,8 @@ class Driver {
   }
 
   displayApps(detected) {
+    this.timer('displayApps');
+
     Object.keys(detected).forEach(appName => {
       const app = detected[appName];
 
@@ -76,6 +84,8 @@ class Driver {
   }
 
   fetch(pageUrl, index, depth) {
+    this.timer('fetch');
+
     return new Promise(resolve => {
       // Return when the URL is a duplicate or maxUrls has been reached
       if ( this.analyzedPageUrls.indexOf(pageUrl.href) !== -1 || this.analyzedPageUrls.length >= this.options.maxUrls ) {
@@ -94,16 +104,22 @@ class Driver {
 
       this.sleep(this.options.delay * index)
         .then(() => {
-          browser.visit(pageUrl.href, error => {
+          this.timer('browser.visit start');
+
+          browser.visit(pageUrl.href, this.options.requestTimeout, error => {
+            this.timer('browser.visit end');
+
             if ( !browser.resources['0'] || !browser.resources['0'].response ) {
               this.wappalyzer.log('No response from server', 'browser', 'error');
 
               return resolve();
             }
 
-            browser.wait()
+            browser.wait(this.options.maxWait)
               .catch(error => this.wappalyzer.log(error.message, 'browser', 'error'))
               .finally(() => {
+                this.timer('browser.wait end');
+
                 const headers = {};
 
                 browser.resources['0'].response.headers._headers.forEach(header => {
@@ -145,6 +161,8 @@ class Driver {
   }
 
   crawl(pageUrl, index = 1, depth = 1) {
+    this.timer('crawl');
+
     return new Promise(resolve => {
       this.fetch(pageUrl, index, depth)
         .then(links => {
@@ -158,13 +176,27 @@ class Driver {
             return Promise.resolve();
           }
         })
-        .then(() => resolve(this.apps));
+        .then(() => {
+          this.timer('done');
+
+          resolve(this.apps)
+        });
     });
   }
 
   sleep(ms) {
     return ms ? new Promise(resolve => setTimeout(resolve, ms)) : Promise.resolve();
   }
+
+  timer(step) {
+    const time = new Date().getTime();
+    const sinceStart = ( Math.round(( time - this.time.start ) / 10) / 100) + 's';
+    const sinceLast = ( Math.round(( time - this.time.last ) / 10) / 100) + 's';
+
+    this.wappalyzer.log('[' + step + '] Time lapsed: ' + sinceLast + ' / ' + sinceStart, 'driver');
+
+    this.time.last = time;
+  }
 };
 
 module.exports = Driver;

src/drivers/npm/package.json

@@ -2,7 +2,7 @@
   "name": "wappalyzer",
   "description": "Uncovers the technologies used on websites",
   "homepage": "https://github.com/AliasIO/Wappalyzer",
-  "version": "5.1.6",
+  "version": "5.2.0",
   "author": "Elbert Alias",
   "license": "GPL-3.0",
   "repository": {

src/drivers/webextension/manifest.json

@@ -4,7 +4,7 @@
 	"author": "Elbert Alias",
 	"homepage_url": "https://www.wappalyzer.com",
 	"description": "Identify web technologies",
-	"version": "5.1.6",
+	"version": "5.2.0",
 	"default_locale": "en",
 	"manifest_version": 2,
 	"icons": {