Build 5.4.3, NPM performance improvements, removed requestTimeout option

src/drivers/npm/README.md

@@ -33,7 +33,6 @@ node index.js [url] [options]
   --max-urls=num     Exit when num URLs have been analyzed.
   --max-wait=ms      Wait no more than ms milliseconds for page resources to load.
   --recursive=0|1    Follow links on pages (crawler).
-  --request-timeout=ms    Wait no more than ms millisecond for the page to load.
   --user-agent=str   Set the user agent string.
 ```
 
@@ -46,9 +45,8 @@ const options = {
   delay: 500,
   maxDepth: 3,
   maxUrls: 10,
-  maxWait: 1000,
+  maxWait: 5000,
   recursive: true,
-  requestTimeout: 3000,
   userAgent: 'Wappalyzer',
 };
 

src/drivers/npm/driver.js

@@ -17,9 +17,8 @@ class Driver {
       delay: 500,
       maxDepth: 3,
       maxUrls: 10,
-      maxWait: 1000,
+      maxWait: 5000,
       recursive: false,
-      requestTimeout: 3000,
       userAgent: 'Mozilla/5.0 (compatible; Wappalyzer)',
     }, options || {});
 
@@ -29,7 +28,6 @@ class Driver {
     this.options.maxUrls = parseInt(this.options.maxUrls, 10);
     this.options.maxWait = parseInt(this.options.maxWait, 10);
     this.options.recursive = Boolean(this.options.recursive);
-    this.options.requestTimeout = parseInt(this.options.requestTimeout, 10);
 
     this.origPageUrl = url.parse(pageUrl);
     this.analyzedPageUrls = [];
@@ -61,8 +59,6 @@ class Driver {
   }
 
   displayApps(detected, meta) {
-    this.timer('displayApps');
-
     this.meta = meta;
 
     Object.keys(detected).forEach(appName => {
@@ -92,36 +88,73 @@ class Driver {
   }
 
   fetch(pageUrl, index, depth) {
-    return new Promise(resolve => {
     // Return when the URL is a duplicate or maxUrls has been reached
     if ( this.analyzedPageUrls.indexOf(pageUrl.href) !== -1 || this.analyzedPageUrls.length >= this.options.maxUrls ) {
-        return resolve();
+      return Promise.resolve();
     }
 
-      this.timer('fetch url: ' + pageUrl.href + '; depth: ' + depth + '; delay: ' + ( this.options.delay * index ) + 'ms');
-
     this.analyzedPageUrls.push(pageUrl.href);
 
+    const timerScope = {
+      last: new Date().getTime()
+    };
+
+    this.timer('fetch; url: ' + pageUrl.href + '; depth: ' + depth + '; delay: ' + ( this.options.delay * index ) + 'ms', timerScope);
+
+    return new Promise(resolve => this.sleep(this.options.delay * index).then(() => this.visit(pageUrl, timerScope, resolve)));
+  }
+
+  visit(pageUrl, timerScope, resolve) {
     const browser = new Browser({
       silent: true,
       userAgent: this.options.userAgent,
       waitDuration: this.options.maxWait,
     });
 
-      this.sleep(this.options.delay * index)
+    this.timer('browser.visit start; url: ' + pageUrl.href, timerScope);
-        .then(() => {
-          this.timer('browser.visit start url: ' + pageUrl.href);
 
-          browser.visit(pageUrl.href, this.options.requestTimeout, error => {
+    browser.visit(pageUrl.href, () => {
-            this.timer('browser.visit end url: ' + pageUrl.href);
+      this.timer('browser.visit end; url: ' + pageUrl.href, timerScope);
 
-            pageUrl.canonical = pageUrl.protocol + '//' + pageUrl.host + pageUrl.pathname;
+      if ( !this.responseOk(browser, pageUrl) ) {
+        return resolve();
+      }
+
+      const headers = this.getHeaders(browser);
+      const html = this.getHtml(browser);
+      const scripts = this.getScripts(browser);
+      const js = this.getJs(browser);
+
+      this.wappalyzer.analyze(pageUrl, {
+        headers,
+        html,
+        scripts,
+        js
+      });
+
+      const links = Array.from(browser.document.getElementsByTagName('a'))
+        .filter(link => link.hostname === this.origPageUrl.hostname)
+        .filter(link => extensions.test(link.pathname))
+        .map(link => { link.hash = ''; return url.parse(link.href) });
+
+      return resolve(links);
+    });
+  }
 
+  responseOk(browser, pageUrl) {
     // Validate response
-            if ( !browser.resources['0'] || !browser.resources['0'].response ) {
+    const resource = browser.resources.length ? browser.resources.filter(resource => resource.response).shift() : null;
-              this.wappalyzer.log('No response from server', 'browser', 'error');
 
-              return resolve();
+    if ( !resource ) {
+      this.wappalyzer.log('No response from server; url: ' + pageUrl.href, 'driver', 'error');
+
+      return false;
+    }
+
+    if ( resource.response.status !== 200 ) {
+      this.wappalyzer.log('Response was not OK; status: ' + resource.response.status + ' ' + resource.response.statusText + '; url: ' + pageUrl.href, 'driver', 'error');
+
+      return false;
     }
 
     const headers = this.getHeaders(browser);
@@ -130,57 +163,37 @@ class Driver {
     const contentType = headers.hasOwnProperty('content-type') ? headers['content-type'].shift() : null;
 
     if ( !contentType || !/\btext\/html\b/.test(contentType) ) {
-              this.wappalyzer.log('Skipping ' + pageUrl.href + ' of content type ' + contentType, 'driver');
+      this.wappalyzer.log('Skipping; url: ' + pageUrl.href + '; content type: ' + contentType, 'driver');
 
       this.analyzedPageUrls.splice(this.analyzedPageUrls.indexOf(pageUrl.href), 1);
 
-              return resolve();
+      return false;
     }
 
-            // Validate document element
+    // Validate document
     if ( !browser.document || !browser.document.documentElement ) {
-              this.wappalyzer.log('No HTML document at ' + pageUrl.href, 'driver', 'error');
+      this.wappalyzer.log('No HTML document; url: ' + pageUrl.href, 'driver', 'error');
 
-              return resolve();
+      return false;
     }
 
-            const html = this.getHtml(browser);
+    return true;
-            const scripts = this.getScripts(browser);
-
-            const links = Array.from(browser.document.getElementsByTagName('a'))
-              .filter(link => link.hostname === this.origPageUrl.hostname)
-              .filter(link => extensions.test(link.pathname))
-              .map(link => { link.hash = ''; return url.parse(link.href) });
-
-            browser.wait(this.options.maxWait, () => {
-              this.timer('browser.wait end url: ' + pageUrl.href);
-
-              const js = this.getJs(browser);
-
-              this.wappalyzer.analyze(pageUrl, {
-                headers,
-                html,
-                scripts,
-                js
-              });
-
-              return resolve(links);
-            });
-          });
-        });
-    });
   }
 
   getHeaders(browser) {
     const headers = {};
 
-    browser.resources['0'].response.headers._headers.forEach(header => {
+    const resource = browser.resources.length ? browser.resources.filter(resource => resource.response).shift() : null;
+
+    if ( resource ) {
+      resource.response.headers._headers.forEach(header => {
         if ( !headers[header[0]] ){
           headers[header[0]] = [];
         }
 
         headers[header[0]].push(header[1]);
       });
+    }
 
     return headers;
   }
@@ -244,6 +257,8 @@ class Driver {
   }
 
   crawl(pageUrl, index = 1, depth = 1) {
+    pageUrl.canonical = pageUrl.protocol + '//' + pageUrl.host + pageUrl.pathname;
+
     return new Promise(resolve => {
       this.fetch(pageUrl, index, depth)
         .then(links => {
@@ -267,14 +282,14 @@ class Driver {
     return ms ? new Promise(resolve => setTimeout(resolve, ms)) : Promise.resolve();
   }
 
-  timer(step) {
+  timer(message, scope) {
     const time = new Date().getTime();
     const sinceStart = ( Math.round(( time - this.time.start ) / 10) / 100) + 's';
-    const sinceLast = ( Math.round(( time - this.time.last ) / 10) / 100) + 's';
+    const sinceLast = ( Math.round(( time - scope.last ) / 10) / 100) + 's';
 
-    this.wappalyzer.log('[' + step + '] Time lapsed: ' + sinceLast + ' / ' + sinceStart, 'driver');
+    this.wappalyzer.log('[timer] ' + message + '; lapsed: ' + sinceLast + ' / ' + sinceStart, 'driver');
 
-    this.time.last = time;
+    scope.last = time;
   }
 };
 

src/drivers/npm/package.json

@@ -2,7 +2,7 @@
   "name": "wappalyzer",
   "description": "Uncovers the technologies used on websites",
   "homepage": "https://github.com/AliasIO/Wappalyzer",
-  "version": "5.4.2",
+  "version": "5.4.3",
   "author": "Elbert Alias",
   "license": "GPL-3.0",
   "repository": {

src/drivers/webextension/manifest.json

@@ -4,7 +4,7 @@
 	"author": "Elbert Alias",
 	"homepage_url": "https://www.wappalyzer.com",
 	"description": "Identify web technologies",
-	"version": "5.4.2",
+	"version": "5.4.3",
 	"default_locale": "en",
 	"manifest_version": 2,
 	"icons": {