Include error messaging per URL in NPM driver response, remove timeout

src/drivers/npm/driver.js

@@ -10,6 +10,12 @@ const json = JSON.parse(fs.readFileSync(path.resolve(__dirname + '/apps.json')))
 
 const extensions = /^([^.]+$|\.(asp|aspx|cgi|htm|html|jsp|php)$)/;
 
+const errorTypes = {
+	RESPONSE_NOT_OK: 'Response was not ok',
+	NO_RESPONSE: 'No response from server',
+	NO_HTML_DOCUMENT: 'No HTML document',
+};
+
 class Driver {
   constructor(pageUrl, options) {
     this.options = Object.assign({}, {
@@ -38,7 +44,7 @@ class Driver {
     this.options.htmlMaxRows = parseInt(this.options.htmlMaxRows, 10);
 
     this.origPageUrl = url.parse(pageUrl);
-    this.analyzedPageUrls = [];
+    this.analyzedPageUrls = {};
     this.apps = [];
     this.meta = {};
 
@@ -99,11 +105,13 @@ class Driver {
 
   fetch(pageUrl, index, depth) {
     // Return when the URL is a duplicate or maxUrls has been reached
-    if ( this.analyzedPageUrls.indexOf(pageUrl.href) !== -1 || this.analyzedPageUrls.length >= this.options.maxUrls ) {
+    if (this.analyzedPageUrls[pageUrl.href] || this.analyzedPageUrls.length >= this.options.maxUrls) {
       return Promise.resolve();
     }
 
-    this.analyzedPageUrls.push(pageUrl.href);
+    this.analyzedPageUrls[pageUrl.href] = {
+			status: 0,
+		};
 
     const timerScope = {
       last: new Date().getTime()
@@ -111,10 +119,10 @@ class Driver {
 
     this.timer('fetch; url: ' + pageUrl.href + '; depth: ' + depth + '; delay: ' + (this.options.delay * index) + 'ms', timerScope);
 
-    return new Promise(resolve => this.sleep(this.options.delay * index).then(() => this.visit(pageUrl, timerScope, resolve)));
+    return new Promise((resolve, reject) => this.sleep(this.options.delay * index).then(() => this.visit(pageUrl, timerScope, resolve, reject)));
   }
 
-  visit(pageUrl, timerScope, resolve) {
+  visit(pageUrl, timerScope, resolve, reject) {
     const browser = new Browser({
 			proxy: this.options.proxy,
       silent: true,
@@ -133,9 +141,13 @@ class Driver {
     browser.visit(pageUrl.href, () => {
       this.timer('browser.visit end; url: ' + pageUrl.href, timerScope);
 
-      if ( !this.responseOk(browser, pageUrl) ) {
+			try {
+				if (!this.checkResponse(browser, pageUrl)) {
 					return resolve();
 				}
+			} catch(error) {
+				return reject(error);
+			}
 
       const headers = this.getHeaders(browser);
       const html = this.getHtml(browser);
@@ -168,20 +180,18 @@ class Driver {
     });
   }
 
-  responseOk(browser, pageUrl) {
+  checkResponse(browser, pageUrl) {
     // Validate response
     const resource = browser.resources.length ? browser.resources.filter(resource => resource.response).shift() : null;
 
     if ( !resource ) {
-      this.wappalyzer.log('No response from server; url: ' + pageUrl.href, 'driver', 'error');
-
-      return false;
+      throw new Error('NO_RESPONSE');
     }
 
-    if ( resource.response.status !== 200 ) {
-      this.wappalyzer.log('Response was not OK; status: ' + resource.response.status + ' ' + resource.response.statusText + '; url: ' + pageUrl.href, 'driver', 'error');
+		this.analyzedPageUrls[pageUrl.href].status = resource.response.status;
 
-      return false;
+    if ( resource.response.status !== 200 ) {
+      throw new Error('RESPONSE_NOT_OK');
     }
 
     const headers = this.getHeaders(browser);
@@ -192,16 +202,14 @@ class Driver {
     if ( !contentType || !/\btext\/html\b/.test(contentType) ) {
       this.wappalyzer.log('Skipping; url: ' + pageUrl.href + '; content type: ' + contentType, 'driver');
 
-      this.analyzedPageUrls.splice(this.analyzedPageUrls.indexOf(pageUrl.href), 1);
+      delete this.analyzedPageUrls[pageUrl.href];
 
       return false;
     }
 
     // Validate document
     if ( !browser.document || !browser.document.documentElement ) {
-      this.wappalyzer.log('No HTML document; url: ' + pageUrl.href, 'driver', 'error');
-
-      return false;
+      throw new Error('NO_HTML_DOCUMENT');
     }
 
     return true;
@@ -303,7 +311,17 @@ class Driver {
 
     return new Promise(resolve => {
       this.fetch(pageUrl, index, depth)
-        .catch(() => {})
+        .catch(error => {
+					const type = error.message && errorTypes[error.message] ? error.message : 'UNKNOWN_ERROR';
+					const message = error.message && errorTypes[error.message] ? errorTypes[error.message] : 'Unknown error';
+
+					this.analyzedPageUrls[pageUrl.href].error = {
+						type,
+						message,
+					};
+
+					this.wappalyzer.log(`${message}; url: ${pageUrl.href}`, 'driver', 'error');
+				})
         .then(links => {
           if ( links && this.options.recursive && depth < this.options.maxDepth ) {
             return this.chunk(links.slice(0, this.options.maxUrls), depth + 1);

src/drivers/npm/index.js

@@ -13,15 +13,15 @@ if ( !url ) {
   process.exit(1);
 }
 
-var options = {};
-var arg;
+let options = {};
+let arg;
 
 while ( arg = args.shift() ) {
-  var matches = /--([^=]+)=(.+)/.exec(arg);
+  let matches = /--([^=]+)=(.+)/.exec(arg);
 
   if ( matches ) {
-    var key = matches[1].replace(/-\w/g, matches => matches[1].toUpperCase());
-    var value = matches[2];
+    let key = matches[1].replace(/-\w/g, matches => matches[1].toUpperCase());
+    let value = matches[2];
 
     options[key] = value;
   }
@@ -29,12 +29,6 @@ while ( arg = args.shift() ) {
 
 const wappalyzer = new Wappalyzer(url, options);
 
-setTimeout(() => {
-  console.log('force quit');
-
-  process.exit(1);
-}, 10000);
-
 wappalyzer.analyze()
   .then(json => {
     process.stdout.write(JSON.stringify(json) + '\n')

src/drivers/npm/package.json

@@ -2,7 +2,7 @@
   "name": "wappalyzer",
   "description": "Uncovers the technologies used on websites",
   "homepage": "https://github.com/AliasIO/Wappalyzer",
-  "version": "5.4.19",
+  "version": "5.4.20",
   "author": "Elbert Alias",
   "license": "GPL-3.0",
   "repository": {

src/drivers/webextension/manifest.json

@@ -4,7 +4,7 @@
 	"author": "Elbert Alias",
 	"homepage_url": "https://www.wappalyzer.com",
 	"description": "Identify web technologies",
-	"version": "5.4.19",
+	"version": "5.4.20",
 	"default_locale": "en",
 	"manifest_version": 2,
 	"icons": {