diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index c9ae64bee..8bb8510f7 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -10,6 +10,12 @@ const json = JSON.parse(fs.readFileSync(path.resolve(__dirname + '/apps.json'))) const extensions = /^([^.]+$|\.(asp|aspx|cgi|htm|html|jsp|php)$)/; +const errorTypes = { + RESPONSE_NOT_OK: 'Response was not ok', + NO_RESPONSE: 'No response from server', + NO_HTML_DOCUMENT: 'No HTML document', +}; + class Driver { constructor(pageUrl, options) { this.options = Object.assign({}, { @@ -38,7 +44,7 @@ class Driver { this.options.htmlMaxRows = parseInt(this.options.htmlMaxRows, 10); this.origPageUrl = url.parse(pageUrl); - this.analyzedPageUrls = []; + this.analyzedPageUrls = {}; this.apps = []; this.meta = {}; @@ -99,22 +105,24 @@ class Driver { fetch(pageUrl, index, depth) { // Return when the URL is a duplicate or maxUrls has been reached - if ( this.analyzedPageUrls.indexOf(pageUrl.href) !== -1 || this.analyzedPageUrls.length >= this.options.maxUrls ) { + if (this.analyzedPageUrls[pageUrl.href] || this.analyzedPageUrls.length >= this.options.maxUrls) { return Promise.resolve(); } - this.analyzedPageUrls.push(pageUrl.href); + this.analyzedPageUrls[pageUrl.href] = { + status: 0, + }; const timerScope = { last: new Date().getTime() }; - this.timer('fetch; url: ' + pageUrl.href + '; depth: ' + depth + '; delay: ' + ( this.options.delay * index ) + 'ms', timerScope); + this.timer('fetch; url: ' + pageUrl.href + '; depth: ' + depth + '; delay: ' + (this.options.delay * index) + 'ms', timerScope); - return new Promise(resolve => this.sleep(this.options.delay * index).then(() => this.visit(pageUrl, timerScope, resolve))); + return new Promise((resolve, reject) => this.sleep(this.options.delay * index).then(() => this.visit(pageUrl, timerScope, resolve, reject))); } - visit(pageUrl, timerScope, resolve) { + visit(pageUrl, timerScope, resolve, reject) { const browser = new Browser({ proxy: this.options.proxy, silent: true, @@ -133,9 +141,13 @@ class Driver { browser.visit(pageUrl.href, () => { this.timer('browser.visit end; url: ' + pageUrl.href, timerScope); - if ( !this.responseOk(browser, pageUrl) ) { - return resolve(); - } + try { + if (!this.checkResponse(browser, pageUrl)) { + return resolve(); + } + } catch(error) { + return reject(error); + } const headers = this.getHeaders(browser); const html = this.getHtml(browser); @@ -168,20 +180,18 @@ class Driver { }); } - responseOk(browser, pageUrl) { + checkResponse(browser, pageUrl) { // Validate response const resource = browser.resources.length ? browser.resources.filter(resource => resource.response).shift() : null; if ( !resource ) { - this.wappalyzer.log('No response from server; url: ' + pageUrl.href, 'driver', 'error'); - - return false; + throw new Error('NO_RESPONSE'); } - if ( resource.response.status !== 200 ) { - this.wappalyzer.log('Response was not OK; status: ' + resource.response.status + ' ' + resource.response.statusText + '; url: ' + pageUrl.href, 'driver', 'error'); + this.analyzedPageUrls[pageUrl.href].status = resource.response.status; - return false; + if ( resource.response.status !== 200 ) { + throw new Error('RESPONSE_NOT_OK'); } const headers = this.getHeaders(browser); @@ -192,16 +202,14 @@ class Driver { if ( !contentType || !/\btext\/html\b/.test(contentType) ) { this.wappalyzer.log('Skipping; url: ' + pageUrl.href + '; content type: ' + contentType, 'driver'); - this.analyzedPageUrls.splice(this.analyzedPageUrls.indexOf(pageUrl.href), 1); + delete this.analyzedPageUrls[pageUrl.href]; return false; } // Validate document if ( !browser.document || !browser.document.documentElement ) { - this.wappalyzer.log('No HTML document; url: ' + pageUrl.href, 'driver', 'error'); - - return false; + throw new Error('NO_HTML_DOCUMENT'); } return true; @@ -234,7 +242,7 @@ class Driver { .slice(0, this.options.htmlMaxRows / 2).concat(html.slice(html.length - this.options.htmlMaxRows / 2)) .map(line => line.substring(0, this.options.htmlMaxCols)) .join('\n'); - } catch ( error ) { + } catch(error) { this.wappalyzer.log(error.message, 'browser', 'error'); } @@ -303,7 +311,17 @@ class Driver { return new Promise(resolve => { this.fetch(pageUrl, index, depth) - .catch(() => {}) + .catch(error => { + const type = error.message && errorTypes[error.message] ? error.message : 'UNKNOWN_ERROR'; + const message = error.message && errorTypes[error.message] ? errorTypes[error.message] : 'Unknown error'; + + this.analyzedPageUrls[pageUrl.href].error = { + type, + message, + }; + + this.wappalyzer.log(`${message}; url: ${pageUrl.href}`, 'driver', 'error'); + }) .then(links => { if ( links && this.options.recursive && depth < this.options.maxDepth ) { return this.chunk(links.slice(0, this.options.maxUrls), depth + 1); diff --git a/src/drivers/npm/index.js b/src/drivers/npm/index.js index bc29ba604..a638e2aea 100755 --- a/src/drivers/npm/index.js +++ b/src/drivers/npm/index.js @@ -13,15 +13,15 @@ if ( !url ) { process.exit(1); } -var options = {}; -var arg; +let options = {}; +let arg; while ( arg = args.shift() ) { - var matches = /--([^=]+)=(.+)/.exec(arg); + let matches = /--([^=]+)=(.+)/.exec(arg); if ( matches ) { - var key = matches[1].replace(/-\w/g, matches => matches[1].toUpperCase()); - var value = matches[2]; + let key = matches[1].replace(/-\w/g, matches => matches[1].toUpperCase()); + let value = matches[2]; options[key] = value; } @@ -29,12 +29,6 @@ while ( arg = args.shift() ) { const wappalyzer = new Wappalyzer(url, options); -setTimeout(() => { - console.log('force quit'); - - process.exit(1); -}, 10000); - wappalyzer.analyze() .then(json => { process.stdout.write(JSON.stringify(json) + '\n') diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index f58a0ec9b..8df5d08a3 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -2,7 +2,7 @@ "name": "wappalyzer", "description": "Uncovers the technologies used on websites", "homepage": "https://github.com/AliasIO/Wappalyzer", - "version": "5.4.19", + "version": "5.4.20", "author": "Elbert Alias", "license": "GPL-3.0", "repository": { diff --git a/src/drivers/webextension/js/popup.js b/src/drivers/webextension/js/popup.js index 6483a9b14..e846dd0f5 100644 --- a/src/drivers/webextension/js/popup.js +++ b/src/drivers/webextension/js/popup.js @@ -104,7 +104,7 @@ function appsToDomTemplate(response) { }, [ 'img', { class: 'detected__app-icon', - src: '../images/icons/' + ( response.apps[appName].icon || 'default.svg' ) + src: '../images/icons/' + (response.apps[appName].icon || 'default.svg') }, ], [ 'span', { diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json index bec283c38..239b465d3 100644 --- a/src/drivers/webextension/manifest.json +++ b/src/drivers/webextension/manifest.json @@ -4,7 +4,7 @@ "author": "Elbert Alias", "homepage_url": "https://www.wappalyzer.com", "description": "Identify web technologies", - "version": "5.4.19", + "version": "5.4.20", "default_locale": "en", "manifest_version": 2, "icons": {