diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 651b6d9c7..7b66838a1 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -131,15 +131,16 @@ class Driver { html, scripts, js - }); - - const links = Array.from(browser.document.getElementsByTagName('a')) - .filter(link => link.protocol === 'http:' || link.protocol === 'https:') - .filter(link => link.hostname === this.origPageUrl.hostname) - .filter(link => extensions.test(link.pathname)) - .map(link => { link.hash = ''; return url.parse(link.href) }); + }) + .then(() => { + const links = Array.from(browser.document.getElementsByTagName('a')) + .filter(link => link.protocol === 'http:' || link.protocol === 'https:') + .filter(link => link.hostname === this.origPageUrl.hostname) + .filter(link => extensions.test(link.pathname)) + .map(link => { link.hash = ''; return url.parse(link.href) }); - return resolve(links); + return resolve(links); + }); }); } diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js index e60df6894..36d920175 100644 --- a/src/drivers/webextension/js/content.js +++ b/src/drivers/webextension/js/content.js @@ -3,11 +3,12 @@ if ( typeof browser !== 'undefined' && typeof document.body !== 'undefined' ) { try { - var html = new XMLSerializer().serializeToString(document); + var html = new XMLSerializer().serializeToString(document).split('\n'); - if ( html.length > 100 * 1024 ) { - html = html.substring(0, 50 * 1024) + html.substring(html.length - 50 * 1024, html.length); - } + html = html + .slice(0, 1000).concat(html.slice(html.length - 1000)) + .map(line => line.substring(0, 1000)) + .join('\n'); const scripts = Array.prototype.slice .apply(document.scripts) diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index bce13cb97..c81f154c5 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -103,10 +103,10 @@ getOption('version') getOption('upgradeMessage', true) .then(upgradeMessage => { if ( upgradeMessage ) { - // openTab({ - // url: wappalyzer.config.websiteURL + 'upgraded?v' + version, - // background: true - // }); + openTab({ + url: wappalyzer.config.websiteURL + 'upgraded?v' + version, + background: true + }); } }); } diff --git a/src/drivers/webextension/js/inject.js b/src/drivers/webextension/js/inject.js index 26fc00de6..55ac2be4d 100644 --- a/src/drivers/webextension/js/inject.js +++ b/src/drivers/webextension/js/inject.js @@ -20,7 +20,7 @@ for ( let index in patterns[appName][chain] ) { const value = detectJs(chain); - if ( value ) { + if ( value && patterns[appName][chain].hasOwnProperty(index) ) { js[appName][chain][index] = value; } } diff --git a/src/wappalyzer.js b/src/wappalyzer.js index df7e3bd1e..54e2afcb3 100644 --- a/src/wappalyzer.js +++ b/src/wappalyzer.js @@ -38,7 +38,13 @@ class Wappalyzer { this.driver.log(message, source || '', type || 'debug'); } + asyncForEach(iterable, iterator) { + return Promise.all(iterable.map(item => new Promise(resolve => setTimeout(() => resolve(iterator(item)), 1)))); + } + analyze(url, data, context) { + const promises = []; + var apps = {}; if ( typeof data.html !== 'string' ) { @@ -62,52 +68,55 @@ class Wappalyzer { this.analyzeUrl(app, url); if ( data.html ) { - this.analyzeHtml(app, data.html); - this.analyzeMeta(app, data.html); + promises.push(this.analyzeHtml(app, data.html)); + promises.push(this.analyzeMeta(app, data.html)); } if ( data.scripts ) { - this.analyzeScripts(app, data.scripts); + promises.push(this.analyzeScripts(app, data.scripts)); } if ( data.headers ) { - this.analyzeHeaders(app, data.headers); + promises.push(this.analyzeHeaders(app, data.headers)); } if ( data.env ) { - this.analyzeEnv(app, data.env); - } - - if ( data.robotsTxt ) { - this.analyzeRobotsTxt(app, data.robotsTxt); + promises.push(this.analyzeEnv(app, data.env)); } }) if ( data.js ) { Object.keys(data.js).forEach(appName => { - this.analyzeJs(apps[appName], data.js[appName]); + promises.push(this.analyzeJs(apps[appName], data.js[appName])); }); } - Object.keys(apps).forEach(appName => { - var app = apps[appName]; + return new Promise(resolve => { + Promise.all(promises) + .then(() => { + Object.keys(apps).forEach(appName => { + var app = apps[appName]; - if ( !app.detected || !app.getConfidence() ) { - delete apps[app.name]; - } - }); + if ( !app.detected || !app.getConfidence() ) { + delete apps[app.name]; + } + }); - this.resolveExcludes(apps); - this.resolveImplies(apps, url.canonical); + this.resolveExcludes(apps); + this.resolveImplies(apps, url.canonical); - this.cacheDetectedApps(apps, url.canonical); - this.trackDetectedApps(apps, url, language); + this.cacheDetectedApps(apps, url.canonical); + this.trackDetectedApps(apps, url, language); - if ( Object.keys(apps).length ) { - this.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url.canonical, 'core'); - } + if ( Object.keys(apps).length ) { + this.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url.canonical, 'core'); + } + + this.driver.displayApps(this.detected[url.canonical], { language }, context); - this.driver.displayApps(this.detected[url.canonical], { language }, context); + resolve(); + }); + }); } /** @@ -393,13 +402,15 @@ class Wappalyzer { analyzeUrl(app, url) { var patterns = this.parsePatterns(app.props.url); - if ( patterns.length ) { - patterns.forEach(pattern => { - if ( pattern.regex.test(url.canonical) ) { - this.addDetected(app, pattern, 'url', url.canonical); - } - }); + if ( !patterns.length ) { + return Promise.resolve(); } + + return this.asyncForEach(patterns, pattern => { + if ( pattern.regex.test(url.canonical) ) { + this.addDetected(app, pattern, 'url', url.canonical); + } + }); } /** @@ -408,13 +419,15 @@ class Wappalyzer { analyzeHtml(app, html) { var patterns = this.parsePatterns(app.props.html); - if ( patterns.length ) { - patterns.forEach(pattern => { - if ( pattern.regex.test(html) ) { - this.addDetected(app, pattern, 'html', html); - } - }); + if ( !patterns.length ) { + return Promise.resolve(); } + + return this.asyncForEach(patterns, pattern => { + if ( pattern.regex.test(html) ) { + this.addDetected(app, pattern, 'html', html); + } + }); } /** @@ -423,67 +436,75 @@ class Wappalyzer { analyzeScripts(app, scripts) { var patterns = this.parsePatterns(app.props.script); - if ( patterns.length ) { - patterns.forEach(pattern => { - var match; + if ( !patterns.length ) { + return Promise.resolve(); + } - scripts.forEach(uri => { - if ( pattern.regex.test(uri) ) { - this.addDetected(app, pattern, 'script', uri); - } - }); + return this.asyncForEach(patterns, pattern => { + var match; + + scripts.forEach(uri => { + if ( pattern.regex.test(uri) ) { + this.addDetected(app, pattern, 'script', uri); + } }); - } + }); } /** * Analyze meta tag */ analyzeMeta(app, html) { - var regex = /]+>/ig; - var patterns = this.parsePatterns(app.props.meta); + const regex = /]+>/ig; + const patterns = this.parsePatterns(app.props.meta); + const promises = []; + var content = ''; var matches = []; while ( patterns && ( matches = regex.exec(html) ) ) { for ( var meta in patterns ) { - const r = new RegExp('(?:name|property)=["\']' + meta + '["\']', 'i'); - if ( new RegExp('(?:name|property)=["\']' + meta + '["\']', 'i').test(matches[0]) ) { + if ( r.test(matches[0]) ) { content = matches[0].match(/content=("|')([^"']+)("|')/i); - patterns[meta].forEach(pattern => { + promises.push(this.asyncForEach(patterns[meta], pattern => { if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { this.addDetected(app, pattern, 'meta', content[2], meta); } - }); + })); } } } + + return promises ? Promise.all(promises) : Promise.resolve(); } /** * analyze response headers */ analyzeHeaders(app, headers) { - var patterns = this.parsePatterns(app.props.headers); + const patterns = this.parsePatterns(app.props.headers); + const promises = []; if ( headers ) { Object.keys(patterns).forEach(headerName => { - patterns[headerName].forEach(pattern => { + this.asyncForEach(patterns[headerName], pattern => { headerName = headerName.toLowerCase(); if ( headerName in headers ) { - headers[headerName].forEach(headerValue => { + promises.push(headers[headerName].forEach(headerValue => { if ( pattern.regex.test(headerValue) ) { this.addDetected(app, pattern, 'headers', headerValue, headerName); } - }); + })); } }); }); } + + return promises ? Promise.all(promises) : Promise.resolve(); } /** @@ -493,45 +514,36 @@ class Wappalyzer { var patterns = this.parsePatterns(app.props.env); if ( patterns.length ) { - patterns.forEach(pattern => { - Object.keys(envs).forEach(env => { - if ( pattern.regex.test(envs[env]) ) { - this.addDetected(app, pattern, 'env', envs[env]); - } - }) - }); + return Promise.resolve(); } + + return this.asyncForEach(patterns, pattern => { + Object.keys(envs).forEach(env => { + if ( pattern.regex.test(envs[env]) ) { + this.addDetected(app, pattern, 'env', envs[env]); + } + }) + }); } /** * Analyze JavaScript variables */ analyzeJs(app, results) { + const promises = []; + Object.keys(results).forEach(string => { - Object.keys(results[string]).forEach(index => { + promises.push(this.asyncForEach(Object.keys(results[string]), index => { const pattern = this.jsPatterns[app.name][string][index]; const value = results[string][index]; - if ( pattern.regex.test(value) ) { + if ( pattern && pattern.regex.test(value) ) { this.addDetected(app, pattern, 'js', value); } - }); + })); }); - } - - /** - * Analyze robots.txt - */ - analyzeRobotsTxt(app, robotsTxt) { - var patterns = this.parsePatterns(app.props.robotsTxt); - if ( patterns.length ) { - patterns.forEach(pattern => { - if ( pattern.regex.test(robotsTxt) ) { - this.addDetected(app, pattern, 'robotsTxt', robotsTxt); - } - }); - } + return promises ? Promise.all(promises) : Promise.resolve(); } /** @@ -540,6 +552,8 @@ class Wappalyzer { addDetected(app, pattern, type, value, key) { app.detected = true; + console.log(app); + // Set confidence level app.confidence[type + ' ' + ( key ? key + ' ' : '' ) + pattern.regex] = pattern.confidence || 100;