From 90f981000b70f7e02ae8168f360a797c9376621c Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Tue, 18 May 2021 09:13:36 +1000 Subject: [PATCH] Non-blocking analyse --- src/README.md | 10 +-- src/drivers/npm/driver.js | 110 ++++++++++++++--------- src/drivers/webextension/js/driver.js | 122 +++++++++++++++----------- src/wappalyzer.js | 42 +++++---- 4 files changed, 169 insertions(+), 115 deletions(-) diff --git a/src/README.md b/src/README.md index 6ffff3ef9..0980442af 100644 --- a/src/README.md +++ b/src/README.md @@ -25,16 +25,16 @@ const { technologies, categories } = JSON.parse( Wappalyzer.setTechnologies(technologies) Wappalyzer.setCategories(categories) -const detections = Wappalyzer.analyze({ +Wappalyzer.analyze({ url: 'https://example.github.io/', meta: { generator: ['WordPress'] }, headers: { server: ['Nginx'] }, scripts: ['jquery-3.0.0.js'], cookies: { awselb: [''] }, html: '
' -}) - -const results = Wappalyzer.resolve(detections) +}).then((detections) => { + const results = Wappalyzer.resolve(detections) -console.log(results) + console.log(results) +}) ``` diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 9c3120e8e..0b12dc486 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -9,6 +9,10 @@ const Wappalyzer = require('./wappalyzer') const { setTechnologies, setCategories, analyze, analyzeManyToMany, resolve } = Wappalyzer +function next() { + return new Promise((resolve) => setImmediate(resolve)) +} + const { AWS_LAMBDA_FUNCTION_NAME, CHROMIUM_BIN, CHROMIUM_DATA_DIR } = process.env @@ -49,53 +53,75 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)) } -function analyzeJs(js) { +async function analyzeJs(js) { return Array.prototype.concat.apply( [], - js.map(({ name, chain, value }) => - analyzeManyToMany( - Wappalyzer.technologies.find(({ name: _name }) => name === _name), - 'js', - { [chain]: [value] } - ) + await Promise.all( + js.map(async ({ name, chain, value }) => { + await next() + + return analyzeManyToMany( + Wappalyzer.technologies.find(({ name: _name }) => name === _name), + 'js', + { [chain]: [value] } + ) + }) ) ) } -function analyzeDom(dom) { +async function analyzeDom(dom) { return Array.prototype.concat.apply( [], - dom.map(({ name, selector, exists, text, property, attribute, value }) => { - const technology = Wappalyzer.technologies.find( - ({ name: _name }) => name === _name - ) + await Promise.all( + dom.map( + async ({ + name, + selector, + exists, + text, + property, + attribute, + value, + }) => { + await next() + + const technology = Wappalyzer.technologies.find( + ({ name: _name }) => name === _name + ) - if (typeof exists !== 'undefined') { - return analyzeManyToMany(technology, 'dom.exists', { - [selector]: [''], - }) - } + if (typeof exists !== 'undefined') { + return analyzeManyToMany(technology, 'dom.exists', { + [selector]: [''], + }) + } - if (typeof text !== 'undefined') { - return analyzeManyToMany(technology, 'dom.text', { - [selector]: [text], - }) - } + if (typeof text !== 'undefined') { + return analyzeManyToMany(technology, 'dom.text', { + [selector]: [text], + }) + } - if (typeof property !== 'undefined') { - return analyzeManyToMany(technology, `dom.properties.${property}`, { - [selector]: [value], - }) - } + if (typeof property !== 'undefined') { + return analyzeManyToMany(technology, `dom.properties.${property}`, { + [selector]: [value], + }) + } - if (typeof attribute !== 'undefined') { - return analyzeManyToMany(technology, `dom.attributes.${attribute}`, { - [selector]: [value], - }) - } + if (typeof attribute !== 'undefined') { + return analyzeManyToMany( + technology, + `dom.attributes.${attribute}`, + { + [selector]: [value], + } + ) + } - return [] - }) + return [] + } + ) + ) ) } @@ -343,10 +369,10 @@ class Site { if (!xhrDebounce.includes(hostname)) { xhrDebounce.push(hostname) - setTimeout(() => { + setTimeout(async () => { xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1) - this.onDetect(analyze({ xhr: hostname })) + this.onDetect(await analyze({ xhr: hostname })) }, 1000) } } @@ -402,7 +428,7 @@ class Site { ? response.securityDetails().issuer() : '' - this.onDetect(analyze({ headers, certIssuer })) + this.onDetect(await analyze({ headers, certIssuer })) await this.emit('response', { page, response, headers, certIssuer }) } @@ -720,7 +746,7 @@ class Site { return dns }, {}) - this.onDetect(analyze({ dns: this.dns })) + this.onDetect(await analyze({ dns: this.dns })) } // Validate response @@ -736,10 +762,10 @@ class Site { throw new Error('No response from server') } - this.onDetect(analyzeDom(dom)) - this.onDetect(analyzeJs(js)) + this.onDetect(await analyzeDom(dom)) + this.onDetect(await analyzeJs(js)) this.onDetect( - analyze({ + await analyze({ url, cookies, html, @@ -874,7 +900,7 @@ class Site { this.log(`get ${path}: ok`) - this.onDetect(analyze({ [file]: body })) + this.onDetect(await analyze({ [file]: body })) } catch (error) { this.error(`get ${path}: ${error.message || error}`) } diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index 566a95bce..ef11323cc 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -1,6 +1,6 @@ 'use strict' /* eslint-env browser */ -/* globals chrome, Wappalyzer, Utils */ +/* globals chrome, Wappalyzer, Utils, next */ const { setTechnologies, @@ -161,17 +161,21 @@ const Driver = { * @param {String} url * @param {Array} js */ - analyzeJs(url, js) { + async analyzeJs(url, js) { return Driver.onDetect( url, Array.prototype.concat.apply( [], - js.map(({ name, chain, value }) => - analyzeManyToMany( - Wappalyzer.technologies.find(({ name: _name }) => name === _name), - 'js', - { [chain]: [value] } - ) + await Promise.all( + js.map(async ({ name, chain, value }) => { + await next() + + return analyzeManyToMany( + Wappalyzer.technologies.find(({ name: _name }) => name === _name), + 'js', + { [chain]: [value] } + ) + }) ) ) ) @@ -182,54 +186,58 @@ const Driver = { * @param {String} url * @param {Array} dom */ - analyzeDom(url, dom) { + async analyzeDom(url, dom) { return Driver.onDetect( url, Array.prototype.concat.apply( [], - dom.map( - ( - { name, selector, exists, text, property, attribute, value }, - index - ) => { - const technology = Wappalyzer.technologies.find( - ({ name: _name }) => name === _name - ) + await Promise.all( + dom.map( + async ( + { name, selector, exists, text, property, attribute, value }, + index + ) => { + await next() + + const technology = Wappalyzer.technologies.find( + ({ name: _name }) => name === _name + ) - if (typeof exists !== 'undefined') { - return analyzeManyToMany(technology, 'dom.exists', { - [selector]: [''], - }) - } + if (typeof exists !== 'undefined') { + return analyzeManyToMany(technology, 'dom.exists', { + [selector]: [''], + }) + } - if (typeof text !== 'undefined') { - return analyzeManyToMany(technology, 'dom.text', { - [selector]: [text], - }) - } + if (typeof text !== 'undefined') { + return analyzeManyToMany(technology, 'dom.text', { + [selector]: [text], + }) + } - if (typeof property !== 'undefined') { - return analyzeManyToMany( - technology, - `dom.properties.${property}`, - { - [selector]: [value], - } - ) - } + if (typeof property !== 'undefined') { + return analyzeManyToMany( + technology, + `dom.properties.${property}`, + { + [selector]: [value], + } + ) + } - if (typeof attribute !== 'undefined') { - return analyzeManyToMany( - technology, - `dom.attributes.${attribute}`, - { - [selector]: [value], - } - ) - } + if (typeof attribute !== 'undefined') { + return analyzeManyToMany( + technology, + `dom.attributes.${attribute}`, + { + [selector]: [value], + } + ) + } - return [] - } + return [] + } + ) ) ) ) @@ -304,7 +312,9 @@ const Driver = { ) }) - Driver.onDetect(request.url, analyze({ headers })).catch(Driver.error) + Driver.onDetect(request.url, await analyze({ headers })).catch( + Driver.error + ) } } catch (error) { Driver.error(error) @@ -332,12 +342,13 @@ const Driver = { if (!xhrDebounce.includes(hostname)) { xhrDebounce.push(hostname) - setTimeout(() => { + setTimeout(async () => { xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1) - Driver.onDetect(request.originUrl, analyze({ xhr: hostname })).catch( - Driver.error - ) + Driver.onDetect( + request.originUrl, + await analyze({ xhr: hostname }) + ).catch(Driver.error) }, 1000) } }, @@ -364,7 +375,12 @@ const Driver = { {} ) - await Driver.onDetect(url, analyze({ url, ...items }), language, true) + await Driver.onDetect( + url, + await analyze({ url, ...items }), + language, + true + ) } catch (error) { Driver.error(error) } diff --git a/src/wappalyzer.js b/src/wappalyzer.js index 02d57a874..171259af4 100644 --- a/src/wappalyzer.js +++ b/src/wappalyzer.js @@ -1,5 +1,13 @@ 'use strict' +function next() { + return new Promise((resolve) => + (typeof setImmediate !== 'undefined' + ? setImmediate + : requestAnimationFrame)(resolve) + ) +} + function toArray(value) { return Array.isArray(value) ? value : [value] } @@ -187,7 +195,7 @@ const Wappalyzer = { * Initialize analyzation. * @param {*} param0 */ - analyze({ + async analyze({ url, xhr, html, @@ -208,20 +216,24 @@ const Wappalyzer = { try { const detections = flatten( - Wappalyzer.technologies.map((technology) => - flatten([ - oo(technology, 'url', url), - oo(technology, 'xhr', xhr), - oo(technology, 'html', html), - oo(technology, 'css', css), - oo(technology, 'robots', robots), - oo(technology, 'certIssuer', certIssuer), - om(technology, 'scripts', scripts), - mm(technology, 'cookies', cookies), - mm(technology, 'meta', meta), - mm(technology, 'headers', headers), - mm(technology, 'dns', dns), - ]) + await Promise.all( + Wappalyzer.technologies.map(async (technology) => { + await next() + + return flatten([ + oo(technology, 'url', url), + oo(technology, 'xhr', xhr), + oo(technology, 'html', html), + oo(technology, 'css', css), + oo(technology, 'robots', robots), + oo(technology, 'certIssuer', certIssuer), + om(technology, 'scripts', scripts), + mm(technology, 'cookies', cookies), + mm(technology, 'meta', meta), + mm(technology, 'headers', headers), + mm(technology, 'dns', dns), + ]) + }) ) ).filter((technology) => technology)