From 8b1f03e8da5680d0c8842442356fc4a0fcbfbbc1 Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Thu, 24 Jun 2021 15:02:24 +1000 Subject: [PATCH] Make 'requires' work with 'dom' and 'js' in WebExtension driver --- src/drivers/webextension/css/styles.css | 2 +- src/drivers/webextension/js/content.js | 292 +++++++++++++----------- src/drivers/webextension/js/driver.js | 49 ++-- src/wappalyzer.js | 7 +- 4 files changed, 191 insertions(+), 159 deletions(-) diff --git a/src/drivers/webextension/css/styles.css b/src/drivers/webextension/css/styles.css index 088a983fd..dabf5fa36 100644 --- a/src/drivers/webextension/css/styles.css +++ b/src/drivers/webextension/css/styles.css @@ -616,7 +616,7 @@ body.dynamic-icon .category__heading:hover .category__pin { color: var(--color-text); border-radius: 3px; font-size: .7rem; - padding: .3rem .3rem .1rem .3rem; + padding: .1rem .3rem; margin-left: .3rem; vertical-align: middle; } diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js index 182cb6a50..605422089 100644 --- a/src/drivers/webextension/js/content.js +++ b/src/drivers/webextension/js/content.js @@ -2,12 +2,129 @@ /* eslint-env browser */ /* globals chrome */ +function getJs(technologies) { + return new Promise((resolve) => { + // Inject a script tag into the page to access methods of the window object + const script = document.createElement('script') + + script.onload = () => { + const onMessage = ({ data }) => { + if (!data.wappalyzer || !data.wappalyzer.js) { + return + } + + window.removeEventListener('message', onMessage) + + resolve(data.wappalyzer.js) + + script.remove() + } + + window.addEventListener('message', onMessage) + + window.postMessage({ + wappalyzer: { + technologies: technologies + .filter(({ js }) => Object.keys(js).length) + .map(({ name, js }) => ({ name, chains: Object.keys(js) })), + }, + }) + } + + script.setAttribute('src', chrome.extension.getURL('js/inject.js')) + + document.body.appendChild(script) + }) +} + +function getDom(technologies) { + return technologies + .filter(({ dom }) => dom && dom.constructor === Object) + .map(({ name, dom }) => ({ name, dom })) + .reduce((technologies, { name, dom }) => { + const toScalar = (value) => + typeof value === 'string' || typeof value === 'number' ? value : !!value + + Object.keys(dom).forEach((selector) => { + let nodes = [] + + try { + nodes = document.querySelectorAll(selector) + } catch (error) { + Content.driver('error', error) + } + + if (!nodes.length) { + return + } + + dom[selector].forEach(({ exists, text, properties, attributes }) => { + nodes.forEach((node) => { + if (exists) { + technologies.push({ + name, + selector, + exists: '', + }) + } + + if (text) { + const value = node.textContent.trim() + + if (value) { + technologies.push({ + name, + selector, + text: value, + }) + } + } + + if (properties) { + Object.keys(properties).forEach((property) => { + if (Object.prototype.hasOwnProperty.call(node, property)) { + const value = node[property] + + if (typeof value !== 'undefined') { + technologies.push({ + name, + selector, + property, + value: toScalar(value), + }) + } + } + }) + } + + if (attributes) { + Object.keys(attributes).forEach((attribute) => { + if (node.hasAttribute(attribute)) { + const value = node.getAttribute(attribute) + + technologies.push({ + name, + selector, + attribute, + value: toScalar(value), + }) + } + }) + } + }) + }) + }) + + return technologies + }, []) +} + const Content = { href: location.href, cache: {}, language: '', - requiresAnalyzed: [], + analyzedRequires: [], /** * Initialise content script @@ -119,7 +236,7 @@ const Content = { Content.cache = { html, css, scripts, meta } - Content.driver('onContentLoad', [ + await Content.driver('onContentLoad', [ Content.href, Content.cache, Content.language, @@ -127,12 +244,12 @@ const Content = { const technologies = await Content.driver('getTechnologies') - Content.onGetTechnologies(technologies) + await Content.onGetTechnologies(technologies) // Delayed second pass to capture async JS await new Promise((resolve) => setTimeout(resolve, 5000)) - Content.onGetTechnologies(technologies) + await Content.onGetTechnologies(technologies) } catch (error) { Content.driver('error', error) } @@ -165,7 +282,7 @@ const Content = { }, driver(func, args) { - return new Promise((resolve, reject) => { + return new Promise((resolve) => { chrome.runtime.sendMessage( { source: 'content.js', @@ -186,7 +303,9 @@ const Content = { : Content.driver( 'error', new Error( - `${chrome.runtime.lastError}: Driver.${func}(${args})` + `${ + chrome.runtime.lastError.message + }: Driver.${func}(${JSON.stringify(args)})` ) ) : resolve(response) @@ -195,147 +314,42 @@ const Content = { }) }, - analyzeRequires(requires) { - Object.keys(requires).forEach((name) => { - if (!Content.requiresAnalyzed.includes(name)) { - Content.requiresAnalyzed.push(name) - - Content.driver('onContentLoad', [ - Content.href, - Content.cache, - Content.language, - name, - ]) - - Content.onGetTechnologies(requires[name].technologies) - } - }) + async analyzeRequires(requires) { + await Promise.all( + Object.keys(requires).map(async (name) => { + if (!Content.analyzedRequires.includes(name)) { + Content.analyzedRequires.push(name) + + const technologies = requires[name].technologies + + await Promise.all([ + Content.onGetTechnologies(technologies, name), + Content.driver('onContentLoad', [ + Content.href, + Content.cache, + Content.language, + name, + ]), + ]) + } + }) + ) }, /** * Callback for getTechnologies * @param {Array} technologies */ - onGetTechnologies(technologies = []) { - // Inject a script tag into the page to access methods of the window object - const script = document.createElement('script') - - script.onload = () => { - const onMessage = ({ data }) => { - if (!data.wappalyzer || !data.wappalyzer.js) { - return - } - - window.removeEventListener('message', onMessage) - - chrome.runtime.sendMessage({ - source: 'content.js', - func: 'analyzeJs', - args: [location.href.split('#')[0], data.wappalyzer.js], - }) - - script.remove() - } - - window.addEventListener('message', onMessage) - - window.postMessage({ - wappalyzer: { - technologies: technologies - .filter(({ js }) => Object.keys(js).length) - .map(({ name, js }) => ({ name, chains: Object.keys(js) })), - }, - }) - } - - script.setAttribute('src', chrome.extension.getURL('js/inject.js')) - - document.body.appendChild(script) - - // DOM - const dom = technologies - .filter(({ dom }) => dom && dom.constructor === Object) - .map(({ name, dom }) => ({ name, dom })) - .reduce((technologies, { name, dom }) => { - const toScalar = (value) => - typeof value === 'string' || typeof value === 'number' - ? value - : !!value - - Object.keys(dom).forEach((selector) => { - let nodes = [] - - try { - nodes = document.querySelectorAll(selector) - } catch (error) { - Content.driver('error', error) - } - - if (!nodes.length) { - return - } - - dom[selector].forEach(({ exists, text, properties, attributes }) => { - nodes.forEach((node) => { - if (exists) { - technologies.push({ - name, - selector, - exists: '', - }) - } - - if (text) { - const value = node.textContent.trim() - - if (value) { - technologies.push({ - name, - selector, - text: value, - }) - } - } - - if (properties) { - Object.keys(properties).forEach((property) => { - if (Object.prototype.hasOwnProperty.call(node, property)) { - const value = node[property] - - if (typeof value !== 'undefined') { - technologies.push({ - name, - selector, - property, - value: toScalar(value), - }) - } - } - }) - } - - if (attributes) { - Object.keys(attributes).forEach((attribute) => { - if (node.hasAttribute(attribute)) { - const value = node.getAttribute(attribute) - - technologies.push({ - name, - selector, - attribute, - value: toScalar(value), - }) - } - }) - } - }) - }) - }) + async onGetTechnologies(technologies = [], requires) { + const url = location.href.split('#')[0] - return technologies - }, []) + const js = await getJs(technologies) + const dom = getDom(technologies) - Content.driver('analyzeDom', [location.href, dom]) + await Promise.all([ + Content.driver('analyzeJs', [url, js, requires]), + Content.driver('analyzeDom', [url, dom, requires]), + ]) }, } diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index e998255aa..da1a07cde 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -42,9 +42,7 @@ const Driver = { pattern: { regex, confidence }, version, }) => ({ - technology: Wappalyzer.technologies.find( - ({ name: _name }) => name === _name - ), + technology: getTechnology(name, true), pattern: { regex: new RegExp(regex, 'i'), confidence, @@ -121,7 +119,7 @@ const Driver = { */ log(message, source = 'driver', type = 'log') { // eslint-disable-next-line no-console - console[type](`wappalyzer | ${source} |`, message) + console[type](message) }, /** @@ -177,7 +175,11 @@ const Driver = { * @param {String} url * @param {Array} js */ - async analyzeJs(url, js) { + async analyzeJs(url, js, requires) { + const technologies = requires + ? Wappalyzer.requires[requires].technologies + : Wappalyzer.technologies + return Driver.onDetect( url, Array.prototype.concat.apply( @@ -187,7 +189,7 @@ const Driver = { await next() return analyzeManyToMany( - Wappalyzer.technologies.find(({ name: _name }) => name === _name), + technologies.find(({ name: _name }) => name === _name), 'js', { [chain]: [value] } ) @@ -202,7 +204,11 @@ const Driver = { * @param {String} url * @param {Array} dom */ - async analyzeDom(url, dom) { + async analyzeDom(url, dom, requires) { + const technologies = requires + ? Wappalyzer.requires[requires].technologies + : Wappalyzer.technologies + return Driver.onDetect( url, Array.prototype.concat.apply( @@ -215,7 +221,7 @@ const Driver = { ) => { await next() - const technology = Wappalyzer.technologies.find( + const technology = technologies.find( ({ name: _name }) => name === _name ) @@ -283,7 +289,9 @@ const Driver = { return } - Driver.log({ source, func, args }) + if (func !== 'log') { + Driver.log({ source, func, args }) + } if (!Driver[func]) { Driver.error(new Error(`Method does not exist: Driver.${func}`)) @@ -307,6 +315,10 @@ const Driver = { return } + if (tab.status !== 'complete') { + throw new Error(`Tab ${tab.id} not ready for sendMessage: ${tab.status}`) + } + return new Promise((resolve, reject) => { chrome.tabs.sendMessage( tab.id, @@ -321,7 +333,9 @@ const Driver = { ? resolve() : Driver.error( new Error( - `${chrome.runtime.lastError}: Driver.${func}(${args})` + `${ + chrome.runtime.lastError.message + }: Driver.${func}(${JSON.stringify(args)})` ) ) : resolve(response) @@ -567,14 +581,15 @@ const Driver = { return detection }) - const requires = Wappalyzer.requires - .filter(({ name, technologies }) => - resolved.some(({ name: _name }) => _name === name) - ) - .map(({ technologies }) => technologies) - .flat() + const requires = Wappalyzer.requires.filter(({ name, technologies }) => + resolved.some(({ name: _name }) => _name === name) + ) - Driver.content(url, 'analyzeRequires', [requires]) + try { + await Driver.content(url, 'analyzeRequires', [requires]) + } catch (error) { + // Continue + } await Driver.setIcon(url, resolved) diff --git a/src/wappalyzer.js b/src/wappalyzer.js index 92fefbcce..3c92428ea 100644 --- a/src/wappalyzer.js +++ b/src/wappalyzer.js @@ -11,7 +11,7 @@ function toArray(value) { const Wappalyzer = { technologies: [], categories: [], - requires: {}, + requires: [], slugify: (string) => string @@ -21,7 +21,10 @@ const Wappalyzer = { .replace(/(?:^-|-$)/g, ''), getTechnology: (name) => - Wappalyzer.technologies.find(({ name: _name }) => name === _name), + [ + ...Wappalyzer.technologies, + ...Wappalyzer.requires.map(({ technologies }) => technologies).flat(), + ].find(({ name: _name }) => name === _name), getCategory: (id) => Wappalyzer.categories.find(({ id: _id }) => id === _id),