From 125f0178cecc3da284b752eedff4676a07f3ba14 Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Wed, 23 Jun 2021 12:16:48 +1000 Subject: [PATCH] Implement requires in WebExtension driver --- src/drivers/webextension/js/content.js | 110 +++++++++++++++++++------ src/drivers/webextension/js/driver.js | 56 ++++++++++++- src/technologies.json | 5 +- src/wappalyzer.js | 61 ++++++++++---- 4 files changed, 184 insertions(+), 48 deletions(-) diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js index 6694e7a82..182cb6a50 100644 --- a/src/drivers/webextension/js/content.js +++ b/src/drivers/webextension/js/content.js @@ -3,6 +3,12 @@ /* globals chrome */ const Content = { + href: location.href, + cache: {}, + language: '', + + requiresAnalyzed: [], + /** * Initialise content script */ @@ -32,7 +38,7 @@ const Content = { html = chunks.join('\n') // Determine language based on the HTML lang attribute or content - const language = + Content.language = document.documentElement.getAttribute('lang') || document.documentElement.getAttribute('xml:lang') || (await new Promise((resolve) => @@ -86,10 +92,37 @@ const Content = { {} ) + // Detect Google Ads + if (/^(www\.)?google(\.[a-z]{2,3}){1,2}$/.test(location.hostname)) { + const ads = document.querySelectorAll( + '#tads [data-text-ad] a[data-pcu]' + ) + + for (const ad of ads) { + Content.driver('detectTechnology', [ad.href, 'Google Ads']) + } + } + + // Detect Microsoft Ads + if (/^(www\.)?bing\.com$/.test(location.hostname)) { + const ads = document.querySelectorAll('.b_ad .b_adurl cite') + + for (const ad of ads) { + const url = ad.textContent.split(' ')[0].trim() + + Content.driver('detectTechnology', [ + url.startsWith('http') ? url : `http://${url}`, + 'Microsoft Advertising', + ]) + } + } + + Content.cache = { html, css, scripts, meta } + Content.driver('onContentLoad', [ - location.href, - { html, css, scripts, meta }, - language, + Content.href, + Content.cache, + Content.language, ]) const technologies = await Content.driver('getTechnologies') @@ -105,6 +138,32 @@ const Content = { } }, + /** + * Enable scripts to call Driver functions through messaging + * @param {Object} message + * @param {Object} sender + * @param {Function} callback + */ + onMessage({ source, func, args }, sender, callback) { + if (!func) { + return + } + + Content.driver('log', { source, func, args }) + + if (!Content[func]) { + Content.error(new Error(`Method does not exist: Content.${func}`)) + + return + } + + Promise.resolve(Content[func].call(Content[func], ...(args || []))) + .then(callback) + .catch(Content.error) + + return !!callback + }, + driver(func, args) { return new Promise((resolve, reject) => { chrome.runtime.sendMessage( @@ -136,6 +195,23 @@ const Content = { }) }, + analyzeRequires(requires) { + Object.keys(requires).forEach((name) => { + if (!Content.requiresAnalyzed.includes(name)) { + Content.requiresAnalyzed.push(name) + + Content.driver('onContentLoad', [ + Content.href, + Content.cache, + Content.language, + name, + ]) + + Content.onGetTechnologies(requires[name].technologies) + } + }) + }, + /** * Callback for getTechnologies * @param {Array} technologies @@ -259,33 +335,13 @@ const Content = { return technologies }, []) - // Detect Google Ads - if (/^(www\.)?google(\.[a-z]{2,3}){1,2}$/.test(location.hostname)) { - const ads = document.querySelectorAll('#tads [data-text-ad] a[data-pcu]') - - for (const ad of ads) { - Content.driver('detectTechnology', [ad.href, 'Google Ads']) - } - } - - // Detect Microsoft Ads - if (/^(www\.)?bing\.com$/.test(location.hostname)) { - const ads = document.querySelectorAll('.b_ad .b_adurl cite') - - for (const ad of ads) { - const url = ad.textContent.split(' ')[0].trim() - - Content.driver('detectTechnology', [ - url.startsWith('http') ? url : `http://${url}`, - 'Microsoft Advertising', - ]) - } - } - Content.driver('analyzeDom', [location.href, dom]) }, } +// Enable messaging between scripts +chrome.runtime.onMessage.addListener(Content.onMessage) + if (/complete|interactive|loaded/.test(document.readyState)) { Content.init() } else { diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index 8d0bc6f5b..e998255aa 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -298,6 +298,38 @@ const Driver = { return !!callback }, + async content(url, func, args) { + const [tab] = await promisify(chrome.tabs, 'query', { + url: globEscape(url), + }) + + if (!tab) { + return + } + + return new Promise((resolve, reject) => { + chrome.tabs.sendMessage( + tab.id, + { + source: 'driver.js', + func, + args: args ? (Array.isArray(args) ? args : [args]) : [], + }, + (response) => { + chrome.runtime.lastError + ? func === 'error' + ? resolve() + : Driver.error( + new Error( + `${chrome.runtime.lastError}: Driver.${func}(${args})` + ) + ) + : resolve(response) + } + ) + }) + }, + /** * Analyse response headers * @param {Object} request @@ -375,7 +407,7 @@ const Driver = { * @param {Object} items * @param {String} language */ - async onContentLoad(url, items, language) { + async onContentLoad(url, items, language, requires) { try { const { hostname } = new URL(url) @@ -393,7 +425,10 @@ const Driver = { await Driver.onDetect( url, - await analyze({ url, ...items }), + await analyze( + { url, ...items }, + requires ? Wappalyzer.requires[requires].technologies : undefined + ), language, true ) @@ -429,7 +464,13 @@ const Driver = { * @param {String} language * @param {Boolean} incrementHits */ - async onDetect(url, detections = [], language, incrementHits = false) { + async onDetect( + url, + detections = [], + language, + incrementHits = false, + analyzeRequires = true + ) { if (!url || !detections.length) { return } @@ -526,6 +567,15 @@ const Driver = { return detection }) + const requires = Wappalyzer.requires + .filter(({ name, technologies }) => + resolved.some(({ name: _name }) => _name === name) + ) + .map(({ technologies }) => technologies) + .flat() + + Driver.content(url, 'analyzeRequires', [requires]) + await Driver.setIcon(url, resolved) if (url) { diff --git a/src/technologies.json b/src/technologies.json index c3ea8cb53..bf797a933 100644 --- a/src/technologies.json +++ b/src/technologies.json @@ -18123,8 +18123,7 @@ "description": "SiteSpect is the A/B testing and optimisation solution.", "icon": "SiteSpect.png", "js": { - "SS": "\\;confidence:50", - "ss_dom_var": "\\;confidence:50" + "ss_dom_var": "" }, "pricing": [ "poa" @@ -24442,4 +24441,4 @@ "website": "https://www.xt-commerce.com" } } -} \ No newline at end of file +} diff --git a/src/wappalyzer.js b/src/wappalyzer.js index 57508d81c..92fefbcce 100644 --- a/src/wappalyzer.js +++ b/src/wappalyzer.js @@ -11,6 +11,7 @@ function toArray(value) { const Wappalyzer = { technologies: [], categories: [], + requires: {}, slugify: (string) => string @@ -194,20 +195,23 @@ const Wappalyzer = { * Initialize analyzation. * @param {*} param0 */ - async analyze({ - url, - xhr, - html, - css, - robots, - magento, - meta, - headers, - dns, - certIssuer, - cookies, - scripts, - }) { + async analyze( + { + url, + xhr, + html, + css, + robots, + magento, + meta, + headers, + dns, + certIssuer, + cookies, + scripts, + }, + technologies = Wappalyzer.technologies + ) { const oo = Wappalyzer.analyzeOneToOne const om = Wappalyzer.analyzeOneToMany const mm = Wappalyzer.analyzeManyToMany @@ -217,7 +221,7 @@ const Wappalyzer = { try { const detections = flatten( await Promise.all( - Wappalyzer.technologies.map(async (technology) => { + technologies.map(async (technology) => { await next() return flatten([ @@ -270,6 +274,7 @@ const Wappalyzer = { js, implies, excludes, + requires, icon, website, cpe, @@ -312,6 +317,9 @@ const Wappalyzer = { excludes: transform(excludes).map(({ value }) => ({ name: value, })), + requires: transform(requires).map(({ value }) => ({ + name: value, + })), icon: icon || 'default.svg', website: website || null, cpe: cpe || null, @@ -319,6 +327,29 @@ const Wappalyzer = { return technologies }, []) + + Wappalyzer.technologies + .filter(({ requires }) => requires.length) + .forEach((technology) => + technology.requires.forEach(({ name }) => { + if (!Wappalyzer.getTechnology(name)) { + throw new Error(`Required technology does not exist: ${name}`) + } + + Wappalyzer.requires[name] = Wappalyzer.requires[name] || [] + + Wappalyzer.requires[name].push(technology) + }) + ) + + Wappalyzer.requires = Object.keys(Wappalyzer.requires).map((name) => ({ + name, + technologies: Wappalyzer.requires[name], + })) + + Wappalyzer.technologies = Wappalyzer.technologies.filter( + ({ requires }) => !requires.length + ) }, /**