From 376345aafb0d0590cc94f8f985351028e74a893c Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Thu, 4 Jun 2020 15:03:53 +1000 Subject: [PATCH] Refactoring --- src/drivers/webextension/css/popup.css | 221 +----- src/drivers/webextension/html/background.html | 5 +- src/drivers/webextension/html/options.html | 2 +- src/drivers/webextension/html/popup.html | 30 +- src/drivers/webextension/js/driver.bak.js | 445 +++++++++++ src/drivers/webextension/js/driver.js | 580 +++++--------- src/drivers/webextension/js/driver2.js | 285 ------- src/drivers/webextension/js/popup.bak.js | 333 ++++++++ src/drivers/webextension/js/popup.js | 379 ++------- src/drivers/webextension/js/utils.js | 47 ++ src/drivers/webextension/js/wappalyzer.bak.js | 727 ++++++++++++++++++ src/drivers/webextension/js/wappalyzer2.js | 326 -------- 12 files changed, 1885 insertions(+), 1495 deletions(-) create mode 100644 src/drivers/webextension/js/driver.bak.js delete mode 100644 src/drivers/webextension/js/driver2.js create mode 100644 src/drivers/webextension/js/popup.bak.js create mode 100644 src/drivers/webextension/js/utils.js create mode 100644 src/drivers/webextension/js/wappalyzer.bak.js delete mode 100644 src/drivers/webextension/js/wappalyzer2.js diff --git a/src/drivers/webextension/css/popup.css b/src/drivers/webextension/css/popup.css index 14ecfd695..e364577df 100644 --- a/src/drivers/webextension/css/popup.css +++ b/src/drivers/webextension/css/popup.css @@ -1,30 +1,47 @@ +:root { + --color-primary: #4608ad; + --color-secondary: #e0e0e0; + --color-text: #4a4a4a; +} + body { background: #fff; direction: __MSG_@@bidi_dir__; font-family: Helvetica, Arial, sans-serif; - font-size: .8rem; + font-size: .9rem; + line-height: 1.5rem; margin: 0; min-width: 30rem; } +a { + color: var(--color-primary); + outline: none; + text-decoration: none; +} + +a:focus { + outline: none; +} + +a:hover { + text-decoration: underline; +} + .header { align-items: center; - border-bottom: 1px solid #dbdbdb; + border-bottom: 1px solid var(--color-secondary); height: 4rem; display: flex; } -.header__link:focus { - outline: none; -} - .header__logo { display: inline-block; margin: .2rem 1.5rem 0 1.5rem; -webkit-backface-visibility: hidden; -webkit-transform: translateZ(0) scale(1.0, 1.0); transform: translateZ(0); - height: 2rem; + height: 2.5rem; } .header__logo--dark { @@ -33,144 +50,37 @@ body { .footer { align-items: center; - border-top: 1px solid #dbdbdb; + border-top: 1px solid var(--color-secondary); height: 3rem; display: flex; padding: 0 1.5rem; } -.footer__link { - color: #4608ad; - text-decoration: none; -} - -.footer__link:hover, .footer__link:active { - color: #4608ad; - text-decoration: underline; -} - -.container { - min-height: 5rem; - padding: 1rem 1.5rem 0rem 1.5rem; -} - -.detected { +.detections { columns: 2; column-gap: 1.5rem; - line-height: 1.4rem; + padding: 1.5rem; } -.detected__category { +.category { page-break-inside: avoid; break-inside: avoid-column; padding-bottom: 1rem; } -.detected__category-name { - display: block; -} - -.detected__category-link { - color: #4608ad; +.category__link { font-weight: bold; line-height: 2rem; text-decoration: none; } -.detected__category-link:hover { - color: #4a4a4a; -} - -.detected__category-pin-wrapper { - margin-left: .2rem; - margin-right: .2rem; -} - -.detected__category-pin { - cursor: pointer; - display: none; - height: 16px; - margin-left: .2rem; - width: 16px; - vertical-align: middle; -} - -.detected__category:hover .detected__category-pin--inactive { - display: inline-block; -} - -.detected__category-pin-wrapper--active .detected__category-pin--inactive, -.detected__category-pin-wrapper:hover .detected__category-pin--inactive { - display: none !important; -} - -.detected__category-pin-wrapper--active .detected__category-pin--active, -.detected__category-pin-wrapper:hover .detected__category-pin--active { - display: inline-block; -} - -.detected__app { - color: #4a4a4a; +.technology { display: block; line-height: 1.7rem; - text-decoration: none; -} - -.detected__app:focus { - display: block; - outline: 0; -} - -.detected__app-icon { - display: inline-block; - height: 16px; - margin-inline-end: .5rem; - vertical-align: -.2rem; - width: 16px; -} - -.detected__app-name { -} - -.detected__app-version, .detected__app-confidence { - background: #eee; - border-radius: 3px; - font-size: .7rem; - margin-left: .3rem; - padding: .1rem .2rem; } -.detected__app:hover .detected__app-name { - border-bottom: 1px solid #4a4a4a; -} - -.detected__app:hover .detected__app-version, -.detected__app:hover .detected__app-confidence { - border-bottom: 1px solid white; -} - -.detected-app { - padding: 7px 0; -} - -.detected-app:first-child { - padding-top: 0; -} - -.detected-app:last-child { - border: none; - padding-bottom: 0; -} - -.empty { - display: flex; - height: 5rem; - margin-bottom: 1rem; - align-items: center; - justify-content: center; -} - -.empty__text { +.technology__link { + color: var(--color-text) } .terms { @@ -216,72 +126,5 @@ body { } .terms__privacy { - color: #4608ad; margin-top: 1rem; } - -@media (prefers-color-scheme: dark) { - /* Add alternative color palette for Dark mode theme. */ - body.theme-mode-sync { - background: linear-gradient(160deg, #32067c, #150233); - } - - .theme-mode-sync .header { - border-bottom: 1px solid rgba(255, 255, 255, .2); - } - - .theme-mode-sync .header__logo--dark { - display: inline-block; - } - - .theme-mode-sync .header__logo--light { - display: none; - } - - .theme-mode-sync .footer { - border-top: 1px solid rgba(255, 255, 255, .2); - } - - .theme-mode-sync .footer__link { - color: rgba(255, 255, 255, .8); - } - - .theme-mode-sync .footer__link:hover, .theme-mode-sync .footer__link:active { - color: rgba(255, 255, 255, .8); - } - - .theme-mode-sync .container { - color: white; - } - - .theme-mode-sync .detected__category-link { - color: #fff; - } - - .theme-mode-sync .detected__app { - color: rgba(255, 255, 255, .8); - } - - .theme-mode-sync .detected__category-link:hover { - color: white; - border-bottom: 1px solid white; - } - - .theme-mode-sync .detected__app-version, .theme-mode-sync .detected__app-confidence { - background-color: #4608ad; - } - - .theme-mode-sync .detected__app:hover .detected__app-name { - border-bottom: 1px solid white; - } - - .theme-mode-sync .detected__app:hover .theme-mode-sync .detected__app-version, - .theme-mode-sync .detected__app:hover .theme-mode-sync .detected__app-confidence { - border-bottom: none; - } - - .theme-mode-sync .terms__accept, - .theme-mode-sync .terms__privacy { - color: white; - } -} diff --git a/src/drivers/webextension/html/background.html b/src/drivers/webextension/html/background.html index f6147dc56..6900b5e3b 100644 --- a/src/drivers/webextension/html/background.html +++ b/src/drivers/webextension/html/background.html @@ -4,8 +4,9 @@ - - + + + diff --git a/src/drivers/webextension/html/options.html b/src/drivers/webextension/html/options.html index a10e8a859..23b49fb83 100644 --- a/src/drivers/webextension/html/options.html +++ b/src/drivers/webextension/html/options.html @@ -12,7 +12,7 @@ - + diff --git a/src/drivers/webextension/html/popup.html b/src/drivers/webextension/html/popup.html index 15fdc91dd..e7306ecc7 100644 --- a/src/drivers/webextension/html/popup.html +++ b/src/drivers/webextension/html/popup.html @@ -1,5 +1,4 @@ - @@ -7,28 +6,35 @@ - - +
- +
-
-
-
-
+
+
+ + + + +
-
-
+
+
diff --git a/src/drivers/webextension/js/driver.bak.js b/src/drivers/webextension/js/driver.bak.js new file mode 100644 index 000000000..0ff7d491c --- /dev/null +++ b/src/drivers/webextension/js/driver.bak.js @@ -0,0 +1,445 @@ +/** + * WebExtension driver + */ + +/* eslint-env browser */ +/* global browser, chrome, Wappalyzer */ + +/** global: browser */ +/** global: chrome */ +/** global: fetch */ +/** global: Wappalyzer */ + +const wappalyzer = new Wappalyzer() + +const tabCache = {} +const robotsTxtQueue = {} + +let categoryOrder = [] + +browser.tabs.onRemoved.addListener((tabId) => { + tabCache[tabId] = null +}) + +function userAgent() { + const url = chrome.extension.getURL('/') + + if (url.startsWith('moz-')) { + return 'firefox' + } + + if (url.startsWith('ms-browser')) { + return 'edge' + } + + return 'chrome' +} + +/** + * Get a value from localStorage + */ +function getOption(name, defaultValue = null) { + return new Promise(async (resolve, reject) => { + let value = defaultValue + + try { + const option = await browser.storage.local.get(name) + + if (option[name] !== undefined) { + value = option[name] + } + } catch (error) { + wappalyzer.log(error.message, 'driver', 'error') + + return reject(error.message) + } + + return resolve(value) + }) +} + +/** + * Set a value in localStorage + */ +function setOption(name, value) { + return new Promise(async (resolve, reject) => { + try { + await browser.storage.local.set({ [name]: value }) + } catch (error) { + wappalyzer.log(error.message, 'driver', 'error') + + return reject(error.message) + } + + return resolve() + }) +} + +/** + * Open a tab + */ +function openTab(args) { + browser.tabs.create({ + url: args.url, + active: args.background === undefined || !args.background + }) +} + +/** + * Make a POST request + */ +async function post(url, body) { + try { + const response = await fetch(url, { + method: 'POST', + body: JSON.stringify(body) + }) + + wappalyzer.log(`POST ${url}: ${response.status}`, 'driver') + } catch (error) { + wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error') + } +} + +// Capture response headers +browser.webRequest.onCompleted.addListener( + async (request) => { + const headers = {} + + if (request.responseHeaders) { + const url = wappalyzer.parseUrl(request.url) + + let tab + + try { + ;[tab] = await browser.tabs.query({ url: [url.href] }) + } catch (error) { + wappalyzer.log(error, 'driver', 'error') + } + + if (tab) { + request.responseHeaders.forEach((header) => { + const name = header.name.toLowerCase() + + headers[name] = headers[name] || [] + + headers[name].push( + (header.value || header.binaryValue || '').toString() + ) + }) + + if ( + headers['content-type'] && + /\/x?html/.test(headers['content-type'][0]) + ) { + wappalyzer.analyze(url, { headers }, { tab }) + } + } + } + }, + { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, + ['responseHeaders'] +) + +browser.runtime.onConnect.addListener((port) => { + port.onMessage.addListener(async (message) => { + if (message.id === undefined) { + return + } + + if (message.id !== 'log') { + wappalyzer.log(`Message from ${port.name}: ${message.id}`, 'driver') + } + + const pinnedCategory = await getOption('pinnedCategory') + + const url = wappalyzer.parseUrl(port.sender.tab ? port.sender.tab.url : '') + + const cookies = await browser.cookies.getAll({ + domain: `.${url.hostname}` + }) + + let response + + switch (message.id) { + case 'log': + wappalyzer.log(message.subject, message.source) + + break + case 'init': + wappalyzer.analyze(url, { cookies }, { tab: port.sender.tab }) + + break + case 'analyze': + if (message.subject.html) { + browser.i18n + .detectLanguage(message.subject.html) + .then(({ languages }) => { + const language = languages + .filter(({ percentage }) => percentage >= 75) + .map(({ language: lang }) => lang)[0] + + message.subject.language = language + + wappalyzer.analyze(url, message.subject, { tab: port.sender.tab }) + }) + } else { + wappalyzer.analyze(url, message.subject, { tab: port.sender.tab }) + } + + await setOption('hostnameCache', wappalyzer.hostnameCache) + + break + case 'ad_log': + wappalyzer.cacheDetectedAds(message.subject) + + break + case 'get_apps': + response = { + tabCache: tabCache[message.tab.id], + apps: wappalyzer.apps, + categories: wappalyzer.categories, + pinnedCategory, + termsAccepted: + userAgent() === 'chrome' || + (await getOption('termsAccepted', false)) + } + + break + case 'set_option': + await setOption(message.key, message.value) + + break + case 'get_js_patterns': + response = { + patterns: wappalyzer.jsPatterns + } + + break + case 'update_theme_mode': + // Sync theme mode to popup. + response = { + themeMode: await getOption('themeMode', false) + } + + break + default: + // Do nothing + } + + if (response) { + port.postMessage({ + id: message.id, + response + }) + } + }) +}) + +wappalyzer.driver.document = document + +/** + * Log messages to console + */ +wappalyzer.driver.log = (message, source, type) => { + const log = ['warn', 'error'].includes(type) ? type : 'log' + + console[log](`[wappalyzer ${type}]`, `[${source}]`, message) // eslint-disable-line no-console +} + +/** + * Display apps + */ +wappalyzer.driver.displayApps = async (detected, meta, context) => { + const { tab } = context + + if (tab === undefined) { + return + } + + tabCache[tab.id] = tabCache[tab.id] || { + detected: [] + } + + tabCache[tab.id].detected = detected + + const pinnedCategory = await getOption('pinnedCategory') + const dynamicIcon = await getOption('dynamicIcon', true) + + let found = false + + // Find the main application to display + ;[pinnedCategory].concat(categoryOrder).forEach((match) => { + Object.keys(detected).forEach((appName) => { + const app = detected[appName] + + app.props.cats.forEach((category) => { + if (category === match && !found) { + let icon = + app.props.icon && dynamicIcon ? app.props.icon : 'default.svg' + + if (/\.svg$/i.test(icon)) { + icon = `converted/${icon.replace(/\.svg$/, '.png')}` + } + + try { + browser.pageAction.setIcon({ + tabId: tab.id, + path: `../images/icons/${icon}` + }) + } catch (e) { + // Firefox for Android does not support setIcon see https://bugzilla.mozilla.org/show_bug.cgi?id=1331746 + } + + found = true + } + }) + }) + }) + + browser.pageAction.show(tab.id) +} + +/** + * Fetch and cache robots.txt for host + */ +wappalyzer.driver.getRobotsTxt = async (host, secure = false) => { + if (robotsTxtQueue[host]) { + return robotsTxtQueue[host] + } + + const tracking = await getOption('tracking', true) + const robotsTxtCache = await getOption('robotsTxtCache', {}) + + robotsTxtQueue[host] = new Promise(async (resolve) => { + if (!tracking) { + return resolve([]) + } + + if (host in robotsTxtCache) { + return resolve(robotsTxtCache[host]) + } + + const timeout = setTimeout(() => resolve([]), 3000) + + let response + + try { + response = await fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { + redirect: 'follow', + mode: 'no-cors' + }) + } catch (error) { + wappalyzer.log(error, 'driver', 'error') + + return resolve([]) + } + + clearTimeout(timeout) + + const robotsTxt = response.ok ? await response.text() : '' + + robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt) + + await setOption('robotsTxtCache', robotsTxtCache) + + delete robotsTxtQueue[host] + + return resolve(robotsTxtCache[host]) + }) + + return robotsTxtQueue[host] +} + +/** + * Anonymously track detected applications for research purposes + */ +wappalyzer.driver.ping = async ( + hostnameCache = { expires: 0, hostnames: {} }, + adCache = [] +) => { + const tracking = await getOption('tracking', true) + const termsAccepted = + userAgent() === 'chrome' || (await getOption('termsAccepted', false)) + + if (tracking && termsAccepted) { + if ( + hostnameCache.hostnames && + Object.keys(hostnameCache.hostnames).length + ) { + post('https://api.wappalyzer.com/ping/v1/', hostnameCache.hostnames) + } + + if (adCache.length) { + post('https://ad.wappalyzer.com/log/wp/', adCache) + } + + await setOption('robotsTxtCache', {}) + } +} + +// Init +;(async () => { + // Technologies + try { + const response = await fetch('../apps.json') + const json = await response.json() + + wappalyzer.apps = json.apps + wappalyzer.categories = json.categories + } catch (error) { + wappalyzer.log(`GET apps.json: ${error.message}`, 'driver', 'error') + } + + wappalyzer.parseJsPatterns() + + categoryOrder = Object.keys(wappalyzer.categories) + .map((categoryId) => parseInt(categoryId, 10)) + .sort( + (a, b) => + wappalyzer.categories[a].priority - wappalyzer.categories[b].priority + ) + + // Version check + const { version } = browser.runtime.getManifest() + const previousVersion = await getOption('version') + const upgradeMessage = await getOption('upgradeMessage', true) + + if (previousVersion === null) { + openTab({ + url: `${wappalyzer.config.websiteURL}installed` + }) + } else if (version !== previousVersion && upgradeMessage) { + openTab({ + url: `${wappalyzer.config.websiteURL}upgraded?v${version}`, + background: true + }) + } + + await setOption('version', version) + + // Hostname cache + wappalyzer.hostnameCache = await getOption('hostnameCache', { + expires: Date.now() + 1000 * 60 * 60 * 24, + hostnames: {} + }) + + // Run content script on all tabs + try { + const tabs = await browser.tabs.query({ + url: ['http://*/*', 'https://*/*'] + }) + + tabs.forEach(async (tab) => { + try { + await browser.tabs.executeScript(tab.id, { + file: '../js/content.js' + }) + } catch (error) { + // + } + }) + } catch (error) { + wappalyzer.log(error, 'driver', 'error') + } +})() diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index 0ff7d491c..ed1e51026 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -1,445 +1,269 @@ -/** - * WebExtension driver - */ - +'use strict' /* eslint-env browser */ -/* global browser, chrome, Wappalyzer */ - -/** global: browser */ -/** global: chrome */ -/** global: fetch */ -/** global: Wappalyzer */ - -const wappalyzer = new Wappalyzer() - -const tabCache = {} -const robotsTxtQueue = {} +/* globals chrome, Wappalyzer, Utils */ -let categoryOrder = [] +const { setTechnologies, setCategories, analyze, resolve, unique } = Wappalyzer +const { promisify, getOption } = Utils -browser.tabs.onRemoved.addListener((tabId) => { - tabCache[tabId] = null -}) - -function userAgent() { - const url = chrome.extension.getURL('/') +const Driver = { + cache: { + hostnames: {}, + robots: {} + }, - if (url.startsWith('moz-')) { - return 'firefox' - } + log(message, source = 'driver', type = 'log') { + // eslint-disable-next-line no-console + console[type](`wappalyzer | ${source} |`, message) + }, - if (url.startsWith('ms-browser')) { - return 'edge' - } + warn(message, source = 'driver') { + Driver.log(message, source, 'warn') + }, - return 'chrome' -} + error(error, source = 'driver') { + Driver.log(error, source, 'error') + }, -/** - * Get a value from localStorage - */ -function getOption(name, defaultValue = null) { - return new Promise(async (resolve, reject) => { - let value = defaultValue + open(url, active = true) { + chrome.tabs.create({ url, active }) + }, + async loadTechnologies() { try { - const option = await browser.storage.local.get(name) + const { apps: technologies, categories } = await ( + await fetch(chrome.extension.getURL('apps.json')) + ).json() - if (option[name] !== undefined) { - value = option[name] - } + setTechnologies(technologies) + setCategories(categories) } catch (error) { - wappalyzer.log(error.message, 'driver', 'error') - - return reject(error.message) + Driver.error(error) } + }, - return resolve(value) - }) -} - -/** - * Set a value in localStorage - */ -function setOption(name, value) { - return new Promise(async (resolve, reject) => { + post(url, body) { try { - await browser.storage.local.set({ [name]: value }) + return fetch(url, { + method: 'POST', + body: JSON.stringify(body) + }) } catch (error) { - wappalyzer.log(error.message, 'driver', 'error') - - return reject(error.message) - } - - return resolve() - }) -} - -/** - * Open a tab - */ -function openTab(args) { - browser.tabs.create({ - url: args.url, - active: args.background === undefined || !args.background - }) -} - -/** - * Make a POST request - */ -async function post(url, body) { - try { - const response = await fetch(url, { - method: 'POST', - body: JSON.stringify(body) - }) - - wappalyzer.log(`POST ${url}: ${response.status}`, 'driver') - } catch (error) { - wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error') - } -} - -// Capture response headers -browser.webRequest.onCompleted.addListener( - async (request) => { - const headers = {} - - if (request.responseHeaders) { - const url = wappalyzer.parseUrl(request.url) - - let tab - - try { - ;[tab] = await browser.tabs.query({ url: [url.href] }) - } catch (error) { - wappalyzer.log(error, 'driver', 'error') - } - - if (tab) { - request.responseHeaders.forEach((header) => { - const name = header.name.toLowerCase() - - headers[name] = headers[name] || [] - - headers[name].push( - (header.value || header.binaryValue || '').toString() - ) - }) - - if ( - headers['content-type'] && - /\/x?html/.test(headers['content-type'][0]) - ) { - wappalyzer.analyze(url, { headers }, { tab }) - } - } + throw new Error(error.message || error.toString()) } }, - { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, - ['responseHeaders'] -) - -browser.runtime.onConnect.addListener((port) => { - port.onMessage.addListener(async (message) => { - if (message.id === undefined) { - return - } - - if (message.id !== 'log') { - wappalyzer.log(`Message from ${port.name}: ${message.id}`, 'driver') - } - const pinnedCategory = await getOption('pinnedCategory') - - const url = wappalyzer.parseUrl(port.sender.tab ? port.sender.tab.url : '') - - const cookies = await browser.cookies.getAll({ - domain: `.${url.hostname}` - }) - - let response - - switch (message.id) { - case 'log': - wappalyzer.log(message.subject, message.source) - - break - case 'init': - wappalyzer.analyze(url, { cookies }, { tab: port.sender.tab }) - - break - case 'analyze': - if (message.subject.html) { - browser.i18n - .detectLanguage(message.subject.html) - .then(({ languages }) => { - const language = languages - .filter(({ percentage }) => percentage >= 75) - .map(({ language: lang }) => lang)[0] - - message.subject.language = language - - wappalyzer.analyze(url, message.subject, { tab: port.sender.tab }) - }) - } else { - wappalyzer.analyze(url, message.subject, { tab: port.sender.tab }) - } - - await setOption('hostnameCache', wappalyzer.hostnameCache) - - break - case 'ad_log': - wappalyzer.cacheDetectedAds(message.subject) - - break - case 'get_apps': - response = { - tabCache: tabCache[message.tab.id], - apps: wappalyzer.apps, - categories: wappalyzer.categories, - pinnedCategory, - termsAccepted: - userAgent() === 'chrome' || - (await getOption('termsAccepted', false)) - } - - break - case 'set_option': - await setOption(message.key, message.value) - - break - case 'get_js_patterns': - response = { - patterns: wappalyzer.jsPatterns - } - - break - case 'update_theme_mode': - // Sync theme mode to popup. - response = { - themeMode: await getOption('themeMode', false) - } + onRuntimeConnect(port) { + port.onMessage.addListener(async ({ func, args }) => { + if (!func) { + return + } - break - default: - // Do nothing - } + Driver.log({ port: port.name, func, args }) - if (response) { port.postMessage({ - id: message.id, - response + func, + args: await Driver[func].call(port.sender, ...(args || [])) }) - } - }) -}) - -wappalyzer.driver.document = document - -/** - * Log messages to console - */ -wappalyzer.driver.log = (message, source, type) => { - const log = ['warn', 'error'].includes(type) ? type : 'log' - - console[log](`[wappalyzer ${type}]`, `[${source}]`, message) // eslint-disable-line no-console -} - -/** - * Display apps - */ -wappalyzer.driver.displayApps = async (detected, meta, context) => { - const { tab } = context - - if (tab === undefined) { - return - } - tabCache[tab.id] = tabCache[tab.id] || { - detected: [] - } + /* + const pinnedCategory = await getOption('pinnedCategory') - tabCache[tab.id].detected = detected + const url = new URL(port.sender.tab.url) - const pinnedCategory = await getOption('pinnedCategory') - const dynamicIcon = await getOption('dynamicIcon', true) + const cookies = await browser.cookies.getAll({ + domain: `.${url.hostname}` + }) - let found = false + let response + + switch (message.id) { + case 'log': + wappalyzer.log(message.subject, message.source) + + break + case 'analyze': + if (message.subject.html) { + browser.i18n + .detectLanguage(message.subject.html) + .then(({ languages }) => { + const language = languages + .filter(({ percentage }) => percentage >= 75) + .map(({ language: lang }) => lang)[0] + + message.subject.language = language + + wappalyzer.analyze(url, message.subject, { + tab: port.sender.tab + }) + }) + } else { + wappalyzer.analyze(url, message.subject, { tab: port.sender.tab }) + } - // Find the main application to display - ;[pinnedCategory].concat(categoryOrder).forEach((match) => { - Object.keys(detected).forEach((appName) => { - const app = detected[appName] + await setOption('hostnameCache', wappalyzer.hostnameCache) + + break + case 'ad_log': + wappalyzer.cacheDetectedAds(message.subject) + + break + case 'get_apps': + response = { + tabCache: tabCache[message.tab.id], + apps: wappalyzer.apps, + categories: wappalyzer.categories, + pinnedCategory, + termsAccepted: + userAgent() === 'chrome' || + (await getOption('termsAccepted', false)) + } - app.props.cats.forEach((category) => { - if (category === match && !found) { - let icon = - app.props.icon && dynamicIcon ? app.props.icon : 'default.svg' + break + case 'set_option': + await setOption(message.key, message.value) - if (/\.svg$/i.test(icon)) { - icon = `converted/${icon.replace(/\.svg$/, '.png')}` + break + case 'get_js_patterns': + response = { + patterns: wappalyzer.jsPatterns } - try { - browser.pageAction.setIcon({ - tabId: tab.id, - path: `../images/icons/${icon}` - }) - } catch (e) { - // Firefox for Android does not support setIcon see https://bugzilla.mozilla.org/show_bug.cgi?id=1331746 + break + case 'update_theme_mode': + // Sync theme mode to popup. + response = { + themeMode: await getOption('themeMode', false) } - found = true - } - }) + break + default: + // Do nothing + } + + if (response) { + port.postMessage({ + id: message.id, + response + }) + } }) - }) + */ + }) + }, - browser.pageAction.show(tab.id) -} + async onWebRequestComplete(request) { + if (request.responseHeaders) { + const headers = {} -/** - * Fetch and cache robots.txt for host - */ -wappalyzer.driver.getRobotsTxt = async (host, secure = false) => { - if (robotsTxtQueue[host]) { - return robotsTxtQueue[host] - } + try { + const url = new URL(request.url) - const tracking = await getOption('tracking', true) - const robotsTxtCache = await getOption('robotsTxtCache', {}) + const [tab] = await promisify(chrome.tabs, 'query', { url: [url.href] }) - robotsTxtQueue[host] = new Promise(async (resolve) => { - if (!tracking) { - return resolve([]) - } + if (tab) { + request.responseHeaders.forEach((header) => { + const name = header.name.toLowerCase() - if (host in robotsTxtCache) { - return resolve(robotsTxtCache[host]) - } + headers[name] = headers[name] || [] - const timeout = setTimeout(() => resolve([]), 3000) + headers[name].push( + (header.value || header.binaryValue || '').toString() + ) + }) - let response + if ( + headers['content-type'] && + /\/x?html/.test(headers['content-type'][0]) + ) { + await Driver.onDetect(url, await analyze(url, { headers }, { tab })) + } + } + } catch (error) { + Driver.error(error) + } + } + }, + async onContentLoad(href, items) { try { - response = await fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { - redirect: 'follow', - mode: 'no-cors' + const url = new URL(href) + + items.cookies = await promisify(chrome.cookies, 'getAll', { + domain: `.${url.hostname}` }) - } catch (error) { - wappalyzer.log(error, 'driver', 'error') - return resolve([]) + await Driver.onDetect(url, await analyze(url, items)) + } catch (error) { + Driver.error(error) } + }, - clearTimeout(timeout) + async onDetect(url, detections = []) { + Driver.cache.hostnames[url.hostname] = unique([ + ...(Driver.cache.hostnames[url.hostname] || []), + ...detections + ]) - const robotsTxt = response.ok ? await response.text() : '' + const resolved = resolve(Driver.cache.hostnames[url.hostname]) - robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt) + await Driver.setIcon(url, resolved) + }, - await setOption('robotsTxtCache', robotsTxtCache) + async setIcon(url, technologies) { + const dynamicIcon = await getOption('dynamicIcon', true) - delete robotsTxtQueue[host] + let icon = 'default.svg' - return resolve(robotsTxtCache[host]) - }) + if (dynamicIcon) { + const pinnedCategory = parseInt(await getOption('pinnedCategory'), 10) - return robotsTxtQueue[host] -} + const pinned = technologies.find(({ categories }) => + categories.some(({ id }) => id === pinnedCategory) + ) -/** - * Anonymously track detected applications for research purposes - */ -wappalyzer.driver.ping = async ( - hostnameCache = { expires: 0, hostnames: {} }, - adCache = [] -) => { - const tracking = await getOption('tracking', true) - const termsAccepted = - userAgent() === 'chrome' || (await getOption('termsAccepted', false)) - - if (tracking && termsAccepted) { - if ( - hostnameCache.hostnames && - Object.keys(hostnameCache.hostnames).length - ) { - post('https://api.wappalyzer.com/ping/v1/', hostnameCache.hostnames) - } + ;({ icon } = + pinned || + technologies.sort(({ categories: a }, { categories: b }) => { + const max = (value) => + value.reduce((max, { priority }) => Math.max(max, priority)) - if (adCache.length) { - post('https://ad.wappalyzer.com/log/wp/', adCache) + return max(a) > max(b) ? -1 : 1 + })[0]) } - await setOption('robotsTxtCache', {}) - } -} - -// Init -;(async () => { - // Technologies - try { - const response = await fetch('../apps.json') - const json = await response.json() - - wappalyzer.apps = json.apps - wappalyzer.categories = json.categories - } catch (error) { - wappalyzer.log(`GET apps.json: ${error.message}`, 'driver', 'error') - } + const tabs = await promisify(chrome.tabs, 'query', { url: [url.href] }) - wappalyzer.parseJsPatterns() + await Promise.all( + tabs.map(async ({ id: tabId }) => { + await promisify(chrome.pageAction, 'setIcon', { + tabId, + path: chrome.extension.getURL(`../images/icons/${icon}`) + }) - categoryOrder = Object.keys(wappalyzer.categories) - .map((categoryId) => parseInt(categoryId, 10)) - .sort( - (a, b) => - wappalyzer.categories[a].priority - wappalyzer.categories[b].priority + chrome.pageAction.show(tabId) + }) ) + }, - // Version check - const { version } = browser.runtime.getManifest() - const previousVersion = await getOption('version') - const upgradeMessage = await getOption('upgradeMessage', true) - - if (previousVersion === null) { - openTab({ - url: `${wappalyzer.config.websiteURL}installed` + async getDetections() { + const [{ url: href }] = await promisify(chrome.tabs, 'query', { + active: true, + currentWindow: true }) - } else if (version !== previousVersion && upgradeMessage) { - openTab({ - url: `${wappalyzer.config.websiteURL}upgraded?v${version}`, - background: true - }) - } - await setOption('version', version) + const url = new URL(href) - // Hostname cache - wappalyzer.hostnameCache = await getOption('hostnameCache', { - expires: Date.now() + 1000 * 60 * 60 * 24, - hostnames: {} - }) + return resolve(Driver.cache.hostnames[url.hostname]) + } +} - // Run content script on all tabs - try { - const tabs = await browser.tabs.query({ - url: ['http://*/*', 'https://*/*'] - }) +;(async function() { + await Driver.loadTechnologies() - tabs.forEach(async (tab) => { - try { - await browser.tabs.executeScript(tab.id, { - file: '../js/content.js' - }) - } catch (error) { - // - } - }) - } catch (error) { - wappalyzer.log(error, 'driver', 'error') - } + chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect) + chrome.webRequest.onCompleted.addListener( + Driver.onWebRequestComplete, + { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, + ['responseHeaders'] + ) })() diff --git a/src/drivers/webextension/js/driver2.js b/src/drivers/webextension/js/driver2.js deleted file mode 100644 index 28883ee66..000000000 --- a/src/drivers/webextension/js/driver2.js +++ /dev/null @@ -1,285 +0,0 @@ -'use strict' -/* eslint-env browser */ -/* globals chrome, Wappalyzer */ - -const { setTechnologies, setCategories, analyze, resolve, unique } = Wappalyzer - -function promisify(context, method, ...args) { - return new Promise((resolve, reject) => { - context[method](...args, (...args) => { - if (chrome.runtime.lastError) { - return reject(chrome.runtime.lastError) - } - - resolve(...args) - }) - }) -} - -const Driver = { - cache: { - hostnames: {}, - robots: {} - }, - - agent: chrome.extension.getURL('/').startsWith('moz-') ? 'firefox' : 'chrome', - - log(message, source = 'driver', type = 'log') { - // eslint-disable-next-line no-console - console[type](`wappalyzer | ${source} |`, message) - }, - - warn(message, source = 'driver') { - Driver.log(message, source, 'warn') - }, - - error(error, source = 'driver') { - Driver.log(error, source, 'error') - }, - - open(url, active = true) { - chrome.tabs.create({ url, active }) - }, - - async loadTechnologies() { - try { - const { apps: technologies, categories } = await ( - await fetch(chrome.extension.getURL('apps.json')) - ).json() - - setTechnologies(technologies) - setCategories(categories) - } catch (error) { - Driver.error(error) - } - }, - - post(url, body) { - try { - return fetch(url, { - method: 'POST', - body: JSON.stringify(body) - }) - } catch (error) { - throw new Error(error.message || error.toString()) - } - }, - - async getOption(name, defaultValue = null) { - try { - const option = await promisify(chrome.storage.local, 'get', name) - - if (option[name] !== undefined) { - return option[name] - } - - return defaultValue - } catch (error) { - throw new Error(error.message || error.toString()) - } - }, - - async setOption(name, value) { - try { - await promisify(chrome.storage.local, 'set', { - [name]: value - }) - } catch (error) { - throw new Error(error.message || error.toString()) - } - }, - - onRuntimeConnect(port) { - port.onMessage.addListener(async (message) => { - const { func, args } = message - - if (!func || !port.sender.tab) { - return - } - - Driver.log(`Message received from ${port.name}: ${func}`) - - await Driver[func](...args) - - /* - const pinnedCategory = await getOption('pinnedCategory') - - const url = new URL(port.sender.tab.url) - - const cookies = await browser.cookies.getAll({ - domain: `.${url.hostname}` - }) - - let response - - switch (message.id) { - case 'log': - wappalyzer.log(message.subject, message.source) - - break - case 'analyze': - if (message.subject.html) { - browser.i18n - .detectLanguage(message.subject.html) - .then(({ languages }) => { - const language = languages - .filter(({ percentage }) => percentage >= 75) - .map(({ language: lang }) => lang)[0] - - message.subject.language = language - - wappalyzer.analyze(url, message.subject, { - tab: port.sender.tab - }) - }) - } else { - wappalyzer.analyze(url, message.subject, { tab: port.sender.tab }) - } - - await setOption('hostnameCache', wappalyzer.hostnameCache) - - break - case 'ad_log': - wappalyzer.cacheDetectedAds(message.subject) - - break - case 'get_apps': - response = { - tabCache: tabCache[message.tab.id], - apps: wappalyzer.apps, - categories: wappalyzer.categories, - pinnedCategory, - termsAccepted: - userAgent() === 'chrome' || - (await getOption('termsAccepted', false)) - } - - break - case 'set_option': - await setOption(message.key, message.value) - - break - case 'get_js_patterns': - response = { - patterns: wappalyzer.jsPatterns - } - - break - case 'update_theme_mode': - // Sync theme mode to popup. - response = { - themeMode: await getOption('themeMode', false) - } - - break - default: - // Do nothing - } - - if (response) { - port.postMessage({ - id: message.id, - response - }) - } - }) - */ - }) - }, - - async onWebRequestComplete(request) { - if (request.responseHeaders) { - const headers = {} - - try { - const url = new URL(request.url) - - const [tab] = await promisify(chrome.tabs, 'query', { url: [url.href] }) - - if (tab) { - request.responseHeaders.forEach((header) => { - const name = header.name.toLowerCase() - - headers[name] = headers[name] || [] - - headers[name].push( - (header.value || header.binaryValue || '').toString() - ) - }) - - if ( - headers['content-type'] && - /\/x?html/.test(headers['content-type'][0]) - ) { - await Driver.onDetect(url, await analyze(url, { headers }, { tab })) - } - } - } catch (error) { - Driver.error(error) - } - } - }, - - async onContentLoad(href, items) { - try { - const url = new URL(href) - - items.cookies = await promisify(chrome.cookies, 'getAll', { - domain: `.${url.hostname}` - }) - - await Driver.onDetect(url, await analyze(url, items)) - } catch (error) { - Driver.error(error) - } - }, - - async onDetect(url, detections = []) { - Driver.cache.hostnames[url.hostname] = unique([ - ...(Driver.cache.hostnames[url.hostname] || []), - ...detections - ]) - - const resolved = resolve(Driver.cache.hostnames[url.hostname]) - - const pinnedCategory = parseInt( - await Driver.getOption('pinnedCategory'), - 10 - ) - - const pinned = resolved.find(({ categories }) => - categories.some(({ id }) => id === pinnedCategory) - ) - - const { icon } = - pinned || - resolved.sort(({ categories: a }, { categories: b }) => { - const max = (value) => - value.reduce((max, { priority }) => Math.max(max, priority)) - - return max(a) > max(b) ? -1 : 1 - })[0] - - const tabs = await promisify(chrome.tabs, 'query', { url: [url.href] }) - - await Promise.all( - tabs.map(({ id: tabId }) => - promisify(chrome.pageAction, 'setIcon', { - tabId, - path: chrome.extension.getURL(`../images/icons/${icon}`) - }) - ) - ) - } -} - -;(async function() { - await Driver.loadTechnologies() - - chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect) - chrome.webRequest.onCompleted.addListener( - Driver.onWebRequestComplete, - { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, - ['responseHeaders'] - ) -})() diff --git a/src/drivers/webextension/js/popup.bak.js b/src/drivers/webextension/js/popup.bak.js new file mode 100644 index 000000000..251df8200 --- /dev/null +++ b/src/drivers/webextension/js/popup.bak.js @@ -0,0 +1,333 @@ +'use strict' +/* eslint-env browser */ +/* globals chrome */ + +let pinnedCategory = null +let termsAccepted = false + +const port = chrome.runtime.connect({ + name: 'popup.js' +}) + +function slugify(string) { + return string + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/--+/g, '-') + .replace(/(?:^-|-$)/, '') +} + +function i18n() { + const nodes = document.querySelectorAll('[data-i18n]') + + Array.prototype.forEach.call(nodes, (node) => { + node.innerHTML = browser.i18n.getMessage(node.dataset.i18n) + }) +} + +function replaceDom(domTemplate) { + const container = document.getElementsByClassName('container')[0] + + while (container.firstChild) { + container.removeChild(container.firstChild) + } + + container.appendChild(jsonToDOM(domTemplate, document, {})) + + i18n() + + Array.from( + document.querySelectorAll('.detected__category-pin-wrapper') + ).forEach((pin) => { + pin.addEventListener('click', () => { + const categoryId = parseInt(pin.dataset.categoryId, 10) + + if (categoryId === pinnedCategory) { + pin.className = 'detected__category-pin-wrapper' + + pinnedCategory = null + } else { + const active = document.querySelector( + '.detected__category-pin-wrapper--active' + ) + + if (active) { + active.className = 'detected__category-pin-wrapper' + } + + pin.className = + 'detected__category-pin-wrapper detected__category-pin-wrapper--active' + + pinnedCategory = categoryId + } + + port.postMessage({ + id: 'set_option', + key: 'pinnedCategory', + value: pinnedCategory + }) + }) + }) + + Array.from(document.querySelectorAll('a')).forEach((link) => { + link.addEventListener('click', () => { + browser.tabs.create({ url: link.href }) + + return false + }) + }) +} + +function replaceDomWhenReady(dom) { + if (/complete|interactive|loaded/.test(document.readyState)) { + replaceDom(dom) + } else { + document.addEventListener('DOMContentLoaded', () => { + replaceDom(dom) + }) + } +} + +function appsToDomTemplate(response) { + let template = [] + + if (response.tabCache && Object.keys(response.tabCache.detected).length > 0) { + const categories = {} + + // Group apps by category + for (const appName in response.tabCache.detected) { + response.apps[appName].cats.forEach((cat) => { + categories[cat] = categories[cat] || { + name: response.categories[cat].name, + apps: [] + } + + categories[cat].apps[appName] = appName + }) + } + + for (const cat in categories) { + const apps = [] + + for (const appName in categories[cat].apps) { + const { confidenceTotal, version } = response.tabCache.detected[appName] + + apps.push([ + 'a', + { + class: 'detected__app', + href: `https://www.wappalyzer.com/technologies/${slugify( + categories[cat].name + )}/${slugify(appName)}` + }, + [ + 'img', + { + class: 'detected__app-icon', + src: `../images/icons/${response.apps[appName].icon || + 'default.svg'}` + } + ], + [ + 'span', + { + class: 'detected__app-name' + }, + appName + ], + version + ? [ + 'span', + { + class: 'detected__app-version' + }, + version + ] + : null, + confidenceTotal < 100 + ? [ + 'span', + { + class: 'detected__app-confidence' + }, + `${confidenceTotal}% sure` + ] + : null + ]) + } + + template.push([ + 'div', + { + class: 'detected__category' + }, + [ + 'div', + { + class: 'detected__category-name' + }, + [ + 'a', + { + class: 'detected__category-link', + href: `https://www.wappalyzer.com/categories/${slugify( + response.categories[cat].name + )}` + }, + browser.i18n.getMessage(`categoryName${cat}`) + ], + [ + 'span', + { + class: `detected__category-pin-wrapper${ + parseInt(pinnedCategory, 10) === parseInt(cat, 10) + ? ' detected__category-pin-wrapper--active' + : '' + }`, + 'data-category-id': cat, + title: browser.i18n.getMessage('categoryPin') + }, + [ + 'img', + { + class: 'detected__category-pin detected__category-pin--active', + src: '../images/pin-active.svg' + } + ], + [ + 'img', + { + class: + 'detected__category-pin detected__category-pin--inactive', + src: '../images/pin.svg' + } + ] + ] + ], + [ + 'div', + { + class: 'detected__apps' + }, + apps + ] + ]) + } + + template = [ + 'div', + { + class: 'detected' + }, + template + ] + } else { + template = [ + 'div', + { + class: 'empty' + }, + [ + 'span', + { + class: 'empty__text' + }, + browser.i18n.getMessage('noAppsDetected') + ] + ] + } + + return template +} + +async function getApps() { + try { + const tabs = await browser.tabs.query({ + active: true, + currentWindow: true + }) + + const url = new URL(tabs[0].url) + + document.querySelector( + '.footer__link' + ).href = `https://www.wappalyzer.com/alerts/manage?url=${encodeURIComponent( + `${url.protocol}//${url.hostname}` + )}` + + port.postMessage({ + id: 'get_apps', + tab: tabs[0] + }) + } catch (error) { + console.error(error) // eslint-disable-line no-console + } +} + +/** + * Async function to update body class based on option. + */ +function getThemeMode() { + try { + port.postMessage({ + id: 'update_theme_mode' + }) + } catch (error) { + console.error(error) // eslint-disable-line no-console + } +} + +/** + * Update theme mode based on browser option. + * @param {object} res Response from port listener. + */ +function updateThemeMode(res) { + if (res.hasOwnProperty('themeMode') && res.themeMode !== false) { + document.body.classList.add('theme-mode-sync') + } +} + +function displayApps(response) { + pinnedCategory = response.pinnedCategory // eslint-disable-line prefer-destructuring + termsAccepted = response.termsAccepted // eslint-disable-line prefer-destructuring + + if (termsAccepted) { + replaceDomWhenReady(appsToDomTemplate(response)) + } else { + i18n() + + const wrapper = document.querySelector('.terms__wrapper') + + document.querySelector('.terms__accept').addEventListener('click', () => { + port.postMessage({ + id: 'set_option', + key: 'termsAccepted', + value: true + }) + + wrapper.classList.remove('terms__wrapper--active') + + getApps() + }) + + wrapper.classList.add('terms__wrapper--active') + } +} + +port.onMessage.addListener((message) => { + switch (message.id) { + case 'get_apps': + displayApps(message.response) + + break + case 'update_theme_mode': + updateThemeMode(message.response) + + break + default: + // Do nothing + } +}) + +getThemeMode() +getApps() diff --git a/src/drivers/webextension/js/popup.js b/src/drivers/webextension/js/popup.js index 3aa561403..9cf2b60c8 100644 --- a/src/drivers/webextension/js/popup.js +++ b/src/drivers/webextension/js/popup.js @@ -1,335 +1,110 @@ +'use strict' /* eslint-env browser */ -/* global browser, jsonToDOM */ +/* globals chrome, Utils */ -/** global: browser */ -/** global: jsonToDOM */ +const { agent, getOption } = Utils -let pinnedCategory = null -let termsAccepted = false +const Popup = { + port: chrome.runtime.connect({ name: 'popup.js' }), -const port = browser.runtime.connect({ - name: 'popup.js' -}) - -function slugify(string) { - return string - .toLowerCase() - .replace(/[^a-z0-9-]/g, '-') - .replace(/--+/g, '-') - .replace(/(?:^-|-$)/, '') -} - -function i18n() { - const nodes = document.querySelectorAll('[data-i18n]') - - Array.prototype.forEach.call(nodes, (node) => { - node.innerHTML = browser.i18n.getMessage(node.dataset.i18n) - }) -} - -function replaceDom(domTemplate) { - const container = document.getElementsByClassName('container')[0] - - while (container.firstChild) { - container.removeChild(container.firstChild) - } - - container.appendChild(jsonToDOM(domTemplate, document, {})) - - i18n() - - Array.from( - document.querySelectorAll('.detected__category-pin-wrapper') - ).forEach((pin) => { - pin.addEventListener('click', () => { - const categoryId = parseInt(pin.dataset.categoryId, 10) - - if (categoryId === pinnedCategory) { - pin.className = 'detected__category-pin-wrapper' - - pinnedCategory = null - } else { - const active = document.querySelector( - '.detected__category-pin-wrapper--active' - ) - - if (active) { - active.className = 'detected__category-pin-wrapper' - } - - pin.className = - 'detected__category-pin-wrapper detected__category-pin-wrapper--active' - - pinnedCategory = categoryId - } - - port.postMessage({ - id: 'set_option', - key: 'pinnedCategory', - value: pinnedCategory - }) - }) - }) - - Array.from(document.querySelectorAll('a')).forEach((link) => { - link.addEventListener('click', () => { - browser.tabs.create({ url: link.href }) + async init() { + Popup.templates = Array.from( + document.querySelectorAll('[data-template]') + ).reduce((templates, template) => { + templates[template.dataset.template] = template - return false - }) - }) -} - -function replaceDomWhenReady(dom) { - if (/complete|interactive|loaded/.test(document.readyState)) { - replaceDom(dom) - } else { - document.addEventListener('DOMContentLoaded', () => { - replaceDom(dom) - }) - } -} - -function appsToDomTemplate(response) { - let template = [] + return templates + }, {}) - if (response.tabCache && Object.keys(response.tabCache.detected).length > 0) { - const categories = {} + Popup.log(agent) - // Group apps by category - for (const appName in response.tabCache.detected) { - response.apps[appName].cats.forEach((cat) => { - categories[cat] = categories[cat] || { - name: response.categories[cat].name, - apps: [] - } + const termsAccepted = + agent === 'chrome' || (await getOption('termsAccepted', false)) - categories[cat].apps[appName] = appName - }) - } - - for (const cat in categories) { - const apps = [] + if (termsAccepted) { + document.querySelector('.terms').style.display = 'none' - for (const appName in categories[cat].apps) { - const { confidenceTotal, version } = response.tabCache.detected[appName] + Popup.driver('getDetections') + } else { + document.querySelector('.detections').style.display = 'none' - apps.push([ - 'a', - { - class: 'detected__app', - href: `https://www.wappalyzer.com/technologies/${slugify( - categories[cat].name - )}/${slugify(appName)}` - }, - [ - 'img', - { - class: 'detected__app-icon', - src: `../images/icons/${response.apps[appName].icon || - 'default.svg'}` - } - ], - [ - 'span', - { - class: 'detected__app-name' - }, - appName - ], - version - ? [ - 'span', - { - class: 'detected__app-version' - }, - version - ] - : null, - confidenceTotal < 100 - ? [ - 'span', - { - class: 'detected__app-confidence' - }, - `${confidenceTotal}% sure` - ] - : null - ]) - } - - template.push([ - 'div', - { - class: 'detected__category' - }, - [ - 'div', - { - class: 'detected__category-name' - }, - [ - 'a', - { - class: 'detected__category-link', - href: `https://www.wappalyzer.com/categories/${slugify( - response.categories[cat].name - )}` - }, - browser.i18n.getMessage(`categoryName${cat}`) - ], - [ - 'span', - { - class: `detected__category-pin-wrapper${ - parseInt(pinnedCategory, 10) === parseInt(cat, 10) - ? ' detected__category-pin-wrapper--active' - : '' - }`, - 'data-category-id': cat, - title: browser.i18n.getMessage('categoryPin') - }, - [ - 'img', - { - class: 'detected__category-pin detected__category-pin--active', - src: '../images/pin-active.svg' - } - ], - [ - 'img', - { - class: - 'detected__category-pin detected__category-pin--inactive', - src: '../images/pin.svg' - } - ] - ] - ], - [ - 'div', - { - class: 'detected__apps' - }, - apps - ] - ]) + Popup.i18n() } + }, - template = [ - 'div', - { - class: 'detected' - }, - template - ] - } else { - template = [ - 'div', - { - class: 'empty' - }, - [ - 'span', - { - class: 'empty__text' - }, - browser.i18n.getMessage('noAppsDetected') - ] - ] - } + driver(func, ...args) { + Popup.port.postMessage({ func, args }) + }, - return template -} + log(message) { + Popup.driver('log', message, 'popup.js') + }, -async function getApps() { - try { - const tabs = await browser.tabs.query({ - active: true, - currentWindow: true - }) + i18n() { + Array.from(document.querySelectorAll('[data-i18n]')).forEach( + (node) => (node.innerHTML = chrome.i18n.getMessage(node.dataset.i18n)) + ) + }, - const url = new URL(tabs[0].url) + categorise(technologies) { + return Object.values( + technologies.reduce((categories, technology) => { + technology.categories.forEach((category) => { + categories[category.id] = categories[category.id] || { + ...category, + technologies: [] + } - document.querySelector( - '.footer__link' - ).href = `https://www.wappalyzer.com/alerts/manage?url=${encodeURIComponent( - `${url.protocol}//${url.hostname}` - )}` + categories[category.id].technologies.push(technology) + }) - port.postMessage({ - id: 'get_apps', - tab: tabs[0] - }) - } catch (error) { - console.error(error) // eslint-disable-line no-console - } -} + return categories + }, {}) + ) + }, -/** - * Async function to update body class based on option. - */ -function getThemeMode() { - try { - port.postMessage({ - id: 'update_theme_mode' - }) - } catch (error) { - console.error(error) // eslint-disable-line no-console - } -} + onGetDetections(detections) { + Popup.categorise(detections).forEach( + ({ name, slug: categorySlug, technologies }) => { + const categoryNode = Popup.templates.category.cloneNode(true) -/** - * Update theme mode based on browser option. - * @param {object} res Response from port listener. - */ -function updateThemeMode(res) { - if (res.hasOwnProperty('themeMode') && res.themeMode !== false) { - document.body.classList.add('theme-mode-sync') - } -} + const link = categoryNode.querySelector('.category__link') -function displayApps(response) { - pinnedCategory = response.pinnedCategory // eslint-disable-line prefer-destructuring - termsAccepted = response.termsAccepted // eslint-disable-line prefer-destructuring + link.href = `https://www.wappalyzer.com/technologie/${categorySlug}` + link.textContent = name - if (termsAccepted) { - replaceDomWhenReady(appsToDomTemplate(response)) - } else { - i18n() + technologies.forEach(({ name, slug, icon, website }) => { + Popup.log(name) + const technologyNode = Popup.templates.technology.cloneNode(true) - const wrapper = document.querySelector('.terms__wrapper') + const link = technologyNode.querySelector('.technology__link') - document.querySelector('.terms__accept').addEventListener('click', () => { - port.postMessage({ - id: 'set_option', - key: 'termsAccepted', - value: true - }) + link.href = `https://www.wappalyzer.com/technologie/${categorySlug}/${slug}` + link.textContent = name - wrapper.classList.remove('terms__wrapper--active') + categoryNode + .querySelector('.technologies') + .appendChild(technologyNode) + }) - getApps() - }) + document.querySelector('.detections').appendChild(categoryNode) + } + ) - wrapper.classList.add('terms__wrapper--active') + Popup.i18n() } } -port.onMessage.addListener((message) => { - switch (message.id) { - case 'get_apps': - displayApps(message.response) +Popup.port.onMessage.addListener(({ func, args }) => { + const onFunc = `on${func.charAt(0).toUpperCase() + func.slice(1)}` - break - case 'update_theme_mode': - updateThemeMode(message.response) - - break - default: - // Do nothing + if (Popup[onFunc]) { + Popup[onFunc](args) } }) -getThemeMode() -getApps() +if (/complete|interactive|loaded/.test(document.readyState)) { + Popup.init() +} else { + document.addEventListener('DOMContentLoaded', Popup.init) +} diff --git a/src/drivers/webextension/js/utils.js b/src/drivers/webextension/js/utils.js new file mode 100644 index 000000000..b35190ff3 --- /dev/null +++ b/src/drivers/webextension/js/utils.js @@ -0,0 +1,47 @@ +'use strict' +/* eslint-env browser */ +/* globals chrome */ + +const Utils = { + agent: chrome.extension.getURL('/').startsWith('moz-') ? 'firefox' : 'chrome', + + promisify(context, method, ...args) { + return new Promise((resolve, reject) => { + context[method](...args, (...args) => { + if (chrome.runtime.lastError) { + return reject(chrome.runtime.lastError) + } + + resolve(...args) + }) + }) + }, + + open(url, active = true) { + chrome.tabs.create({ url, active }) + }, + + async getOption(name, defaultValue = null) { + try { + const option = await Utils.promisify(chrome.storage.local, 'get', name) + + if (option[name] !== undefined) { + return option[name] + } + + return defaultValue + } catch (error) { + throw new Error(error.message || error.toString()) + } + }, + + async setOption(name, value) { + try { + await Utils.promisify(chrome.storage.local, 'set', { + [name]: value + }) + } catch (error) { + throw new Error(error.message || error.toString()) + } + } +} diff --git a/src/drivers/webextension/js/wappalyzer.bak.js b/src/drivers/webextension/js/wappalyzer.bak.js new file mode 100644 index 000000000..e69399e87 --- /dev/null +++ b/src/drivers/webextension/js/wappalyzer.bak.js @@ -0,0 +1,727 @@ +const validation = { + hostname: /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/, + hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/ +} + +/** + * Enclose string in array + */ +function asArray(value) { + return Array.isArray(value) ? value : [value] +} + +/** + * + */ +function asyncForEach(iterable, iterator) { + return Promise.all( + (iterable || []).map( + (item) => + new Promise((resolve) => setTimeout(() => resolve(iterator(item)), 1)) + ) + ) +} + +/** + * Mark application as detected, set confidence and version + */ +function addDetected(app, pattern, type, value, key) { + app.detected = true + + // Set confidence level + app.confidence[`${type} ${key ? `${key} ` : ''}${pattern.regex}`] = + pattern.confidence === undefined ? 100 : parseInt(pattern.confidence, 10) + + // Detect version number + if (pattern.version) { + const versions = [] + const matches = pattern.regex.exec(value) + + let { version } = pattern + + if (matches) { + matches.forEach((match, i) => { + // Parse ternary operator + const ternary = new RegExp(`\\\\${i}\\?([^:]+):(.*)$`).exec(version) + + if (ternary && ternary.length === 3) { + version = version.replace(ternary[0], match ? ternary[1] : ternary[2]) + } + + // Replace back references + version = version + .trim() + .replace(new RegExp(`\\\\${i}`, 'g'), match || '') + }) + + if (version && !versions.includes(version)) { + versions.push(version) + } + + if (versions.length) { + // Use the longest detected version number + app.version = versions.reduce((a, b) => (a.length > b.length ? a : b)) + } + } + } +} + +function resolveExcludes(apps, detected) { + const excludes = [] + const detectedApps = Object.assign({}, apps, detected) + + // Exclude app in detected apps only + Object.keys(detectedApps).forEach((appName) => { + const app = detectedApps[appName] + + if (app.props.excludes) { + asArray(app.props.excludes).forEach((excluded) => { + excludes.push(excluded) + }) + } + }) + + // Remove excluded applications + Object.keys(apps).forEach((appName) => { + if (excludes.includes(appName)) { + delete apps[appName] + } + }) +} + +class Application { + constructor(name, props, detected) { + this.confidence = {} + this.confidenceTotal = 0 + this.detected = Boolean(detected) + this.excludes = [] + this.name = name + this.props = props + this.version = '' + } + + /** + * Calculate confidence total + */ + getConfidence() { + let total = 0 + + Object.keys(this.confidence).forEach((id) => { + total += this.confidence[id] + }) + + this.confidenceTotal = Math.min(total, 100) + + return this.confidenceTotal + } +} + +class Wappalyzer { + constructor() { + this.apps = {} + this.categories = {} + this.driver = {} + this.jsPatterns = {} + this.detected = {} + this.hostnameCache = { + expires: Date.now() + 1000 * 60 * 60 * 24, + hostnames: {} + } + this.adCache = [] + + this.config = { + websiteURL: 'https://www.wappalyzer.com/', + twitterURL: 'https://twitter.com/Wappalyzer', + githubURL: 'https://github.com/AliasIO/Wappalyzer' + } + } + + /** + * Log messages to console + */ + log(message, source, type) { + if (this.driver.log) { + this.driver.log(message, source || '', type || 'debug') + } + } + + analyze(url, data, context) { + const apps = {} + const promises = [] + const startTime = new Date() + const { scripts, cookies, headers, js } = data + + let { html } = data + + if (this.detected[url.canonical] === undefined) { + this.detected[url.canonical] = {} + } + + const metaTags = [] + + // Additional information + let language = null + + if (html) { + if (typeof html !== 'string') { + html = '' + } + + let matches = data.html.match( + new RegExp(']*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"', 'i') + ) + + language = matches && matches.length ? matches[1] : data.language || null + + // Meta tags + const regex = /]+>/gi + + do { + matches = regex.exec(html) + + if (!matches) { + break + } + + metaTags.push(matches[0]) + } while (matches) + } + + Object.keys(this.apps).forEach((appName) => { + apps[appName] = + this.detected[url.canonical] && this.detected[url.canonical][appName] + ? this.detected[url.canonical][appName] + : new Application(appName, this.apps[appName]) + + const app = apps[appName] + + promises.push(this.analyzeUrl(app, url)) + + if (html) { + promises.push(this.analyzeHtml(app, html)) + promises.push(this.analyzeMeta(app, metaTags)) + } + + if (scripts) { + promises.push(this.analyzeScripts(app, scripts)) + } + + if (cookies) { + promises.push(this.analyzeCookies(app, cookies)) + } + + if (headers) { + promises.push(this.analyzeHeaders(app, headers)) + } + }) + + if (js) { + Object.keys(js).forEach((appName) => { + if (typeof js[appName] !== 'function') { + promises.push(this.analyzeJs(apps[appName], js[appName])) + } + }) + } + + return new Promise(async (resolve) => { + await Promise.all(promises) + + Object.keys(apps).forEach((appName) => { + const app = apps[appName] + + if (!app.detected || !app.getConfidence()) { + delete apps[app.name] + } + }) + + resolveExcludes(apps, this.detected[url]) + this.resolveImplies(apps, url.canonical) + + this.cacheDetectedApps(apps, url.canonical) + this.trackDetectedApps(apps, url, language) + + this.log( + `Processing ${Object.keys(data).join(', ')} took ${( + (new Date() - startTime) / + 1000 + ).toFixed(2)}s (${url.hostname})`, + 'core' + ) + + if (Object.keys(apps).length) { + this.log( + `Identified ${Object.keys(apps).join(', ')} (${url.hostname})`, + 'core' + ) + } + + this.driver.displayApps( + this.detected[url.canonical], + { language }, + context + ) + + return resolve() + }) + } + + /** + * Cache detected ads + */ + cacheDetectedAds(ad) { + this.adCache.push(ad) + } + + /** + * + */ + robotsTxtAllows(url) { + return new Promise(async (resolve, reject) => { + const parsed = this.parseUrl(url) + + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + return reject() + } + + const robotsTxt = await this.driver.getRobotsTxt( + parsed.host, + parsed.protocol === 'https:' + ) + + if ( + robotsTxt.some( + (disallowedPath) => parsed.pathname.indexOf(disallowedPath) === 0 + ) + ) { + return reject() + } + + return resolve() + }) + } + + /** + * Parse a URL + */ + parseUrl(url) { + const a = this.driver.document.createElement('a') + + a.href = url + + a.canonical = `${a.protocol}//${a.host}${a.pathname}` + + return a + } + + /** + * + */ + static parseRobotsTxt(robotsTxt) { + const disallow = [] + + let userAgent + + robotsTxt.split('\n').forEach((line) => { + let matches = /^User-agent:\s*(.+)$/i.exec(line.trim()) + + if (matches) { + userAgent = matches[1].toLowerCase() + } else if (userAgent === '*' || userAgent === 'wappalyzer') { + matches = /^Disallow:\s*(.+)$/i.exec(line.trim()) + + if (matches) { + disallow.push(matches[1]) + } + } + }) + + return disallow + } + + /** + * + */ + ping() { + if ( + !this.hostnameCache.hostnames || + Object.keys(this.hostnameCache.hostnames).length > 50 || + this.hostnameCache.expires < Date.now() + ) { + this.driver.ping(this.hostnameCache) + + this.hostnameCache = { + expires: Date.now() + 1000 * 60 * 60 * 24, + hostnames: {} + } + } + + if (this.adCache.length > 50) { + this.driver.ping(undefined, this.adCache) + + this.adCache = [] + } + } + + /** + * Parse apps.json patterns + */ + parsePatterns(patterns) { + if (!patterns) { + return [] + } + + let parsed = {} + + // Convert string to object containing array containing string + if (typeof patterns === 'string' || Array.isArray(patterns)) { + patterns = { + main: asArray(patterns) + } + } + + Object.keys(patterns).forEach((key) => { + parsed[key] = [] + + asArray(patterns[key]).forEach((pattern) => { + const attrs = {} + + pattern.split('\\;').forEach((attr, i) => { + if (i) { + // Key value pairs + attr = attr.split(':') + + if (attr.length > 1) { + attrs[attr.shift()] = attr.join(':') + } + } else { + attrs.string = attr + + try { + attrs.regex = new RegExp(attr.replace('/', '/'), 'i') // Escape slashes in regular expression + } catch (error) { + attrs.regex = new RegExp() + + this.log(`${error.message}: ${attr}`, 'error', 'core') + } + } + }) + + parsed[key].push(attrs) + }) + }) + + // Convert back to array if the original pattern list was an array (or string) + if ('main' in parsed) { + parsed = parsed.main + } + + return parsed + } + + /** + * Parse JavaScript patterns + */ + parseJsPatterns() { + Object.keys(this.apps).forEach((appName) => { + if (this.apps[appName].js) { + this.jsPatterns[appName] = this.parsePatterns(this.apps[appName].js) + } + }) + } + + resolveImplies(apps, url) { + let checkImplies = true + + const resolve = (appName) => { + const app = apps[appName] + + if (app && app.props.implies) { + asArray(app.props.implies).forEach((implied) => { + ;[implied] = this.parsePatterns(implied) + + if (!this.apps[implied.string]) { + this.log( + `Implied application ${implied.string} does not exist`, + 'core', + 'warn' + ) + + return + } + + if (!(implied.string in apps)) { + apps[implied.string] = + this.detected[url] && this.detected[url][implied.string] + ? this.detected[url][implied.string] + : new Application( + implied.string, + this.apps[implied.string], + true + ) + + checkImplies = true + } + + // Apply app confidence to implied app + Object.keys(app.confidence).forEach((id) => { + apps[implied.string].confidence[`${id} implied by ${appName}`] = + app.confidence[id] * + (implied.confidence === undefined ? 1 : implied.confidence / 100) + }) + }) + } + } + + // Implied applications + // Run several passes as implied apps may imply other apps + while (checkImplies) { + checkImplies = false + + Object.keys(apps).forEach(resolve) + } + } + + /** + * Cache detected applications + */ + cacheDetectedApps(apps, url) { + Object.keys(apps).forEach((appName) => { + const app = apps[appName] + + // Per URL + this.detected[url][appName] = app + + Object.keys(app.confidence).forEach((id) => { + this.detected[url][appName].confidence[id] = app.confidence[id] + }) + }) + + if (this.driver.ping instanceof Function) { + this.ping() + } + } + + /** + * Track detected applications + */ + trackDetectedApps(apps, url, language) { + if (!(this.driver.ping instanceof Function)) { + return + } + + const hostname = `${url.protocol}//${url.hostname}` + + Object.keys(apps).forEach((appName) => { + const app = apps[appName] + + if (this.detected[url.canonical][appName].getConfidence() >= 100) { + if ( + validation.hostname.test(url.hostname) && + !validation.hostnameBlacklist.test(url.hostname) + ) { + if (!(hostname in this.hostnameCache.hostnames)) { + this.hostnameCache.hostnames[hostname] = { + applications: {}, + meta: {} + } + } + + if ( + !(appName in this.hostnameCache.hostnames[hostname].applications) + ) { + this.hostnameCache.hostnames[hostname].applications[appName] = { + hits: 0 + } + } + + this.hostnameCache.hostnames[hostname].applications[appName].hits += 1 + + if (apps[appName].version) { + this.hostnameCache.hostnames[hostname].applications[ + appName + ].version = app.version + } + } + } + }) + + if (hostname in this.hostnameCache.hostnames) { + this.hostnameCache.hostnames[hostname].meta.language = language + } + + this.ping() + } + + /** + * Analyze URL + */ + analyzeUrl(app, url) { + const patterns = this.parsePatterns(app.props.url) + + if (!patterns.length) { + return Promise.resolve() + } + + return asyncForEach(patterns, (pattern) => { + if (pattern.regex.test(url.canonical)) { + addDetected(app, pattern, 'url', url.canonical) + } + }) + } + + /** + * Analyze HTML + */ + analyzeHtml(app, html) { + const patterns = this.parsePatterns(app.props.html) + + if (!patterns.length) { + return Promise.resolve() + } + + return asyncForEach(patterns, (pattern) => { + if (pattern.regex.test(html)) { + addDetected(app, pattern, 'html', html) + } + }) + } + + /** + * Analyze script tag + */ + analyzeScripts(app, scripts) { + const patterns = this.parsePatterns(app.props.script) + + if (!patterns.length) { + return Promise.resolve() + } + + return asyncForEach(patterns, (pattern) => { + scripts.forEach((uri) => { + if (pattern.regex.test(uri)) { + addDetected(app, pattern, 'script', uri) + } + }) + }) + } + + /** + * Analyze meta tag + */ + analyzeMeta(app, metaTags) { + const patterns = this.parsePatterns(app.props.meta) + const promises = [] + + if (!app.props.meta) { + return Promise.resolve() + } + + metaTags.forEach((match) => { + Object.keys(patterns).forEach((meta) => { + const r = new RegExp(`(?:name|property)=["']${meta}["']`, 'i') + + if (r.test(match)) { + const content = match.match(/content=("|')([^"']+)("|')/i) + + promises.push( + asyncForEach(patterns[meta], (pattern) => { + if ( + content && + content.length === 4 && + pattern.regex.test(content[2]) + ) { + addDetected(app, pattern, 'meta', content[2], meta) + } + }) + ) + } + }) + }) + + return Promise.all(promises) + } + + /** + * Analyze response headers + */ + analyzeHeaders(app, headers) { + const patterns = this.parsePatterns(app.props.headers) + const promises = [] + + Object.keys(patterns).forEach((headerName) => { + if (typeof patterns[headerName] !== 'function') { + promises.push( + asyncForEach(patterns[headerName], (pattern) => { + headerName = headerName.toLowerCase() + + if (headerName in headers) { + headers[headerName].forEach((headerValue) => { + if (pattern.regex.test(headerValue)) { + addDetected(app, pattern, 'headers', headerValue, headerName) + } + }) + } + }) + ) + } + }) + + return promises ? Promise.all(promises) : Promise.resolve() + } + + /** + * Analyze cookies + */ + analyzeCookies(app, cookies) { + const patterns = this.parsePatterns(app.props.cookies) + const promises = [] + + Object.keys(patterns).forEach((cookieName) => { + if (typeof patterns[cookieName] !== 'function') { + const cookieNameLower = cookieName.toLowerCase() + + promises.push( + asyncForEach(patterns[cookieName], (pattern) => { + const cookie = cookies.find( + (_cookie) => _cookie.name.toLowerCase() === cookieNameLower + ) + + if (cookie && pattern.regex.test(cookie.value)) { + addDetected(app, pattern, 'cookies', cookie.value, cookieName) + } + }) + ) + } + }) + + return promises ? Promise.all(promises) : Promise.resolve() + } + + /** + * Analyze JavaScript variables + */ + analyzeJs(app, results) { + const promises = [] + + Object.keys(results).forEach((string) => { + if (typeof results[string] !== 'function') { + promises.push( + asyncForEach(Object.keys(results[string]), (index) => { + const pattern = this.jsPatterns[app.name][string][index] + const value = results[string][index] + + if (pattern && pattern.regex.test(value)) { + addDetected(app, pattern, 'js', value, string) + } + }) + ) + } + }) + + return promises ? Promise.all(promises) : Promise.resolve() + } +} + +if (typeof module === 'object') { + module.exports = Wappalyzer +} diff --git a/src/drivers/webextension/js/wappalyzer2.js b/src/drivers/webextension/js/wappalyzer2.js deleted file mode 100644 index ee8049a80..000000000 --- a/src/drivers/webextension/js/wappalyzer2.js +++ /dev/null @@ -1,326 +0,0 @@ -'use strict' - -const Wappalyzer = { - technologies: [], - categories: [], - - slugify(string) { - return string - .toLowerCase() - .replace(/[^a-z0-9-]/g, '-') - .replace(/--+/g, '-') - .replace(/(?:^-|-$)/, '') - }, - - unique(detections) { - return detections.filter( - ({ technology: { name }, pattern: { regex } }, index) => { - return ( - detections.findIndex( - ({ technology: { name: _name }, pattern: { regex: _regex } }) => - name === _name && (!regex || regex === _regex) - ) === index - ) - } - ) - }, - - getTechnology(name) { - return Wappalyzer.technologies.find(({ name: _name }) => name === _name) - }, - - getCategory(id) { - return Wappalyzer.categories.find(({ id: _id }) => id === _id) - }, - - resolve(detections) { - const resolved = detections.reduce((resolved, { technology }) => { - if ( - resolved.findIndex( - ({ technology: { name } }) => name === technology.name - ) === -1 - ) { - let version = '' - let confidence = 0 - - detections.forEach(({ technology: { name }, pattern, match }) => { - if (name === technology.name) { - const versionValue = Wappalyzer.resolveVersion(pattern, match) - - confidence = Math.min(100, confidence + pattern.confidence) - version = - versionValue.length > version.length && versionValue.length <= 10 - ? versionValue - : version - } - }) - - resolved.push({ technology, confidence, version }) - } - - return resolved - }, []) - - Wappalyzer.resolveExcludes(resolved) - Wappalyzer.resolveImplies(resolved) - - return resolved.map( - ({ - technology: { name, slug, categories, icon, website }, - confidence, - version - }) => ({ - name, - slug, - categories: categories.map((id) => Wappalyzer.getCategory(id)), - confidence, - version, - icon, - website - }) - ) - }, - - resolveVersion({ version, regex }, match) { - let resolved = version - - if (version) { - const matches = regex.exec(match) - - if (matches) { - matches.forEach((match, index) => { - // Parse ternary operator - const ternary = new RegExp(`\\\\${index}\\?([^:]+):(.*)$`).exec( - version - ) - - if (ternary && ternary.length === 3) { - resolved = version.replace( - ternary[0], - match ? ternary[1] : ternary[2] - ) - } - - // Replace back references - resolved = resolved - .trim() - .replace(new RegExp(`\\\\${index}`, 'g'), match || '') - }) - } - } - - return resolved - }, - - resolveExcludes(resolved) { - resolved.forEach(({ technology }) => { - technology.excludes.forEach((name) => { - const excluded = Wappalyzer.getTechnology(name) - - const index = resolved.findIndex(({ name }) => name === excluded.name) - - if (index === -1) { - resolved.splice(index, 1) - } - }) - }) - }, - - resolveImplies(resolved) { - let done = false - - while (!done) { - resolved.forEach(({ technology, confidence }) => { - done = true - - technology.implies.forEach((name) => { - const implied = Wappalyzer.getTechnology(name) - - if ( - resolved.findIndex( - ({ technology: { name } }) => name === implied.name - ) === -1 - ) { - resolved.push({ technology: implied, confidence, version: '' }) - - done = false - } - }) - }) - } - }, - - async analyze(url, { html, meta, headers, cookies, scripts, js }) { - const oo = Wappalyzer.analyzeOneToOne - const om = Wappalyzer.analyzeOneToMany - const mm = Wappalyzer.analyzeManyToMany - - const flatten = (array) => Array.prototype.concat.apply([], array) - - try { - const detections = flatten( - flatten( - await Promise.all( - Wappalyzer.technologies.map((technology) => - Promise.all([ - oo(technology, 'url', url), - oo(technology, 'html', html), - om(technology, 'meta', meta), - mm(technology, 'headers', headers), - om(technology, 'cookies', cookies), - om(technology, 'scripts', scripts) - ]) - ) - ) - ) - ).filter((technology) => technology) - - return detections - } catch (error) { - throw new Error(error.message || error.toString()) - } - }, - - setTechnologies(data) { - const transform = Wappalyzer.transformPatterns - - Wappalyzer.technologies = Object.keys(data).reduce((technologies, name) => { - const { - cats, - url, - html, - meta, - headers, - cookies, - script, - implies, - excludes, - icon, - website - } = data[name] - - technologies.push({ - name, - categories: cats || [], - slug: Wappalyzer.slugify(name), - url: transform(url), - headers: transform(headers), - cookies: transform(cookies), - html: transform(html), - meta: transform(meta), - scripts: transform(script), - implies: typeof implies === 'string' ? [implies] : implies || [], - excludes: typeof excludes === 'string' ? [excludes] : excludes || [], - icon: icon || 'default.svg', - website: website || '' - }) - - return technologies - }, []) - }, - - setCategories(data) { - Wappalyzer.categories = Object.keys(data) - .reduce((categories, id) => { - const category = data[id] - - categories.push({ - id: parseInt(id, 10), - slug: Wappalyzer.slugify(category.name), - ...category - }) - - return categories - }, []) - .sort(({ priority: a }, { priority: b }) => (a > b ? -1 : 0)) - }, - - transformPatterns(patterns) { - if (!patterns) { - return [] - } - - const toArray = (value) => (Array.isArray(value) ? value : [value]) - - if (typeof patterns === 'string' || Array.isArray(patterns)) { - patterns = { main: patterns } - } - - const parsed = Object.keys(patterns).reduce((parsed, key) => { - parsed[key.toLowerCase()] = toArray(patterns[key]).map((pattern) => { - const { regex, confidence, version } = pattern - .split('\\;') - .reduce((attrs, attr, i) => { - if (i) { - // Key value pairs - attr = attr.split(':') - - if (attr.length > 1) { - attrs[attr.shift()] = attr.join(':') - } - } else { - // Escape slashes in regular expression - attrs.regex = new RegExp(attr.replace(/\//g, '\\/'), 'i') - } - - return attrs - }, {}) - - return { - regex, - confidence: parseInt(confidence || 100, 10), - version: version || '' - } - }) - - return parsed - }, {}) - - return 'main' in parsed ? parsed.main : parsed - }, - - analyzeOneToOne(technology, type, value) { - return technology[type].reduce((technologies, pattern) => { - if (pattern.regex.test(value)) { - technologies.push({ technology, pattern, match: value }) - } - - return technologies - }, []) - }, - - analyzeOneToMany(technology, type, items = []) { - return items.reduce((technologies, { key, value }) => { - const patterns = technology[type][key] || [] - - patterns.forEach((pattern) => { - if (pattern.regex.test(value)) { - technologies.push({ technology, pattern, match: value }) - } - }) - - return technologies - }, []) - }, - - analyzeManyToMany(technology, type, items = {}) { - return Object.keys(technology[type]).reduce((technologies, key) => { - const patterns = technology[type][key] || [] - const values = items[key] || [] - - patterns.forEach((pattern) => { - values.forEach((value) => { - if (pattern.regex.test(value)) { - technologies.push({ technology, pattern, match: value }) - } - }) - }) - - return technologies - }, []) - } -} - -if (typeof module !== 'undefined') { - module.exports = Wappalyzer -}