From 2aca8275c5b73673f3cd0dc70fa8bcdd98050520 Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Wed, 3 Jun 2020 15:43:28 +1000 Subject: [PATCH 1/7] Refactoring --- src/drivers/webextension/html/background.html | 5 +- src/drivers/webextension/js/content.js | 47 +-- src/drivers/webextension/js/driver.js | 8 +- src/drivers/webextension/js/driver2.js | 285 +++++++++++++++ src/drivers/webextension/js/wappalyzer2.js | 326 ++++++++++++++++++ src/wappalyzer.js | 8 - src/wappalyzer.spec.js | 289 ++++++++-------- 7 files changed, 785 insertions(+), 183 deletions(-) create mode 100644 src/drivers/webextension/js/driver2.js create mode 100644 src/drivers/webextension/js/wappalyzer2.js diff --git a/src/drivers/webextension/html/background.html b/src/drivers/webextension/html/background.html index 8a03c7a9d..f6147dc56 100644 --- a/src/drivers/webextension/html/background.html +++ b/src/drivers/webextension/html/background.html @@ -4,9 +4,8 @@ - - - + + diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js index 66d620f4a..3be7fa391 100644 --- a/src/drivers/webextension/js/content.js +++ b/src/drivers/webextension/js/content.js @@ -1,20 +1,14 @@ -/** global: browser */ -/** global: XMLSerializer */ - -/* global browser */ +'use strict' /* eslint-env browser */ +/* globals chrome */ -const port = browser.runtime.connect({ - name: 'content.js' -}) +const port = chrome.runtime.connect({ name: 'content.js' }) ;(async function() { - if (typeof browser !== 'undefined' && typeof document.body !== 'undefined') { + if (typeof chrome !== 'undefined' && typeof document.body !== 'undefined') { await new Promise((resolve) => setTimeout(resolve, 1000)) try { - port.postMessage({ id: 'init' }) - // HTML let html = new XMLSerializer().serializeToString(document) @@ -23,9 +17,7 @@ const port = browser.runtime.connect({ const maxRows = 3000 const rows = html.length / maxCols - let i - - for (i = 0; i < rows; i += 1) { + for (let i = 0; i < rows; i += 1) { if (i < maxRows / 2 || i > rows - maxRows / 2) { chunks.push(html.slice(i * maxCols, (i + 1) * maxCols)) } @@ -34,13 +26,23 @@ const port = browser.runtime.connect({ html = chunks.join('\n') // Scripts - const scripts = Array.prototype.slice - .apply(document.scripts) - .filter((script) => script.src) - .map((script) => script.src) + const scripts = Array.from(document.scripts) + .filter(({ src }) => src) + .map(({ src }) => src) .filter((script) => script.indexOf('data:text/javascript;') !== 0) - port.postMessage({ id: 'analyze', subject: { html, scripts } }) + // Meta + const meta = Array.from(document.querySelectorAll('meta')) + .map((meta) => ({ + key: meta.getAttribute('name') || meta.getAttribute('property'), + value: meta.getAttribute('content') + })) + .filter(({ value }) => value) + + port.postMessage({ + func: 'onContentLoad', + args: [location.href, { html, scripts, meta }] + }) // JavaScript variables const script = document.createElement('script') @@ -53,7 +55,10 @@ const port = browser.runtime.connect({ window.removeEventListener('message', onMessage) - port.postMessage({ id: 'analyze', subject: { js: event.data.js } }) + port.postMessage({ + func: 'analyze', + args: [new URL(location.href), { js: event.data.js }] + }) script.remove() } @@ -63,11 +68,11 @@ const port = browser.runtime.connect({ port.postMessage({ id: 'get_js_patterns' }) } - script.setAttribute('src', browser.extension.getURL('js/inject.js')) + script.setAttribute('src', chrome.extension.getURL('js/inject.js')) document.body.appendChild(script) } catch (error) { - port.postMessage({ id: 'log', subject: error }) + port.postMessage({ func: 'error', args: [error, 'content.js'] }) } } })() diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index 9742f8906..0ff7d491c 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -410,10 +410,10 @@ wappalyzer.driver.ping = async ( url: `${wappalyzer.config.websiteURL}installed` }) } else if (version !== previousVersion && upgradeMessage) { - // openTab({ - // url: `${wappalyzer.config.websiteURL}upgraded?v${version}`, - // background: true - // }) + openTab({ + url: `${wappalyzer.config.websiteURL}upgraded?v${version}`, + background: true + }) } await setOption('version', version) diff --git a/src/drivers/webextension/js/driver2.js b/src/drivers/webextension/js/driver2.js new file mode 100644 index 000000000..28883ee66 --- /dev/null +++ b/src/drivers/webextension/js/driver2.js @@ -0,0 +1,285 @@ +'use strict' +/* eslint-env browser */ +/* globals chrome, Wappalyzer */ + +const { setTechnologies, setCategories, analyze, resolve, unique } = Wappalyzer + +function promisify(context, method, ...args) { + return new Promise((resolve, reject) => { + context[method](...args, (...args) => { + if (chrome.runtime.lastError) { + return reject(chrome.runtime.lastError) + } + + resolve(...args) + }) + }) +} + +const Driver = { + cache: { + hostnames: {}, + robots: {} + }, + + agent: chrome.extension.getURL('/').startsWith('moz-') ? 'firefox' : 'chrome', + + log(message, source = 'driver', type = 'log') { + // eslint-disable-next-line no-console + console[type](`wappalyzer | ${source} |`, message) + }, + + warn(message, source = 'driver') { + Driver.log(message, source, 'warn') + }, + + error(error, source = 'driver') { + Driver.log(error, source, 'error') + }, + + open(url, active = true) { + chrome.tabs.create({ url, active }) + }, + + async loadTechnologies() { + try { + const { apps: technologies, categories } = await ( + await fetch(chrome.extension.getURL('apps.json')) + ).json() + + setTechnologies(technologies) + setCategories(categories) + } catch (error) { + Driver.error(error) + } + }, + + post(url, body) { + try { + return fetch(url, { + method: 'POST', + body: JSON.stringify(body) + }) + } catch (error) { + throw new Error(error.message || error.toString()) + } + }, + + async getOption(name, defaultValue = null) { + try { + const option = await promisify(chrome.storage.local, 'get', name) + + if (option[name] !== undefined) { + return option[name] + } + + return defaultValue + } catch (error) { + throw new Error(error.message || error.toString()) + } + }, + + async setOption(name, value) { + try { + await promisify(chrome.storage.local, 'set', { + [name]: value + }) + } catch (error) { + throw new Error(error.message || error.toString()) + } + }, + + onRuntimeConnect(port) { + port.onMessage.addListener(async (message) => { + const { func, args } = message + + if (!func || !port.sender.tab) { + return + } + + Driver.log(`Message received from ${port.name}: ${func}`) + + await Driver[func](...args) + + /* + const pinnedCategory = await getOption('pinnedCategory') + + const url = new URL(port.sender.tab.url) + + const cookies = await browser.cookies.getAll({ + domain: `.${url.hostname}` + }) + + let response + + switch (message.id) { + case 'log': + wappalyzer.log(message.subject, message.source) + + break + case 'analyze': + if (message.subject.html) { + browser.i18n + .detectLanguage(message.subject.html) + .then(({ languages }) => { + const language = languages + .filter(({ percentage }) => percentage >= 75) + .map(({ language: lang }) => lang)[0] + + message.subject.language = language + + wappalyzer.analyze(url, message.subject, { + tab: port.sender.tab + }) + }) + } else { + wappalyzer.analyze(url, message.subject, { tab: port.sender.tab }) + } + + await setOption('hostnameCache', wappalyzer.hostnameCache) + + break + case 'ad_log': + wappalyzer.cacheDetectedAds(message.subject) + + break + case 'get_apps': + response = { + tabCache: tabCache[message.tab.id], + apps: wappalyzer.apps, + categories: wappalyzer.categories, + pinnedCategory, + termsAccepted: + userAgent() === 'chrome' || + (await getOption('termsAccepted', false)) + } + + break + case 'set_option': + await setOption(message.key, message.value) + + break + case 'get_js_patterns': + response = { + patterns: wappalyzer.jsPatterns + } + + break + case 'update_theme_mode': + // Sync theme mode to popup. + response = { + themeMode: await getOption('themeMode', false) + } + + break + default: + // Do nothing + } + + if (response) { + port.postMessage({ + id: message.id, + response + }) + } + }) + */ + }) + }, + + async onWebRequestComplete(request) { + if (request.responseHeaders) { + const headers = {} + + try { + const url = new URL(request.url) + + const [tab] = await promisify(chrome.tabs, 'query', { url: [url.href] }) + + if (tab) { + request.responseHeaders.forEach((header) => { + const name = header.name.toLowerCase() + + headers[name] = headers[name] || [] + + headers[name].push( + (header.value || header.binaryValue || '').toString() + ) + }) + + if ( + headers['content-type'] && + /\/x?html/.test(headers['content-type'][0]) + ) { + await Driver.onDetect(url, await analyze(url, { headers }, { tab })) + } + } + } catch (error) { + Driver.error(error) + } + } + }, + + async onContentLoad(href, items) { + try { + const url = new URL(href) + + items.cookies = await promisify(chrome.cookies, 'getAll', { + domain: `.${url.hostname}` + }) + + await Driver.onDetect(url, await analyze(url, items)) + } catch (error) { + Driver.error(error) + } + }, + + async onDetect(url, detections = []) { + Driver.cache.hostnames[url.hostname] = unique([ + ...(Driver.cache.hostnames[url.hostname] || []), + ...detections + ]) + + const resolved = resolve(Driver.cache.hostnames[url.hostname]) + + const pinnedCategory = parseInt( + await Driver.getOption('pinnedCategory'), + 10 + ) + + const pinned = resolved.find(({ categories }) => + categories.some(({ id }) => id === pinnedCategory) + ) + + const { icon } = + pinned || + resolved.sort(({ categories: a }, { categories: b }) => { + const max = (value) => + value.reduce((max, { priority }) => Math.max(max, priority)) + + return max(a) > max(b) ? -1 : 1 + })[0] + + const tabs = await promisify(chrome.tabs, 'query', { url: [url.href] }) + + await Promise.all( + tabs.map(({ id: tabId }) => + promisify(chrome.pageAction, 'setIcon', { + tabId, + path: chrome.extension.getURL(`../images/icons/${icon}`) + }) + ) + ) + } +} + +;(async function() { + await Driver.loadTechnologies() + + chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect) + chrome.webRequest.onCompleted.addListener( + Driver.onWebRequestComplete, + { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, + ['responseHeaders'] + ) +})() diff --git a/src/drivers/webextension/js/wappalyzer2.js b/src/drivers/webextension/js/wappalyzer2.js new file mode 100644 index 000000000..ee8049a80 --- /dev/null +++ b/src/drivers/webextension/js/wappalyzer2.js @@ -0,0 +1,326 @@ +'use strict' + +const Wappalyzer = { + technologies: [], + categories: [], + + slugify(string) { + return string + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/--+/g, '-') + .replace(/(?:^-|-$)/, '') + }, + + unique(detections) { + return detections.filter( + ({ technology: { name }, pattern: { regex } }, index) => { + return ( + detections.findIndex( + ({ technology: { name: _name }, pattern: { regex: _regex } }) => + name === _name && (!regex || regex === _regex) + ) === index + ) + } + ) + }, + + getTechnology(name) { + return Wappalyzer.technologies.find(({ name: _name }) => name === _name) + }, + + getCategory(id) { + return Wappalyzer.categories.find(({ id: _id }) => id === _id) + }, + + resolve(detections) { + const resolved = detections.reduce((resolved, { technology }) => { + if ( + resolved.findIndex( + ({ technology: { name } }) => name === technology.name + ) === -1 + ) { + let version = '' + let confidence = 0 + + detections.forEach(({ technology: { name }, pattern, match }) => { + if (name === technology.name) { + const versionValue = Wappalyzer.resolveVersion(pattern, match) + + confidence = Math.min(100, confidence + pattern.confidence) + version = + versionValue.length > version.length && versionValue.length <= 10 + ? versionValue + : version + } + }) + + resolved.push({ technology, confidence, version }) + } + + return resolved + }, []) + + Wappalyzer.resolveExcludes(resolved) + Wappalyzer.resolveImplies(resolved) + + return resolved.map( + ({ + technology: { name, slug, categories, icon, website }, + confidence, + version + }) => ({ + name, + slug, + categories: categories.map((id) => Wappalyzer.getCategory(id)), + confidence, + version, + icon, + website + }) + ) + }, + + resolveVersion({ version, regex }, match) { + let resolved = version + + if (version) { + const matches = regex.exec(match) + + if (matches) { + matches.forEach((match, index) => { + // Parse ternary operator + const ternary = new RegExp(`\\\\${index}\\?([^:]+):(.*)$`).exec( + version + ) + + if (ternary && ternary.length === 3) { + resolved = version.replace( + ternary[0], + match ? ternary[1] : ternary[2] + ) + } + + // Replace back references + resolved = resolved + .trim() + .replace(new RegExp(`\\\\${index}`, 'g'), match || '') + }) + } + } + + return resolved + }, + + resolveExcludes(resolved) { + resolved.forEach(({ technology }) => { + technology.excludes.forEach((name) => { + const excluded = Wappalyzer.getTechnology(name) + + const index = resolved.findIndex(({ name }) => name === excluded.name) + + if (index === -1) { + resolved.splice(index, 1) + } + }) + }) + }, + + resolveImplies(resolved) { + let done = false + + while (!done) { + resolved.forEach(({ technology, confidence }) => { + done = true + + technology.implies.forEach((name) => { + const implied = Wappalyzer.getTechnology(name) + + if ( + resolved.findIndex( + ({ technology: { name } }) => name === implied.name + ) === -1 + ) { + resolved.push({ technology: implied, confidence, version: '' }) + + done = false + } + }) + }) + } + }, + + async analyze(url, { html, meta, headers, cookies, scripts, js }) { + const oo = Wappalyzer.analyzeOneToOne + const om = Wappalyzer.analyzeOneToMany + const mm = Wappalyzer.analyzeManyToMany + + const flatten = (array) => Array.prototype.concat.apply([], array) + + try { + const detections = flatten( + flatten( + await Promise.all( + Wappalyzer.technologies.map((technology) => + Promise.all([ + oo(technology, 'url', url), + oo(technology, 'html', html), + om(technology, 'meta', meta), + mm(technology, 'headers', headers), + om(technology, 'cookies', cookies), + om(technology, 'scripts', scripts) + ]) + ) + ) + ) + ).filter((technology) => technology) + + return detections + } catch (error) { + throw new Error(error.message || error.toString()) + } + }, + + setTechnologies(data) { + const transform = Wappalyzer.transformPatterns + + Wappalyzer.technologies = Object.keys(data).reduce((technologies, name) => { + const { + cats, + url, + html, + meta, + headers, + cookies, + script, + implies, + excludes, + icon, + website + } = data[name] + + technologies.push({ + name, + categories: cats || [], + slug: Wappalyzer.slugify(name), + url: transform(url), + headers: transform(headers), + cookies: transform(cookies), + html: transform(html), + meta: transform(meta), + scripts: transform(script), + implies: typeof implies === 'string' ? [implies] : implies || [], + excludes: typeof excludes === 'string' ? [excludes] : excludes || [], + icon: icon || 'default.svg', + website: website || '' + }) + + return technologies + }, []) + }, + + setCategories(data) { + Wappalyzer.categories = Object.keys(data) + .reduce((categories, id) => { + const category = data[id] + + categories.push({ + id: parseInt(id, 10), + slug: Wappalyzer.slugify(category.name), + ...category + }) + + return categories + }, []) + .sort(({ priority: a }, { priority: b }) => (a > b ? -1 : 0)) + }, + + transformPatterns(patterns) { + if (!patterns) { + return [] + } + + const toArray = (value) => (Array.isArray(value) ? value : [value]) + + if (typeof patterns === 'string' || Array.isArray(patterns)) { + patterns = { main: patterns } + } + + const parsed = Object.keys(patterns).reduce((parsed, key) => { + parsed[key.toLowerCase()] = toArray(patterns[key]).map((pattern) => { + const { regex, confidence, version } = pattern + .split('\\;') + .reduce((attrs, attr, i) => { + if (i) { + // Key value pairs + attr = attr.split(':') + + if (attr.length > 1) { + attrs[attr.shift()] = attr.join(':') + } + } else { + // Escape slashes in regular expression + attrs.regex = new RegExp(attr.replace(/\//g, '\\/'), 'i') + } + + return attrs + }, {}) + + return { + regex, + confidence: parseInt(confidence || 100, 10), + version: version || '' + } + }) + + return parsed + }, {}) + + return 'main' in parsed ? parsed.main : parsed + }, + + analyzeOneToOne(technology, type, value) { + return technology[type].reduce((technologies, pattern) => { + if (pattern.regex.test(value)) { + technologies.push({ technology, pattern, match: value }) + } + + return technologies + }, []) + }, + + analyzeOneToMany(technology, type, items = []) { + return items.reduce((technologies, { key, value }) => { + const patterns = technology[type][key] || [] + + patterns.forEach((pattern) => { + if (pattern.regex.test(value)) { + technologies.push({ technology, pattern, match: value }) + } + }) + + return technologies + }, []) + }, + + analyzeManyToMany(technology, type, items = {}) { + return Object.keys(technology[type]).reduce((technologies, key) => { + const patterns = technology[type][key] || [] + const values = items[key] || [] + + patterns.forEach((pattern) => { + values.forEach((value) => { + if (pattern.regex.test(value)) { + technologies.push({ technology, pattern, match: value }) + } + }) + }) + + return technologies + }, []) + } +} + +if (typeof module !== 'undefined') { + module.exports = Wappalyzer +} diff --git a/src/wappalyzer.js b/src/wappalyzer.js index 043975167..e69399e87 100644 --- a/src/wappalyzer.js +++ b/src/wappalyzer.js @@ -1,11 +1,3 @@ -/** - * Wappalyzer v5 - * - * Created by Elbert Alias - * - * License: GPLv3 http://www.gnu.org/licenses/gpl-3.0.txt - */ - const validation = { hostname: /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/, hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/ diff --git a/src/wappalyzer.spec.js b/src/wappalyzer.spec.js index f51d7fb10..0df917d63 100644 --- a/src/wappalyzer.spec.js +++ b/src/wappalyzer.spec.js @@ -1,51 +1,50 @@ /* eslint-env mocha */ -const { assert, expect } = require('chai'); -const Wappalyzer = require('../src/wappalyzer'); +const { assert, expect } = require('chai') +const Wappalyzer = require('../src/wappalyzer') const appsJson = { appUrl: { - url: 'test', + url: 'test' }, appCookies: { cookies: { - test: 'test', - }, + test: 'test' + } }, appUppercaseCookies: { cookies: { - Test: 'Test', - }, + Test: 'Test' + } }, appHeaders: { headers: { - 'X-Powered-By': 'test', - }, + 'X-Powered-By': 'test' + } }, appHtml: { html: 'test v(\\d)\\;confidence:50\\;version:\\1', implies: 'appImplies', - excludes: 'appExcludes', + excludes: 'appExcludes' }, appMeta: { meta: { - generator: 'test', - }, + generator: 'test' + } }, appScript: { - script: 'test', + script: 'test' }, appJs: { js: { - key: 'value', - }, - }, - appImplies: { + key: 'value' + } }, + appImplies: {}, appExcludes: { - html: 'test', - }, -}; + html: 'test' + } +} const driverData = { cookies: [ @@ -53,92 +52,86 @@ const driverData = { name: 'test', value: 'test', domain: '', - path: '', - }, + path: '' + } ], headers: { - 'x-powered-by': [ - 'test', - ], + 'x-powered-by': ['test'] }, html: ' html test v1', - scripts: [ - 'test', - ], + scripts: ['test'], js: { appJs: { - key: [ - 'value', - ], - }, - }, -}; + key: ['value'] + } + } +} describe('Wappalyzer', () => { describe('#analyze()', () => { - let apps; + let apps before(async () => { - const wappalyzer = new Wappalyzer(); + const wappalyzer = new Wappalyzer() - wappalyzer.apps = appsJson; + wappalyzer.apps = appsJson - wappalyzer.parseJsPatterns(); + wappalyzer.parseJsPatterns() wappalyzer.driver.displayApps = (detected) => { - apps = detected; - }; + apps = detected + } - await wappalyzer.analyze({ canonical: 'test' }, driverData); - }); + await wappalyzer.analyze({ canonical: 'test' }, driverData) + }) it('should identify technologies using URLs', () => { - expect(apps).to.have.any.keys('appUrl'); - }); + expect(apps).to.have.any.keys('appUrl') + }) it('should identify technologies using HTML', () => { - expect(apps).to.have.any.keys('appHtml'); - }); + expect(apps).to.have.any.keys('appHtml') + }) it('should identify technologies using meta tags', () => { - expect(apps).to.have.any.keys('appMeta'); - }); + expect(apps).to.have.any.keys('appMeta') + }) it('should identify technologies using script URLs', () => { - expect(apps).to.have.any.keys('appScript'); - }); + expect(apps).to.have.any.keys('appScript') + }) it('should identify technologies using headers', () => { - expect(apps).to.have.any.keys('appHeaders'); - }); + expect(apps).to.have.any.keys('appHeaders') + }) it('should identify technologies using cookies', () => { - expect(apps).to.have.any.keys('appCookies'); - }); + expect(apps).to.have.any.keys('appCookies') + }) it('should identify technologies using uppercase named cookies', () => { - expect(apps).to.have.any.keys('appUppercaseCookies'); - }); + expect(apps).to.have.any.keys('appUppercaseCookies') + }) it('should identify technologies using JavaScript', () => { - expect(apps).to.have.any.keys('appJs'); - }); + expect(apps).to.have.any.keys('appJs') + }) it('should return the implied technology', () => { - expect(apps).to.have.any.keys('appImplies'); - }); + expect(apps).to.have.any.keys('appImplies') + }) it('should not return the excluded technology', () => { - expect(apps).to.not.have.any.keys('appExcludes'); - }); + expect(apps).to.not.have.any.keys('appExcludes') + }) it('should return the confidence value', () => { - assert.equal(apps.appHtml.confidenceTotal, 50); - }); + assert.equal(apps.appHtml.confidenceTotal, 50) + }) it('should return the version number', () => { - assert.equal(apps.appHtml.version, '1'); - }); + assert.equal(apps.appHtml.version, '1') + }) it('should analyze html', async () => { const html = ` @@ -156,123 +149,125 @@ describe('Wappalyzer', () => { - `; - const wappalyzer = new Wappalyzer(); + ` + const wappalyzer = new Wappalyzer() wappalyzer.apps = { - "Google Tag Manager": { - "html": [ - "googletagmanager\\.com/ns\\.html[^>]+>", - "" + 'Google Tag Manager': { + html: [ + 'googletagmanager\\.com/ns\\.html[^>]+>', + '' ] } - }; - var applications = null; + } + let applications = null wappalyzer.driver = { - log () {}, - displayApps (detectedMap) { - applications = detectedMap; + log() {}, + displayApps(detectedMap) { + applications = detectedMap } - }; + } - await wappalyzer.analyze({ canonical: 'example.com' }, { html }); - assert.equal(applications['Google Tag Manager'].name, 'Google Tag Manager'); - }); + await wappalyzer.analyze({ canonical: 'example.com' }, { html }) + assert.equal( + applications['Google Tag Manager'].name, + 'Google Tag Manager' + ) + }) it('should analyze scripts', async () => { const scripts = [ 'http://www.google-analytics.com/analytics.js', 'http://example.com/assets/js/jquery.min.js' - ]; - const wappalyzer = new Wappalyzer(); + ] + const wappalyzer = new Wappalyzer() wappalyzer.apps = { - "Google Analytics": { - "cats": [ - 10 - ], - "script": "google-analytics\\.com\\/(?:ga|urchin|(analytics))\\.js\\;version:\\1?UA:" + 'Google Analytics': { + cats: [10], + script: + 'google-analytics\\.com\\/(?:ga|urchin|(analytics))\\.js\\;version:\\1?UA:' }, - "jQuery": { - "script": [ - "jquery(?:\\-|\\.)([\\d.]*\\d)[^/]*\\.js\\;version:\\1", - "/([\\d.]+)/jquery(?:\\.min)?\\.js\\;version:\\1", - "jquery.*\\.js(?:\\?ver(?:sion)?=([\\d.]+))?\\;version:\\1" + jQuery: { + script: [ + 'jquery(?:\\-|\\.)([\\d.]*\\d)[^/]*\\.js\\;version:\\1', + '/([\\d.]+)/jquery(?:\\.min)?\\.js\\;version:\\1', + 'jquery.*\\.js(?:\\?ver(?:sion)?=([\\d.]+))?\\;version:\\1' ] } - }; - var applications = null; + } + let applications = null wappalyzer.driver = { - log () {}, - displayApps (detectedMap) { - applications = detectedMap; + log() {}, + displayApps(detectedMap) { + applications = detectedMap } - }; + } - await wappalyzer.analyze({ canonical: 'example.com' }, { scripts }); - assert.equal(applications['Google Analytics'].name, 'Google Analytics'); - assert.equal(applications['jQuery'].name, 'jQuery'); - }); + await wappalyzer.analyze({ canonical: 'example.com' }, { scripts }) + assert.equal(applications['Google Analytics'].name, 'Google Analytics') + assert.equal(applications.jQuery.name, 'jQuery') + }) it('should analyze headers', async () => { const headers = { - 'date': [ 'Thu, 01 Feb 2018 11:34:18 GMT' ], - 'connection': [ 'keep-alive' ], - 'x-powered-by': [ 'Express'], - 'etag': [ 'W/125-1jQLmiya7mfec43xR3Eb3pjdu64s' ], - 'content-length': [ '293' ], - 'content-type': [ 'text/html; charset=utf-8' ] - }; - const wappalyzer = new Wappalyzer(); + date: ['Thu, 01 Feb 2018 11:34:18 GMT'], + connection: ['keep-alive'], + 'x-powered-by': ['Express'], + etag: ['W/125-1jQLmiya7mfec43xR3Eb3pjdu64s'], + 'content-length': ['293'], + 'content-type': ['text/html; charset=utf-8'] + } + const wappalyzer = new Wappalyzer() wappalyzer.apps = { - "Express": { - "headers": { - "X-Powered-By": "^Express$" + Express: { + headers: { + 'X-Powered-By': '^Express$' } } - }; - var applications = null; + } + let applications = null wappalyzer.driver = { - log () {}, - displayApps (detectedMap) { - applications = detectedMap; + log() {}, + displayApps(detectedMap) { + applications = detectedMap } - }; + } - await wappalyzer.analyze({ canonical: 'example.com' }, { headers }); - assert.equal(applications['Express'].name, 'Express'); - }); + await wappalyzer.analyze({ canonical: 'example.com' }, { headers }) + assert.equal(applications.Express.name, 'Express') + }) it('should analyze js globals', async () => { const js = { - 'Moment.js': { 'moment': { '0': true } }, - 'Google Font API': { 'WebFonts': { '0': true } } - }; - const wappalyzer = new Wappalyzer(); + 'Moment.js': { moment: { '0': true } }, + 'Google Font API': { WebFonts: { '0': true } } + } + const wappalyzer = new Wappalyzer() wappalyzer.apps = { - "Moment.js": { - "js": { - "moment": "", - "moment.version": "(.*)\\;version:\\1" + 'Moment.js': { + js: { + moment: '', + 'moment.version': '(.*)\\;version:\\1' } }, - "Google Font API": { - "js": { - "WebFonts": "" + 'Google Font API': { + js: { + WebFonts: '' } } - }; - var applications = null; + } + let applications = null wappalyzer.driver = { - log () {}, - displayApps (detectedMap) { - applications = detectedMap; + log() {}, + displayApps(detectedMap) { + applications = detectedMap } - }; + } - wappalyzer.parseJsPatterns(); - await wappalyzer.analyze({ canonical: 'example.com' }, { js }); + wappalyzer.parseJsPatterns() + await wappalyzer.analyze({ canonical: 'example.com' }, { js }) - assert.equal(applications['Google Font API'].name, 'Google Font API'); - assert.equal(applications['Moment.js'].name, 'Moment.js'); - }); - }); -}); + assert.equal(applications['Google Font API'].name, 'Google Font API') + assert.equal(applications['Moment.js'].name, 'Moment.js') + }) + }) +}) From 97b39037e7fc906199f1b30ba7742e70d3818a8a Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Thu, 4 Jun 2020 09:11:33 +1000 Subject: [PATCH 2/7] Fix for relative redirects --- src/drivers/npm/driver.js | 2 +- src/drivers/npm/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index d6958a696..fcc6dd6be 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -329,7 +329,7 @@ class Site { if (response.status() >= 300 && response.status() < 400) { if (this.headers.location) { - url = new URL(this.headers.location.slice(-1)) + url = new URL(this.headers.location.slice(-1), url) } } else { responseReceived = true diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index a9c0a503d..c016d3fb0 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -13,7 +13,7 @@ "software" ], "homepage": "https://www.wappalyzer.com", - "version": "5.10.3", + "version": "6.0.3", "author": "Wappalyzer", "license": "MIT", "repository": { From 376345aafb0d0590cc94f8f985351028e74a893c Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Thu, 4 Jun 2020 15:03:53 +1000 Subject: [PATCH 3/7] Refactoring --- src/drivers/webextension/css/popup.css | 221 +----- src/drivers/webextension/html/background.html | 5 +- src/drivers/webextension/html/options.html | 2 +- src/drivers/webextension/html/popup.html | 30 +- src/drivers/webextension/js/driver.bak.js | 445 +++++++++++ src/drivers/webextension/js/driver.js | 580 +++++--------- src/drivers/webextension/js/driver2.js | 285 ------- src/drivers/webextension/js/popup.bak.js | 333 ++++++++ src/drivers/webextension/js/popup.js | 379 ++------- src/drivers/webextension/js/utils.js | 47 ++ src/drivers/webextension/js/wappalyzer.bak.js | 727 ++++++++++++++++++ src/drivers/webextension/js/wappalyzer2.js | 326 -------- 12 files changed, 1885 insertions(+), 1495 deletions(-) create mode 100644 src/drivers/webextension/js/driver.bak.js delete mode 100644 src/drivers/webextension/js/driver2.js create mode 100644 src/drivers/webextension/js/popup.bak.js create mode 100644 src/drivers/webextension/js/utils.js create mode 100644 src/drivers/webextension/js/wappalyzer.bak.js delete mode 100644 src/drivers/webextension/js/wappalyzer2.js diff --git a/src/drivers/webextension/css/popup.css b/src/drivers/webextension/css/popup.css index 14ecfd695..e364577df 100644 --- a/src/drivers/webextension/css/popup.css +++ b/src/drivers/webextension/css/popup.css @@ -1,30 +1,47 @@ +:root { + --color-primary: #4608ad; + --color-secondary: #e0e0e0; + --color-text: #4a4a4a; +} + body { background: #fff; direction: __MSG_@@bidi_dir__; font-family: Helvetica, Arial, sans-serif; - font-size: .8rem; + font-size: .9rem; + line-height: 1.5rem; margin: 0; min-width: 30rem; } +a { + color: var(--color-primary); + outline: none; + text-decoration: none; +} + +a:focus { + outline: none; +} + +a:hover { + text-decoration: underline; +} + .header { align-items: center; - border-bottom: 1px solid #dbdbdb; + border-bottom: 1px solid var(--color-secondary); height: 4rem; display: flex; } -.header__link:focus { - outline: none; -} - .header__logo { display: inline-block; margin: .2rem 1.5rem 0 1.5rem; -webkit-backface-visibility: hidden; -webkit-transform: translateZ(0) scale(1.0, 1.0); transform: translateZ(0); - height: 2rem; + height: 2.5rem; } .header__logo--dark { @@ -33,144 +50,37 @@ body { .footer { align-items: center; - border-top: 1px solid #dbdbdb; + border-top: 1px solid var(--color-secondary); height: 3rem; display: flex; padding: 0 1.5rem; } -.footer__link { - color: #4608ad; - text-decoration: none; -} - -.footer__link:hover, .footer__link:active { - color: #4608ad; - text-decoration: underline; -} - -.container { - min-height: 5rem; - padding: 1rem 1.5rem 0rem 1.5rem; -} - -.detected { +.detections { columns: 2; column-gap: 1.5rem; - line-height: 1.4rem; + padding: 1.5rem; } -.detected__category { +.category { page-break-inside: avoid; break-inside: avoid-column; padding-bottom: 1rem; } -.detected__category-name { - display: block; -} - -.detected__category-link { - color: #4608ad; +.category__link { font-weight: bold; line-height: 2rem; text-decoration: none; } -.detected__category-link:hover { - color: #4a4a4a; -} - -.detected__category-pin-wrapper { - margin-left: .2rem; - margin-right: .2rem; -} - -.detected__category-pin { - cursor: pointer; - display: none; - height: 16px; - margin-left: .2rem; - width: 16px; - vertical-align: middle; -} - -.detected__category:hover .detected__category-pin--inactive { - display: inline-block; -} - -.detected__category-pin-wrapper--active .detected__category-pin--inactive, -.detected__category-pin-wrapper:hover .detected__category-pin--inactive { - display: none !important; -} - -.detected__category-pin-wrapper--active .detected__category-pin--active, -.detected__category-pin-wrapper:hover .detected__category-pin--active { - display: inline-block; -} - -.detected__app { - color: #4a4a4a; +.technology { display: block; line-height: 1.7rem; - text-decoration: none; -} - -.detected__app:focus { - display: block; - outline: 0; -} - -.detected__app-icon { - display: inline-block; - height: 16px; - margin-inline-end: .5rem; - vertical-align: -.2rem; - width: 16px; -} - -.detected__app-name { -} - -.detected__app-version, .detected__app-confidence { - background: #eee; - border-radius: 3px; - font-size: .7rem; - margin-left: .3rem; - padding: .1rem .2rem; } -.detected__app:hover .detected__app-name { - border-bottom: 1px solid #4a4a4a; -} - -.detected__app:hover .detected__app-version, -.detected__app:hover .detected__app-confidence { - border-bottom: 1px solid white; -} - -.detected-app { - padding: 7px 0; -} - -.detected-app:first-child { - padding-top: 0; -} - -.detected-app:last-child { - border: none; - padding-bottom: 0; -} - -.empty { - display: flex; - height: 5rem; - margin-bottom: 1rem; - align-items: center; - justify-content: center; -} - -.empty__text { +.technology__link { + color: var(--color-text) } .terms { @@ -216,72 +126,5 @@ body { } .terms__privacy { - color: #4608ad; margin-top: 1rem; } - -@media (prefers-color-scheme: dark) { - /* Add alternative color palette for Dark mode theme. */ - body.theme-mode-sync { - background: linear-gradient(160deg, #32067c, #150233); - } - - .theme-mode-sync .header { - border-bottom: 1px solid rgba(255, 255, 255, .2); - } - - .theme-mode-sync .header__logo--dark { - display: inline-block; - } - - .theme-mode-sync .header__logo--light { - display: none; - } - - .theme-mode-sync .footer { - border-top: 1px solid rgba(255, 255, 255, .2); - } - - .theme-mode-sync .footer__link { - color: rgba(255, 255, 255, .8); - } - - .theme-mode-sync .footer__link:hover, .theme-mode-sync .footer__link:active { - color: rgba(255, 255, 255, .8); - } - - .theme-mode-sync .container { - color: white; - } - - .theme-mode-sync .detected__category-link { - color: #fff; - } - - .theme-mode-sync .detected__app { - color: rgba(255, 255, 255, .8); - } - - .theme-mode-sync .detected__category-link:hover { - color: white; - border-bottom: 1px solid white; - } - - .theme-mode-sync .detected__app-version, .theme-mode-sync .detected__app-confidence { - background-color: #4608ad; - } - - .theme-mode-sync .detected__app:hover .detected__app-name { - border-bottom: 1px solid white; - } - - .theme-mode-sync .detected__app:hover .theme-mode-sync .detected__app-version, - .theme-mode-sync .detected__app:hover .theme-mode-sync .detected__app-confidence { - border-bottom: none; - } - - .theme-mode-sync .terms__accept, - .theme-mode-sync .terms__privacy { - color: white; - } -} diff --git a/src/drivers/webextension/html/background.html b/src/drivers/webextension/html/background.html index f6147dc56..6900b5e3b 100644 --- a/src/drivers/webextension/html/background.html +++ b/src/drivers/webextension/html/background.html @@ -4,8 +4,9 @@ - - + + + diff --git a/src/drivers/webextension/html/options.html b/src/drivers/webextension/html/options.html index a10e8a859..23b49fb83 100644 --- a/src/drivers/webextension/html/options.html +++ b/src/drivers/webextension/html/options.html @@ -12,7 +12,7 @@ - + diff --git a/src/drivers/webextension/html/popup.html b/src/drivers/webextension/html/popup.html index 15fdc91dd..e7306ecc7 100644 --- a/src/drivers/webextension/html/popup.html +++ b/src/drivers/webextension/html/popup.html @@ -1,5 +1,4 @@ - @@ -7,28 +6,35 @@ - - +
- +
-
-
-
-
+
+
+ + + + +
-
-
+
+
diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js index 3be7fa391..e637475c5 100644 --- a/src/drivers/webextension/js/content.js +++ b/src/drivers/webextension/js/content.js @@ -2,10 +2,10 @@ /* eslint-env browser */ /* globals chrome */ -const port = chrome.runtime.connect({ name: 'content.js' }) +const Content = { + port: chrome.runtime.connect({ name: 'content.js' }), -;(async function() { - if (typeof chrome !== 'undefined' && typeof document.body !== 'undefined') { + async init() { await new Promise((resolve) => setTimeout(resolve, 1000)) try { @@ -25,13 +25,13 @@ const port = chrome.runtime.connect({ name: 'content.js' }) html = chunks.join('\n') - // Scripts + // Script tags const scripts = Array.from(document.scripts) .filter(({ src }) => src) .map(({ src }) => src) .filter((script) => script.indexOf('data:text/javascript;') !== 0) - // Meta + // Meta tags const meta = Array.from(document.querySelectorAll('meta')) .map((meta) => ({ key: meta.getAttribute('name') || meta.getAttribute('property'), @@ -39,61 +39,64 @@ const port = chrome.runtime.connect({ name: 'content.js' }) })) .filter(({ value }) => value) - port.postMessage({ + Content.port.postMessage({ func: 'onContentLoad', args: [location.href, { html, scripts, meta }] }) - // JavaScript variables - const script = document.createElement('script') - - script.onload = () => { - const onMessage = (event) => { - if (event.data.id !== 'js') { - return - } - - window.removeEventListener('message', onMessage) + Content.port.postMessage({ func: 'getTechnologies' }) + } catch (error) { + Content.port.postMessage({ func: 'error', args: [error, 'content.js'] }) + } + }, - port.postMessage({ - func: 'analyze', - args: [new URL(location.href), { js: event.data.js }] - }) + onGetTechnologies(technologies) { + const script = document.createElement('script') - script.remove() + script.onload = () => { + const onMessage = ({ data }) => { + if (!data.wappalyzer || !data.wappalyzer.js) { + return } - window.addEventListener('message', onMessage) + window.removeEventListener('message', onMessage) + + Content.port.postMessage({ + func: 'analyzeJs', + args: [location.href, data.wappalyzer.js] + }) - port.postMessage({ id: 'get_js_patterns' }) + script.remove() } - script.setAttribute('src', chrome.extension.getURL('js/inject.js')) + window.addEventListener('message', onMessage) - document.body.appendChild(script) - } catch (error) { - port.postMessage({ func: 'error', args: [error, 'content.js'] }) + window.postMessage({ + wappalyzer: { + technologies: technologies + .filter(({ js }) => Object.keys(js).length) + .filter(({ name }) => name === 'jQuery') + .map(({ name, js }) => ({ name, chains: Object.keys(js) })) + } + }) } + + script.setAttribute('src', chrome.extension.getURL('js/inject.js')) + + document.body.appendChild(script) } -})() - -port.onMessage.addListener((message) => { - switch (message.id) { - case 'get_js_patterns': - postMessage( - { - id: 'patterns', - patterns: message.response.patterns - }, - window.location.href - ) - - break - default: - // Do nothing +} + +Content.port.onMessage.addListener(({ func, args }) => { + const onFunc = `on${func.charAt(0).toUpperCase() + func.slice(1)}` + + if (Content[onFunc]) { + Content[onFunc](args) } }) -// https://stackoverflow.com/a/44774834 -// https://developer.mozilla.org/en-US/Add-ons/WebExtensions/API/tabs/executeScript#Return_value -undefined // eslint-disable-line no-unused-expressions +if (/complete|interactive|loaded/.test(document.readyState)) { + Content.init() +} else { + document.addEventListener('DOMContentLoaded', Content.init) +} diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index 5671c66ef..0dd587cc3 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -2,7 +2,14 @@ /* eslint-env browser */ /* globals chrome, Wappalyzer, Utils */ -const { setTechnologies, setCategories, analyze, resolve, unique } = Wappalyzer +const { + setTechnologies, + setCategories, + analyze, + analyzeManyToMany, + resolve, + unique +} = Wappalyzer const { promisify, getOption } = Utils const Driver = { @@ -52,6 +59,26 @@ const Driver = { } }, + async analyzeJs(href, js) { + const url = new URL(href) + + await Driver.onDetect( + url, + Array.prototype.concat.apply( + [], + await Promise.all( + js.map(({ name, chain, value }) => + analyzeManyToMany( + Wappalyzer.technologies.find(({ name: _name }) => name === _name), + 'js', + { [chain]: [value] } + ) + ) + ) + ) + ) + }, + onRuntimeConnect(port) { port.onMessage.addListener(async ({ func, args }) => { if (!func) { @@ -60,6 +87,12 @@ const Driver = { Driver.log({ port: port.name, func, args }) + if (!Driver[func]) { + Driver.error(new Error(`Method does not exist: Driver.${func}`)) + + return + } + port.postMessage({ func, args: await Driver[func].call(port.sender, ...(args || [])) @@ -175,7 +208,10 @@ const Driver = { headers['content-type'] && /\/x?html/.test(headers['content-type'][0]) ) { - await Driver.onDetect(url, await analyze(url, { headers }, { tab })) + await Driver.onDetect( + url, + await analyze(url.href, { headers }, { tab }) + ) } } } catch (error) { @@ -192,12 +228,16 @@ const Driver = { domain: `.${url.hostname}` }) - await Driver.onDetect(url, await analyze(url, items)) + await Driver.onDetect(url, await analyze(href, items)) } catch (error) { Driver.error(error) } }, + getTechnologies() { + return Wappalyzer.technologies + }, + async onDetect(url, detections = []) { Driver.cache.hostnames[url.hostname] = unique([ ...(Driver.cache.hostnames[url.hostname] || []), diff --git a/src/drivers/webextension/js/inject.js b/src/drivers/webextension/js/inject.js index e23cb48b9..b0807124e 100644 --- a/src/drivers/webextension/js/inject.js +++ b/src/drivers/webextension/js/inject.js @@ -1,62 +1,44 @@ /* eslint-env browser */ -/* eslint-disable no-restricted-globals, no-prototype-builtins */ -;(() => { +;(function() { try { - const detectJs = (chain) => { - const properties = chain.split('.') - - let value = properties.length ? window : null - - for (let i = 0; i < properties.length; i += 1) { - const property = properties[i] - - if (value && value.hasOwnProperty(property)) { - value = value[property] - } else { - value = null - - break - } - } - - return typeof value === 'string' || typeof value === 'number' - ? value - : !!value - } - - const onMessage = (event) => { - if (event.data.id !== 'patterns') { + const onMessage = ({ data }) => { + if (!data.wappalyzer) { return } - removeEventListener('message', onMessage) - - const patterns = event.data.patterns || {} - - const js = {} - - for (const appName in patterns) { - if (patterns.hasOwnProperty(appName)) { - js[appName] = {} + const { technologies } = data.wappalyzer || {} - for (const chain in patterns[appName]) { - if (patterns[appName].hasOwnProperty(chain)) { - js[appName][chain] = {} - - for (const index in patterns[appName][chain]) { - const value = detectJs(chain) + removeEventListener('message', onMessage) - if (value && patterns[appName][chain].hasOwnProperty(index)) { - js[appName][chain][index] = value - } - } - } - } + postMessage({ + wappalyzer: { + js: technologies.reduce((results, { name, chains }) => { + chains.forEach((chain) => { + const value = chain + .split('.') + .reduce( + (value, method) => + value && value.hasOwnProperty(method) + ? value[method] + : undefined, + window + ) + + technologies.push({ + name, + chain, + value: + typeof value === 'string' || typeof value === 'number' + ? value + : !!value + }) + }) + + return technologies + }, []) } - } - - postMessage({ id: 'js', js }, window.location.href) + }) } addEventListener('message', onMessage) diff --git a/src/drivers/webextension/js/options.js b/src/drivers/webextension/js/options.js index 29d11d5d4..505117b52 100644 --- a/src/drivers/webextension/js/options.js +++ b/src/drivers/webextension/js/options.js @@ -1,109 +1,44 @@ -/** global: browser */ -/** global: Wappalyzer */ -/* globals browser Wappalyzer */ +'use strict' /* eslint-env browser */ +/* globals Utils */ -const wappalyzer = new Wappalyzer() +const { i18n, getOption, setOption } = Utils -/** - * Get a value from localStorage - */ -function getOption(name, defaultValue = null) { - return new Promise(async (resolve, reject) => { - let value = defaultValue +const Options = { + async init() { + // Theme mode + const themeMode = await getOption('themeMode', false) - try { - const option = await browser.storage.local.get(name) - - if (option[name] !== undefined) { - value = option[name] - } - } catch (error) { - wappalyzer.log(error.message, 'driver', 'error') - - return reject(error.message) + if (themeMode) { + document.querySelector('body').classList.add('theme-mode') } - return resolve(value) - }) + ;[ + ['upgradeMessage', true], + ['dynamicIcon', true], + ['tracking', true], + ['themeMode', false] + ].map(async ([option, defaultValue]) => { + const el = document + .querySelector( + `[data-i18n="option${option.charAt(0).toUpperCase() + + option.slice(1)}"]` + ) + .parentNode.querySelector('input') + + el.checked = !!(await getOption(option, defaultValue)) + + el.addEventListener('click', async () => { + await setOption(option, !!el.checked) + }) + }) + + i18n() + } } -/** - * Set a value in localStorage - */ -function setOption(name, value) { - return new Promise(async (resolve, reject) => { - try { - await browser.storage.local.set({ [name]: value }) - } catch (error) { - wappalyzer.log(error.message, 'driver', 'error') - - return reject(error.message) - } - - return resolve() - }) +if (/complete|interactive|loaded/.test(document.readyState)) { + Options.init() +} else { + document.addEventListener('DOMContentLoaded', Options.init) } - -document.addEventListener('DOMContentLoaded', async () => { - const nodes = document.querySelectorAll('[data-i18n]') - - Array.prototype.forEach.call(nodes, (node) => { - node.childNodes[0].nodeValue = browser.i18n.getMessage(node.dataset.i18n) - }) - - document.querySelector('#github').addEventListener('click', () => { - window.open(wappalyzer.config.githubURL) - }) - - document.querySelector('#twitter').addEventListener('click', () => { - window.open(wappalyzer.config.twitterURL) - }) - - document.querySelector('#wappalyzer').addEventListener('click', () => { - window.open(wappalyzer.config.websiteURL) - }) - - let el - let value - - // Upgrade message - value = await getOption('upgradeMessage', true) - - el = document.querySelector('#option-upgrade-message') - - el.checked = value - - el.addEventListener('change', (e) => - setOption('upgradeMessage', e.target.checked) - ) - - // Dynamic icon - value = await getOption('dynamicIcon', true) - - el = document.querySelector('#option-dynamic-icon') - - el.checked = value - - el.addEventListener('change', (e) => - setOption('dynamicIcon', e.target.checked) - ) - - // Tracking - value = await getOption('tracking', true) - - el = document.querySelector('#option-tracking') - - el.checked = value - - el.addEventListener('change', (e) => setOption('tracking', e.target.checked)) - - // Theme Mode - value = await getOption('themeMode', false) - - el = document.querySelector('#option-theme-mode') - - el.checked = value - - el.addEventListener('change', (e) => setOption('themeMode', e.target.checked)) -}) diff --git a/src/drivers/webextension/js/popup.js b/src/drivers/webextension/js/popup.js index 8170aa98c..a58e1cb79 100644 --- a/src/drivers/webextension/js/popup.js +++ b/src/drivers/webextension/js/popup.js @@ -2,7 +2,7 @@ /* eslint-env browser */ /* globals chrome, Utils */ -const { agent, getOption, setOption, promisify } = Utils +const { agent, i18n, getOption, setOption, promisify } = Utils const Popup = { port: chrome.runtime.connect({ name: 'popup.js' }), @@ -37,7 +37,7 @@ const Popup = { } else { document.querySelector('.detections').style.display = 'none' - Popup.i18n() + i18n() } // Alert @@ -65,12 +65,6 @@ const Popup = { Popup.driver('log', message, 'popup.js') }, - i18n() { - Array.from(document.querySelectorAll('[data-i18n]')).forEach( - (node) => (node.innerHTML = chrome.i18n.getMessage(node.dataset.i18n)) - ) - }, - categorise(technologies) { return Object.values( technologies.reduce((categories, technology) => { @@ -91,6 +85,10 @@ const Popup = { async onGetDetections(detections) { const pinnedCategory = await getOption('pinnedCategory') + if (detections.length) { + document.querySelector('.empty').remove() + } + Popup.categorise(detections).forEach( ({ id, name, slug: categorySlug, technologies }) => { const categoryNode = Popup.templates.category.cloneNode(true) @@ -149,7 +147,7 @@ const Popup = { a.addEventListener('click', () => Popup.driver('open', a.href)) ) - Popup.i18n() + i18n() } } diff --git a/src/drivers/webextension/js/utils.js b/src/drivers/webextension/js/utils.js index b35190ff3..3f83449d1 100644 --- a/src/drivers/webextension/js/utils.js +++ b/src/drivers/webextension/js/utils.js @@ -43,5 +43,11 @@ const Utils = { } catch (error) { throw new Error(error.message || error.toString()) } + }, + + i18n() { + Array.from(document.querySelectorAll('[data-i18n]')).forEach( + (node) => (node.innerHTML = chrome.i18n.getMessage(node.dataset.i18n)) + ) } } From 575809793e9c4be6ed5a2c8f2c77aca73ad12380 Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Tue, 9 Jun 2020 15:17:40 +1000 Subject: [PATCH 6/7] Refactoring --- src/drivers/webextension/css/styles.css | 22 ++ src/drivers/webextension/html/popup.html | 5 +- src/drivers/webextension/js/content.js | 15 +- src/drivers/webextension/js/driver.js | 393 +++++++++++++------ src/drivers/webextension/js/lib/iframe.js | 11 +- src/drivers/webextension/js/lib/jsontodom.js | 63 --- src/drivers/webextension/js/lib/network.js | 159 ++------ src/drivers/webextension/js/popup.js | 44 ++- 8 files changed, 405 insertions(+), 307 deletions(-) delete mode 100644 src/drivers/webextension/js/lib/jsontodom.js diff --git a/src/drivers/webextension/css/styles.css b/src/drivers/webextension/css/styles.css index 9ed0f55a3..d4a45aaa3 100644 --- a/src/drivers/webextension/css/styles.css +++ b/src/drivers/webextension/css/styles.css @@ -150,6 +150,21 @@ a:hover { color: var(--color-text); } +.technology__confidence { + opacity: .5; + font-size: .7rem; + margin-left: .2rem; +} + +.technology__version { + background: var(--color-secondary); + border-radius: 3px; + font-size: .7rem; + padding: .1rem .3rem; + margin-left: .4rem; + vertical-align: middle; +} + .terms { align-items: center; display: flex; @@ -235,6 +250,13 @@ a:hover { color: #fff } + .theme-mode .technology__confidence { + } + + .theme-mode .technology__version { + background: var(--color-primary); + } + .theme-mode .footer { border-color: var(--color-secondary-dark) } diff --git a/src/drivers/webextension/html/popup.html b/src/drivers/webextension/html/popup.html index 5cf376251..593fc255b 100644 --- a/src/drivers/webextension/html/popup.html +++ b/src/drivers/webextension/html/popup.html @@ -55,8 +55,11 @@ + +   + +   -  
diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js index e637475c5..405be35d6 100644 --- a/src/drivers/webextension/js/content.js +++ b/src/drivers/webextension/js/content.js @@ -25,6 +25,19 @@ const Content = { html = chunks.join('\n') + const language = + document.documentElement.getAttribute('lang') || + document.documentElement.getAttribute('xml:lang') || + (await new Promise((resolve) => + chrome.i18n.detectLanguage(html, ({ languages }) => + resolve( + languages + .filter(({ percentage }) => percentage >= 75) + .map(({ language: lang }) => lang)[0] + ) + ) + )) + // Script tags const scripts = Array.from(document.scripts) .filter(({ src }) => src) @@ -41,7 +54,7 @@ const Content = { Content.port.postMessage({ func: 'onContentLoad', - args: [location.href, { html, scripts, meta }] + args: [location.href, { html, scripts, meta }, language] }) Content.port.postMessage({ func: 'getTechnologies' }) diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index 0dd587cc3..7daf679af 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -7,15 +7,61 @@ const { setCategories, analyze, analyzeManyToMany, - resolve, - unique + resolve } = Wappalyzer -const { promisify, getOption } = Utils +const { agent, promisify, getOption, setOption } = Utils + +const expiry = 1000 * 60 * 60 * 24 const Driver = { - cache: { - hostnames: {}, - robots: {} + lastPing: Date.now(), + + async init() { + await Driver.loadTechnologies() + + const hostnameCache = (await getOption('hostnames')) || {} + + Driver.cache = { + hostnames: Object.keys(hostnameCache).reduce( + (cache, hostname) => ({ + ...cache, + [hostname]: { + ...hostnameCache[hostname], + detections: hostnameCache[hostname].detections.map( + ({ + pattern: { regex, confidence, version }, + match, + technology: name, + hits + }) => ({ + pattern: { + regex: new RegExp(regex, 'i'), + confidence, + version + }, + match, + technology: Wappalyzer.technologies.find( + ({ name: _name }) => name === _name + ), + hits + }) + ) + } + }), + {} + ), + tabs: {}, + robots: (await getOption('robots')) || {}, + ads: (await getOption('ads')) || [] + } + + chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect) + chrome.webRequest.onCompleted.addListener( + Driver.onWebRequestComplete, + { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, + ['responseHeaders'] + ) + chrome.tabs.onRemoved.addListener((id) => (Driver.cache.tabs[id] = null)) }, log(message, source = 'driver', type = 'log') { @@ -97,90 +143,6 @@ const Driver = { func, args: await Driver[func].call(port.sender, ...(args || [])) }) - - /* - const pinnedCategory = await getOption('pinnedCategory') - - const url = new URL(port.sender.tab.url) - - const cookies = await browser.cookies.getAll({ - domain: `.${url.hostname}` - }) - - let response - - switch (message.id) { - case 'log': - wappalyzer.log(message.subject, message.source) - - break - case 'analyze': - if (message.subject.html) { - browser.i18n - .detectLanguage(message.subject.html) - .then(({ languages }) => { - const language = languages - .filter(({ percentage }) => percentage >= 75) - .map(({ language: lang }) => lang)[0] - - message.subject.language = language - - wappalyzer.analyze(url, message.subject, { - tab: port.sender.tab - }) - }) - } else { - wappalyzer.analyze(url, message.subject, { tab: port.sender.tab }) - } - - await setOption('hostnameCache', wappalyzer.hostnameCache) - - break - case 'ad_log': - wappalyzer.cacheDetectedAds(message.subject) - - break - case 'get_apps': - response = { - tabCache: tabCache[message.tab.id], - apps: wappalyzer.apps, - categories: wappalyzer.categories, - pinnedCategory, - termsAccepted: - userAgent() === 'chrome' || - (await getOption('termsAccepted', false)) - } - - break - case 'set_option': - await setOption(message.key, message.value) - - break - case 'get_js_patterns': - response = { - patterns: wappalyzer.jsPatterns - } - - break - case 'update_theme_mode': - // Sync theme mode to popup. - response = { - themeMode: await getOption('themeMode', false) - } - - break - default: - // Do nothing - } - - if (response) { - port.postMessage({ - id: message.id, - response - }) - } - }) - */ }) }, @@ -220,7 +182,7 @@ const Driver = { } }, - async onContentLoad(href, items) { + async onContentLoad(href, items, language) { try { const url = new URL(href) @@ -228,7 +190,7 @@ const Driver = { domain: `.${url.hostname}` }) - await Driver.onDetect(url, await analyze(href, items)) + await Driver.onDetect(url, await analyze(href, items), language) } catch (error) { Driver.error(error) } @@ -238,15 +200,98 @@ const Driver = { return Wappalyzer.technologies }, - async onDetect(url, detections = []) { - Driver.cache.hostnames[url.hostname] = unique([ - ...(Driver.cache.hostnames[url.hostname] || []), - ...detections - ]) + async onDetect(url, detections = [], language) { + // Cache detections + // eslint-disable-next-line standard/computed-property-even-spacing + Driver.cache.hostnames[url.hostname] = { + ...(Driver.cache.hostnames[url.hostname] || { + detections: [] + }), + dateTime: Date.now() + } + + Driver.cache.hostnames[url.hostname].language = + Driver.cache.hostnames[url.hostname].language || language + + detections.forEach((detection) => { + const foo = Driver.cache.hostnames[url.hostname].detections + const { + technology: { name }, + pattern: { regex } + } = detection + + const cache = foo.find( + ({ technology: { name: _name }, pattern: { regex: _regex } }) => + name === _name && (!regex || regex) === _regex + ) + + if (cache) { + cache.hits += 1 + } else { + foo.push({ + ...detection, + hits: 1 + }) + } + }) + + // Expire cache + Driver.cache.hostnames = Object.keys(Driver.cache.hostnames).reduce( + (hostnames, hostname) => { + const cache = Driver.cache.hostnames[hostname] + + if (cache.dateTime > Date.now() - expiry) { + hostnames[hostname] = cache + } + + return hostnames + }, + {} + ) + + await setOption( + 'hostnames', + Object.keys(Driver.cache.hostnames).reduce( + (cache, hostname) => ({ + ...cache, + [hostname]: { + ...Driver.cache.hostnames[hostname], + detections: Driver.cache.hostnames[hostname].detections.map( + ({ + pattern: { regex, confidence, version }, + match, + technology: { name: technology } + }) => ({ + technology, + pattern: { + regex: regex.source, + confidence, + version + }, + match + }) + ) + } + }), + {} + ) + ) - const resolved = resolve(Driver.cache.hostnames[url.hostname]) + const resolved = resolve(Driver.cache.hostnames[url.hostname].detections) await Driver.setIcon(url, resolved) + + const tabs = await promisify(chrome.tabs, 'query', { url: [url.href] }) + + tabs.forEach(({ id }) => (Driver.cache.tabs[id] = resolved)) + + await Driver.ping() + }, + + async onAd(ad) { + Driver.cache.ads.push(ad) + + await setOption('ads', Driver.cache.ads) }, async setIcon(url, technologies) { @@ -292,24 +337,152 @@ const Driver = { }, async getDetections() { - const [{ url: href }] = await promisify(chrome.tabs, 'query', { + const [{ id }] = await promisify(chrome.tabs, 'query', { active: true, currentWindow: true }) + return Driver.cache.tabs[id] + }, + + async getRobots(hostname, secure = false) { + if (!(await getOption('tracking', true))) { + return + } + + if (typeof Driver.cache.robots[hostname] !== 'undefined') { + return Driver.cache.robots[hostname] + } + + try { + Driver.cache.robots[hostname] = await Promise.race([ + new Promise(async (resolve) => { + const response = await fetch( + `http${secure ? 's' : ''}://${hostname}/robots.txt`, + { + redirect: 'follow', + mode: 'no-cors' + } + ) + + if (!response.ok) { + Driver.error(new Error(response.statusText)) + + resolve('') + } + + let agent + + resolve( + (await response.text()).split('\n').reduce((disallows, line) => { + let matches = /^User-agent:\s*(.+)$/i.exec(line.trim()) + + if (matches) { + agent = matches[1].toLowerCase() + } else if (agent === '*' || agent === 'wappalyzer') { + matches = /^Disallow:\s*(.+)$/i.exec(line.trim()) + + if (matches) { + disallows.push(matches[1]) + } + } + + return disallows + }, []) + ) + }), + new Promise((resolve) => setTimeout(() => resolve(''), 5000)) + ]) + + Driver.cache.robots = Object.keys(Driver.cache.robots) + .slice(-50) + .reduce( + (cache, hostname) => ({ + ...cache, + [hostname]: Driver.cache.robots[hostname] + }), + {} + ) + + await setOption('robots', Driver.cache.robots) + + return Driver.cache.robots[hostname] + } catch (error) { + Driver.error(error) + } + }, + + async checkRobots(href) { const url = new URL(href) - return resolve(Driver.cache.hostnames[url.hostname]) + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error('Invalid protocol') + } + + const robots = await Driver.getRobots( + url.hostname, + url.protocol === 'https:' + ) + + if (robots.some((disallowed) => url.pathname.indexOf(disallowed) === 0)) { + throw new Error('Disallowed') + } + }, + + async ping() { + const tracking = await getOption('tracking', true) + const termsAccepted = + agent === 'chrome' || (await getOption('termsAccepted', false)) + + if (tracking && termsAccepted) { + const count = Object.keys(Driver.cache.hostnames).length + + if (count && (count >= 50 || Driver.lastPing < Date.now() - 5000)) { + await Driver.post( + 'https://api.wappalyzer.com/ping/v1/', + Object.keys(Driver.cache.hostnames).reduce((hostnames, hostname) => { + const { language, detections } = Driver.cache.hostnames[hostname] + + hostnames[hostname] = hostnames[hostname] || { + applications: {}, + meta: { + language + } + } + + resolve(detections).forEach(({ name, confidence, version }) => { + if (confidence === 100) { + console.log( + name, + detections.find( + ({ technology: { name: _name } }) => name === _name + ) + ) + hostnames[hostname].applications[name] = { + version, + hits: detections.find( + ({ technology: { name: _name } }) => name === _name + ).pattern.hits + } + } + }) + + return hostnames + }, {}) + ) + + await setOption('hostnames', (Driver.cache.hostnames = {})) + + Driver.lastPing = Date.now() + } + + if (Driver.cache.ads.length > 50) { + await Driver.post('https://ad.wappalyzer.com/log/wp/', Driver.cache.ads) + + await setOption('ads', (Driver.cache.ads = [])) + } + } } } -;(async function() { - await Driver.loadTechnologies() - - chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect) - chrome.webRequest.onCompleted.addListener( - Driver.onWebRequestComplete, - { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, - ['responseHeaders'] - ) -})() +Driver.init() diff --git a/src/drivers/webextension/js/lib/iframe.js b/src/drivers/webextension/js/lib/iframe.js index e132433c8..38ea3af38 100644 --- a/src/drivers/webextension/js/lib/iframe.js +++ b/src/drivers/webextension/js/lib/iframe.js @@ -120,8 +120,8 @@ var exports = {}; return dict; }, sendToBackground: function(message, event, responseMessage) { - if ( typeof browser !== 'undefined' || typeof chrome !== 'undefined' ) { - var port = browser.runtime.connect({name:"adparser"}); + if ( typeof chrome !== 'undefined' ) { + var port = chrome.runtime.connect({name:"adparser"}); port.onMessage.addListener((message) => { if ( message && message.tracking_enabled ) { @@ -1088,8 +1088,8 @@ var exports = {}; } function addBackgroundListener(event, callback) { - if ( typeof browser !== 'undefined' || typeof chrome !== 'undefined' ) { - browser.runtime.onMessage.addListener(function(msg) { + if ( typeof chrome !== 'undefined' ) { + chrome.runtime.onMessage.addListener(function(msg) { if ( msg.event === event ) { callback(msg); } @@ -1111,6 +1111,7 @@ var exports = {}; if ( origUrl.indexOf('google.com/_/chrome/newtab') === -1 ) { var onBlockedRobotsMessage = function() { + return // TODO var log; log = _logGen.log('invalid-robotstxt', []); log.doc.finalPageUrl = log.doc.url; @@ -1173,7 +1174,7 @@ if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { })(window); (function(adparser, pageUrl) { function onAdFound(log) { - adparser.sendToBackground({ id: 'ad_log', subject: log }, 'ad_log', '', function(){}); + adparser.sendToBackground({ func: 'onAd', args: [log] }, 'onAd', '', function(){}); } if ( adparser && adparser.inWindowTop ) { diff --git a/src/drivers/webextension/js/lib/jsontodom.js b/src/drivers/webextension/js/lib/jsontodom.js deleted file mode 100644 index 24d9e4c29..000000000 --- a/src/drivers/webextension/js/lib/jsontodom.js +++ /dev/null @@ -1,63 +0,0 @@ -jsonToDOM.namespaces = { - html: 'http://www.w3.org/1999/xhtml', - xul: 'http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul', -}; - -jsonToDOM.defaultNamespace = jsonToDOM.namespaces.html; - -function jsonToDOM(jsonTemplate, doc, nodes) { - function namespace(name) { - const reElemNameParts = /^(?:(.*):)?(.*)$/.exec(name); - return { namespace: jsonToDOM.namespaces[reElemNameParts[1]], shortName: reElemNameParts[2] }; - } - - // Note that 'elemNameOrArray' is: either the full element name (eg. [html:]div) or an array of elements in JSON notation - function tag(elemNameOrArray, elemAttr) { - // Array of elements? Parse each one... - if (Array.isArray(elemNameOrArray)) { - const frag = doc.createDocumentFragment(); - Array.prototype.forEach.call(arguments, (thisElem) => { - frag.appendChild(tag(...thisElem)); - }); - return frag; - } - - // Single element? Parse element namespace prefix (if none exists, default to defaultNamespace), and create element - const elemNs = namespace(elemNameOrArray); - const elem = doc.createElementNS(elemNs.namespace || jsonToDOM.defaultNamespace, elemNs.shortName); - - // Set element's attributes and/or callback functions (eg. onclick) - for (const key in elemAttr) { - const val = elemAttr[key]; - if (nodes && key == 'key') { - nodes[val] = elem; - continue; - } - - const attrNs = namespace(key); - if (typeof val === 'function') { - // Special case for function attributes; don't just add them as 'on...' attributes, but as events, using addEventListener - elem.addEventListener(key.replace(/^on/, ''), val, false); - } else { - // Note that the default namespace for XML attributes is, and should be, blank (ie. they're not in any namespace) - elem.setAttributeNS(attrNs.namespace || '', attrNs.shortName, val); - } - } - - // Create and append this element's children - const childElems = Array.prototype.slice.call(arguments, 2); - childElems.forEach((childElem) => { - if (childElem != null) { - elem.appendChild( - childElem instanceof doc.defaultView.Node ? childElem - : Array.isArray(childElem) ? tag(...childElem) - : doc.createTextNode(childElem), - ); - } - }); - - return elem; - } - - return tag(...jsonTemplate); -} diff --git a/src/drivers/webextension/js/lib/network.js b/src/drivers/webextension/js/lib/network.js index 7c46fcf9f..f9f58d843 100644 --- a/src/drivers/webextension/js/lib/network.js +++ b/src/drivers/webextension/js/lib/network.js @@ -1,19 +1,6 @@ -(function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i= MIN_FF_MAJOR_VERSION) { - callback(); - } else { - elseCallback(); - } - }); - } catch (err) { - - elseCallback(); - } - } else { - elseCallback(); - } + chrome.webNavigation.getFrame(getFrameDetails, callback); } function ifTrackingEnabled(details, ifCallback, elseCallback) { @@ -112,24 +72,19 @@ allowedByRobotsTxt(details, ifCallback, elseCallback); }; - browser.storage.local.get('tracking').then(function(item) { - - if ( item.hasOwnProperty('tracking') ) { - if ( item.tracking ) { - fullIfCallback(); - } else { - elseCallback(); - } - } else { - fullIfCallback(); - } + Utils.getOption('tracking', true).then(function(tracking) { + if ( tracking ) { + fullIfCallback(); + } else { + elseCallback(); + } }); } function allowedByRobotsTxt(details, ifCallback, elseCallback) { if ( details.url && !details.url.startsWith('chrome://') ) { - robotsTxtAllows(details.url).then(ifCallback, elseCallback); + Driver.checkRobots(details.url, details.url.startsWith('https:')).then(ifCallback).catch(elseCallback); } else { elseCallback(); } @@ -268,7 +223,7 @@ PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() { var logMessage = Array.from(arguments).join(' '); var message = {message: logMessage, event: 'console-log-message'}; - browser.tabs.sendMessage(this.tabId, message); + chrome.tabs.sendMessage(this.tabId, message); }; PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, adTrackingEvent) { @@ -298,7 +253,7 @@ msg.origUrl = curPageUrl; msg.displayAdFound = this.displayAdFound; - browser.tabs.sendMessage(this.tabId, msg); + chrome.tabs.sendMessage(this.tabId, msg); }; PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) { @@ -615,7 +570,7 @@ var _this = this, origPageUrl, msgAssetReq; msgAssetReq = this.msgsBeingSent[msgKey]; - browser.tabs.get(this.tabId).then(function(tab) { + chrome.tabs.get(this.tabId, function(tab) { origPageUrl = tab.url; }); @@ -697,110 +652,85 @@ function registerListeners() { - browser.webRequest.onBeforeRequest.addListener( + chrome.webRequest.onBeforeRequest.addListener( onBeforeRequestListener, {urls: ['http://*/*', 'https://*/*']}, [] ); - browser.webRequest.onSendHeaders.addListener( + chrome.webRequest.onSendHeaders.addListener( onSendHeadersListener, {urls: ['http://*/*', 'https://*/*']}, ['requestHeaders'] ); - browser.webRequest.onHeadersReceived.addListener( + chrome.webRequest.onHeadersReceived.addListener( onHeadersReceivedListener, {urls: ['http://*/*', 'https://*/*']}, ['responseHeaders'] ); - browser.webRequest.onBeforeRedirect.addListener( + chrome.webRequest.onBeforeRedirect.addListener( onBeforeRedirectListener, {urls: ['http://*/*', 'https://*/*']}, [] ); - browser.webRequest.onResponseStarted.addListener( + chrome.webRequest.onResponseStarted.addListener( onResponseStartedListener, {urls: ['http://*/*', 'https://*/*']}, ['responseHeaders'] ); - browser.webNavigation.onCommitted.addListener(onCommittedListener); - browser.webNavigation.onCompleted.addListener(onCompletedListener); - browser.tabs.onRemoved.addListener(onRemovedListener); - browser.runtime.onMessage.addListener(onMessageListener); + chrome.webNavigation.onCommitted.addListener(onCommittedListener); + chrome.webNavigation.onCompleted.addListener(onCompletedListener); + chrome.tabs.onRemoved.addListener(onRemovedListener); + chrome.runtime.onMessage.addListener(onMessageListener); areListenersRegistered = true; } function unregisterListeners() { - browser.webRequest.onBeforeRequest.removeListener( + chrome.webRequest.onBeforeRequest.removeListener( onBeforeRequestListener ); - browser.webRequest.onSendHeaders.removeListener( + chrome.webRequest.onSendHeaders.removeListener( onSendHeadersListener ); - browser.webRequest.onHeadersReceived.removeListener( + chrome.webRequest.onHeadersReceived.removeListener( onHeadersReceivedListener ); - browser.webRequest.onBeforeRedirect.removeListener( + chrome.webRequest.onBeforeRedirect.removeListener( onBeforeRedirectListener ); - browser.webRequest.onResponseStarted.removeListener( + chrome.webRequest.onResponseStarted.removeListener( onResponseStartedListener ); - browser.webNavigation.onCommitted.removeListener(onCommittedListener); - browser.webNavigation.onCompleted.removeListener(onCompletedListener); - browser.tabs.onRemoved.removeListener(onRemovedListener); - browser.runtime.onMessage.removeListener(onMessageListener); + chrome.webNavigation.onCommitted.removeListener(onCommittedListener); + chrome.webNavigation.onCompleted.removeListener(onCompletedListener); + chrome.tabs.onRemoved.removeListener(onRemovedListener); + chrome.runtime.onMessage.removeListener(onMessageListener); areListenersRegistered = false; } - function areRequiredBrowserApisAvailable() { - return requiredBrowserApis.every(function(api) { - return typeof api !== 'undefined'; - }); - } - - if ( areRequiredBrowserApisAvailable() ) { - ifBrowserValid( - function() { - browser.webNavigation.onBeforeNavigate.addListener( - function(details) { - if ( details.frameId === 0 ) { - globalPageContainer.onNewNavigation(details); - } - }, - { - url: [{urlMatches: 'http://*/*'}, {urlMatches: 'https://*/*'}] - } - ); - }, function() { - - } - ); - } - - browser.runtime.onConnect.addListener((port) => { - port.onMessage.addListener((message) => { - if ( message === 'is_browser_valid' ) { - ifBrowserValid( - port.postMessage({'browser_valid': true}), - port.postMessage({'browser_valid': false}) - ); - } - }); - }); - - browser.runtime.onConnect.addListener((port) => { + chrome.webNavigation.onBeforeNavigate.addListener( + function(details) { + if ( details.frameId === 0 ) { + globalPageContainer.onNewNavigation(details); + } + }, + { + url: [{urlMatches: 'http://*/*'}, {urlMatches: 'https://*/*'}] + } + ); + + chrome.runtime.onConnect.addListener((port) => { port.onMessage.addListener((message) => { if ( message === 'is_tracking_enabled' ) { ifTrackingEnabled( @@ -816,7 +746,4 @@ return true; }); }); - })(); - -},{}]},{},[1]); diff --git a/src/drivers/webextension/js/popup.js b/src/drivers/webextension/js/popup.js index a58e1cb79..5852af745 100644 --- a/src/drivers/webextension/js/popup.js +++ b/src/drivers/webextension/js/popup.js @@ -122,22 +122,44 @@ const Popup = { }) ) - technologies.forEach(({ name, slug, icon, website }) => { - const technologyNode = Popup.templates.technology.cloneNode(true) + technologies + .filter(({ confidence }) => confidence) + .forEach(({ name, slug, confidence, version, icon, website }) => { + const technologyNode = Popup.templates.technology.cloneNode(true) - const image = technologyNode.querySelector('.technology__icon') + const image = technologyNode.querySelector('.technology__icon') - image.src = `../images/icons/${icon}` + image.src = `../images/icons/${icon}` - const link = technologyNode.querySelector('.technology__link') + const link = technologyNode.querySelector('.technology__link') - link.href = `https://www.wappalyzer.com/technologies/${categorySlug}/${slug}` - link.textContent = name + link.href = `https://www.wappalyzer.com/technologies/${categorySlug}/${slug}` + link.textContent = name - categoryNode - .querySelector('.technologies') - .appendChild(technologyNode) - }) + const confidenceNode = technologyNode.querySelector( + '.technology__confidence' + ) + + if (confidence < 100) { + confidenceNode.textContent = `${confidence}% sure` + } else { + confidenceNode.remove() + } + + const versionNode = technologyNode.querySelector( + '.technology__version' + ) + + if (version) { + versionNode.textContent = version + } else { + versionNode.remove() + } + + categoryNode + .querySelector('.technologies') + .appendChild(technologyNode) + }) document.querySelector('.detections').appendChild(categoryNode) } From 27e9b2fcbbd387b1ae150c0b5db25efd73d704cd Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Wed, 10 Jun 2020 09:29:31 +1000 Subject: [PATCH 7/7] Refactoring --- bin/validate | 6 +- src/drivers/webextension/js/content.js | 20 +- src/drivers/webextension/js/driver.bak.js | 445 --------- src/drivers/webextension/js/driver.js | 113 +-- src/drivers/webextension/js/lib/iframe.js | 13 +- src/drivers/webextension/js/popup.bak.js | 333 ------- src/drivers/webextension/js/wappalyzer.bak.js | 727 -------------- src/drivers/webextension/manifest.json | 6 - src/wappalyzer.js | 928 +++++------------- 9 files changed, 339 insertions(+), 2252 deletions(-) delete mode 100644 src/drivers/webextension/js/driver.bak.js delete mode 100644 src/drivers/webextension/js/popup.bak.js delete mode 100644 src/drivers/webextension/js/wappalyzer.bak.js diff --git a/bin/validate b/bin/validate index 3e5007a71..67f567b51 100755 --- a/bin/validate +++ b/bin/validate @@ -16,6 +16,6 @@ echo "Validating icons..." ./bin/validate-icons -echo "Running tests..." - -yarn run test +# echo "Running tests..." +# +# yarn run test diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js index 405be35d6..a1deeb4a6 100644 --- a/src/drivers/webextension/js/content.js +++ b/src/drivers/webextension/js/content.js @@ -3,11 +3,19 @@ /* globals chrome */ const Content = { - port: chrome.runtime.connect({ name: 'content.js' }), - async init() { await new Promise((resolve) => setTimeout(resolve, 1000)) + Content.port = chrome.runtime.connect({ name: 'content.js' }) + + Content.port.onMessage.addListener(({ func, args }) => { + const onFunc = `on${func.charAt(0).toUpperCase() + func.slice(1)}` + + if (Content[onFunc]) { + Content[onFunc](args) + } + }) + try { // HTML let html = new XMLSerializer().serializeToString(document) @@ -100,14 +108,6 @@ const Content = { } } -Content.port.onMessage.addListener(({ func, args }) => { - const onFunc = `on${func.charAt(0).toUpperCase() + func.slice(1)}` - - if (Content[onFunc]) { - Content[onFunc](args) - } -}) - if (/complete|interactive|loaded/.test(document.readyState)) { Content.init() } else { diff --git a/src/drivers/webextension/js/driver.bak.js b/src/drivers/webextension/js/driver.bak.js deleted file mode 100644 index 0ff7d491c..000000000 --- a/src/drivers/webextension/js/driver.bak.js +++ /dev/null @@ -1,445 +0,0 @@ -/** - * WebExtension driver - */ - -/* eslint-env browser */ -/* global browser, chrome, Wappalyzer */ - -/** global: browser */ -/** global: chrome */ -/** global: fetch */ -/** global: Wappalyzer */ - -const wappalyzer = new Wappalyzer() - -const tabCache = {} -const robotsTxtQueue = {} - -let categoryOrder = [] - -browser.tabs.onRemoved.addListener((tabId) => { - tabCache[tabId] = null -}) - -function userAgent() { - const url = chrome.extension.getURL('/') - - if (url.startsWith('moz-')) { - return 'firefox' - } - - if (url.startsWith('ms-browser')) { - return 'edge' - } - - return 'chrome' -} - -/** - * Get a value from localStorage - */ -function getOption(name, defaultValue = null) { - return new Promise(async (resolve, reject) => { - let value = defaultValue - - try { - const option = await browser.storage.local.get(name) - - if (option[name] !== undefined) { - value = option[name] - } - } catch (error) { - wappalyzer.log(error.message, 'driver', 'error') - - return reject(error.message) - } - - return resolve(value) - }) -} - -/** - * Set a value in localStorage - */ -function setOption(name, value) { - return new Promise(async (resolve, reject) => { - try { - await browser.storage.local.set({ [name]: value }) - } catch (error) { - wappalyzer.log(error.message, 'driver', 'error') - - return reject(error.message) - } - - return resolve() - }) -} - -/** - * Open a tab - */ -function openTab(args) { - browser.tabs.create({ - url: args.url, - active: args.background === undefined || !args.background - }) -} - -/** - * Make a POST request - */ -async function post(url, body) { - try { - const response = await fetch(url, { - method: 'POST', - body: JSON.stringify(body) - }) - - wappalyzer.log(`POST ${url}: ${response.status}`, 'driver') - } catch (error) { - wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error') - } -} - -// Capture response headers -browser.webRequest.onCompleted.addListener( - async (request) => { - const headers = {} - - if (request.responseHeaders) { - const url = wappalyzer.parseUrl(request.url) - - let tab - - try { - ;[tab] = await browser.tabs.query({ url: [url.href] }) - } catch (error) { - wappalyzer.log(error, 'driver', 'error') - } - - if (tab) { - request.responseHeaders.forEach((header) => { - const name = header.name.toLowerCase() - - headers[name] = headers[name] || [] - - headers[name].push( - (header.value || header.binaryValue || '').toString() - ) - }) - - if ( - headers['content-type'] && - /\/x?html/.test(headers['content-type'][0]) - ) { - wappalyzer.analyze(url, { headers }, { tab }) - } - } - } - }, - { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, - ['responseHeaders'] -) - -browser.runtime.onConnect.addListener((port) => { - port.onMessage.addListener(async (message) => { - if (message.id === undefined) { - return - } - - if (message.id !== 'log') { - wappalyzer.log(`Message from ${port.name}: ${message.id}`, 'driver') - } - - const pinnedCategory = await getOption('pinnedCategory') - - const url = wappalyzer.parseUrl(port.sender.tab ? port.sender.tab.url : '') - - const cookies = await browser.cookies.getAll({ - domain: `.${url.hostname}` - }) - - let response - - switch (message.id) { - case 'log': - wappalyzer.log(message.subject, message.source) - - break - case 'init': - wappalyzer.analyze(url, { cookies }, { tab: port.sender.tab }) - - break - case 'analyze': - if (message.subject.html) { - browser.i18n - .detectLanguage(message.subject.html) - .then(({ languages }) => { - const language = languages - .filter(({ percentage }) => percentage >= 75) - .map(({ language: lang }) => lang)[0] - - message.subject.language = language - - wappalyzer.analyze(url, message.subject, { tab: port.sender.tab }) - }) - } else { - wappalyzer.analyze(url, message.subject, { tab: port.sender.tab }) - } - - await setOption('hostnameCache', wappalyzer.hostnameCache) - - break - case 'ad_log': - wappalyzer.cacheDetectedAds(message.subject) - - break - case 'get_apps': - response = { - tabCache: tabCache[message.tab.id], - apps: wappalyzer.apps, - categories: wappalyzer.categories, - pinnedCategory, - termsAccepted: - userAgent() === 'chrome' || - (await getOption('termsAccepted', false)) - } - - break - case 'set_option': - await setOption(message.key, message.value) - - break - case 'get_js_patterns': - response = { - patterns: wappalyzer.jsPatterns - } - - break - case 'update_theme_mode': - // Sync theme mode to popup. - response = { - themeMode: await getOption('themeMode', false) - } - - break - default: - // Do nothing - } - - if (response) { - port.postMessage({ - id: message.id, - response - }) - } - }) -}) - -wappalyzer.driver.document = document - -/** - * Log messages to console - */ -wappalyzer.driver.log = (message, source, type) => { - const log = ['warn', 'error'].includes(type) ? type : 'log' - - console[log](`[wappalyzer ${type}]`, `[${source}]`, message) // eslint-disable-line no-console -} - -/** - * Display apps - */ -wappalyzer.driver.displayApps = async (detected, meta, context) => { - const { tab } = context - - if (tab === undefined) { - return - } - - tabCache[tab.id] = tabCache[tab.id] || { - detected: [] - } - - tabCache[tab.id].detected = detected - - const pinnedCategory = await getOption('pinnedCategory') - const dynamicIcon = await getOption('dynamicIcon', true) - - let found = false - - // Find the main application to display - ;[pinnedCategory].concat(categoryOrder).forEach((match) => { - Object.keys(detected).forEach((appName) => { - const app = detected[appName] - - app.props.cats.forEach((category) => { - if (category === match && !found) { - let icon = - app.props.icon && dynamicIcon ? app.props.icon : 'default.svg' - - if (/\.svg$/i.test(icon)) { - icon = `converted/${icon.replace(/\.svg$/, '.png')}` - } - - try { - browser.pageAction.setIcon({ - tabId: tab.id, - path: `../images/icons/${icon}` - }) - } catch (e) { - // Firefox for Android does not support setIcon see https://bugzilla.mozilla.org/show_bug.cgi?id=1331746 - } - - found = true - } - }) - }) - }) - - browser.pageAction.show(tab.id) -} - -/** - * Fetch and cache robots.txt for host - */ -wappalyzer.driver.getRobotsTxt = async (host, secure = false) => { - if (robotsTxtQueue[host]) { - return robotsTxtQueue[host] - } - - const tracking = await getOption('tracking', true) - const robotsTxtCache = await getOption('robotsTxtCache', {}) - - robotsTxtQueue[host] = new Promise(async (resolve) => { - if (!tracking) { - return resolve([]) - } - - if (host in robotsTxtCache) { - return resolve(robotsTxtCache[host]) - } - - const timeout = setTimeout(() => resolve([]), 3000) - - let response - - try { - response = await fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { - redirect: 'follow', - mode: 'no-cors' - }) - } catch (error) { - wappalyzer.log(error, 'driver', 'error') - - return resolve([]) - } - - clearTimeout(timeout) - - const robotsTxt = response.ok ? await response.text() : '' - - robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt) - - await setOption('robotsTxtCache', robotsTxtCache) - - delete robotsTxtQueue[host] - - return resolve(robotsTxtCache[host]) - }) - - return robotsTxtQueue[host] -} - -/** - * Anonymously track detected applications for research purposes - */ -wappalyzer.driver.ping = async ( - hostnameCache = { expires: 0, hostnames: {} }, - adCache = [] -) => { - const tracking = await getOption('tracking', true) - const termsAccepted = - userAgent() === 'chrome' || (await getOption('termsAccepted', false)) - - if (tracking && termsAccepted) { - if ( - hostnameCache.hostnames && - Object.keys(hostnameCache.hostnames).length - ) { - post('https://api.wappalyzer.com/ping/v1/', hostnameCache.hostnames) - } - - if (adCache.length) { - post('https://ad.wappalyzer.com/log/wp/', adCache) - } - - await setOption('robotsTxtCache', {}) - } -} - -// Init -;(async () => { - // Technologies - try { - const response = await fetch('../apps.json') - const json = await response.json() - - wappalyzer.apps = json.apps - wappalyzer.categories = json.categories - } catch (error) { - wappalyzer.log(`GET apps.json: ${error.message}`, 'driver', 'error') - } - - wappalyzer.parseJsPatterns() - - categoryOrder = Object.keys(wappalyzer.categories) - .map((categoryId) => parseInt(categoryId, 10)) - .sort( - (a, b) => - wappalyzer.categories[a].priority - wappalyzer.categories[b].priority - ) - - // Version check - const { version } = browser.runtime.getManifest() - const previousVersion = await getOption('version') - const upgradeMessage = await getOption('upgradeMessage', true) - - if (previousVersion === null) { - openTab({ - url: `${wappalyzer.config.websiteURL}installed` - }) - } else if (version !== previousVersion && upgradeMessage) { - openTab({ - url: `${wappalyzer.config.websiteURL}upgraded?v${version}`, - background: true - }) - } - - await setOption('version', version) - - // Hostname cache - wappalyzer.hostnameCache = await getOption('hostnameCache', { - expires: Date.now() + 1000 * 60 * 60 * 24, - hostnames: {} - }) - - // Run content script on all tabs - try { - const tabs = await browser.tabs.query({ - url: ['http://*/*', 'https://*/*'] - }) - - tabs.forEach(async (tab) => { - try { - await browser.tabs.executeScript(tab.id, { - file: '../js/content.js' - }) - } catch (error) { - // - } - }) - } catch (error) { - wappalyzer.log(error, 'driver', 'error') - } -})() diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index 7daf679af..d83d12b94 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -17,6 +17,8 @@ const Driver = { lastPing: Date.now(), async init() { + chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect) + await Driver.loadTechnologies() const hostnameCache = (await getOption('hostnames')) || {} @@ -31,8 +33,7 @@ const Driver = { ({ pattern: { regex, confidence, version }, match, - technology: name, - hits + technology: name }) => ({ pattern: { regex: new RegExp(regex, 'i'), @@ -42,8 +43,7 @@ const Driver = { match, technology: Wappalyzer.technologies.find( ({ name: _name }) => name === _name - ), - hits + ) }) ) } @@ -55,7 +55,6 @@ const Driver = { ads: (await getOption('ads')) || [] } - chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect) chrome.webRequest.onCompleted.addListener( Driver.onWebRequestComplete, { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, @@ -126,6 +125,8 @@ const Driver = { }, onRuntimeConnect(port) { + Driver.log(`Connected to ${port.name}`) + port.onMessage.addListener(async ({ func, args }) => { if (!func) { return @@ -190,7 +191,7 @@ const Driver = { domain: `.${url.hostname}` }) - await Driver.onDetect(url, await analyze(href, items), language) + await Driver.onDetect(url, await analyze(href, items), language, true) } catch (error) { Driver.error(error) } @@ -200,40 +201,35 @@ const Driver = { return Wappalyzer.technologies }, - async onDetect(url, detections = [], language) { + async onDetect(url, detections = [], language, incrementHits = false) { + if (!detections.length) { + return + } + + const { hostname, href } = url + // Cache detections - // eslint-disable-next-line standard/computed-property-even-spacing - Driver.cache.hostnames[url.hostname] = { - ...(Driver.cache.hostnames[url.hostname] || { - detections: [] + const cache = (Driver.cache.hostnames[hostname] = { + ...(Driver.cache.hostnames[hostname] || { + detections: [], + hits: 0 }), dateTime: Date.now() - } - - Driver.cache.hostnames[url.hostname].language = - Driver.cache.hostnames[url.hostname].language || language + }) - detections.forEach((detection) => { - const foo = Driver.cache.hostnames[url.hostname].detections - const { - technology: { name }, - pattern: { regex } - } = detection + // Remove duplicates + cache.detections = cache.detections = cache.detections.concat(detections) - const cache = foo.find( - ({ technology: { name: _name }, pattern: { regex: _regex } }) => - name === _name && (!regex || regex) === _regex - ) + cache.detections.filter( + ({ technology: { name }, pattern: { regex } }, index) => + cache.detections.findIndex( + ({ technology: { name: _name }, pattern: { regex: _regex } }) => + name === _name && (!regex || regex.toString() === _regex.toString()) + ) === index + ) - if (cache) { - cache.hits += 1 - } else { - foo.push({ - ...detection, - hits: 1 - }) - } - }) + cache.hits += incrementHits ? 1 : 0 + cache.language = cache.language || language // Expire cache Driver.cache.hostnames = Object.keys(Driver.cache.hostnames).reduce( @@ -277,14 +273,16 @@ const Driver = { ) ) - const resolved = resolve(Driver.cache.hostnames[url.hostname].detections) + const resolved = resolve(Driver.cache.hostnames[hostname].detections) await Driver.setIcon(url, resolved) - const tabs = await promisify(chrome.tabs, 'query', { url: [url.href] }) + const tabs = await promisify(chrome.tabs, 'query', { url: [href] }) tabs.forEach(({ id }) => (Driver.cache.tabs[id] = resolved)) + Driver.log({ hostname, technologies: resolved }) + await Driver.ping() }, @@ -306,14 +304,13 @@ const Driver = { categories.some(({ id }) => id === pinnedCategory) ) - ;({ icon } = - pinned || + ;({ icon } = pinned || technologies.sort(({ categories: a }, { categories: b }) => { const max = (value) => value.reduce((max, { priority }) => Math.max(max, priority)) return max(a) > max(b) ? -1 : 1 - })[0]) + })[0] || { icon }) } const tabs = await promisify(chrome.tabs, 'query', { url: [url.href] }) @@ -437,36 +434,34 @@ const Driver = { if (tracking && termsAccepted) { const count = Object.keys(Driver.cache.hostnames).length - if (count && (count >= 50 || Driver.lastPing < Date.now() - 5000)) { + if (count && (count >= 50 || Driver.lastPing < Date.now() - expiry)) { await Driver.post( 'https://api.wappalyzer.com/ping/v1/', Object.keys(Driver.cache.hostnames).reduce((hostnames, hostname) => { - const { language, detections } = Driver.cache.hostnames[hostname] + // eslint-disable-next-line standard/computed-property-even-spacing + const { language, detections, hits } = Driver.cache.hostnames[ + hostname + ] hostnames[hostname] = hostnames[hostname] || { - applications: {}, + applications: resolve(detections).reduce( + (technologies, { name, confidence, version }) => { + if (confidence === 100) { + technologies[name] = { + version, + hits + } + + return technologies + } + }, + {} + ), meta: { language } } - resolve(detections).forEach(({ name, confidence, version }) => { - if (confidence === 100) { - console.log( - name, - detections.find( - ({ technology: { name: _name } }) => name === _name - ) - ) - hostnames[hostname].applications[name] = { - version, - hits: detections.find( - ({ technology: { name: _name } }) => name === _name - ).pattern.hits - } - } - }) - return hostnames }, {}) ) diff --git a/src/drivers/webextension/js/lib/iframe.js b/src/drivers/webextension/js/lib/iframe.js index 38ea3af38..9fd152494 100644 --- a/src/drivers/webextension/js/lib/iframe.js +++ b/src/drivers/webextension/js/lib/iframe.js @@ -124,12 +124,12 @@ var exports = {}; var port = chrome.runtime.connect({name:"adparser"}); port.onMessage.addListener((message) => { - if ( message && message.tracking_enabled ) { - - utilCallback(); - } else { - - utilElseCallback(); + if ( message && typeof message.tracking_enabled !== 'undefined' ) { + if (message.tracking_enabled) { + utilCallback(); + } else { + utilElseCallback(); + } } }); @@ -1111,7 +1111,6 @@ var exports = {}; if ( origUrl.indexOf('google.com/_/chrome/newtab') === -1 ) { var onBlockedRobotsMessage = function() { - return // TODO var log; log = _logGen.log('invalid-robotstxt', []); log.doc.finalPageUrl = log.doc.url; diff --git a/src/drivers/webextension/js/popup.bak.js b/src/drivers/webextension/js/popup.bak.js deleted file mode 100644 index 251df8200..000000000 --- a/src/drivers/webextension/js/popup.bak.js +++ /dev/null @@ -1,333 +0,0 @@ -'use strict' -/* eslint-env browser */ -/* globals chrome */ - -let pinnedCategory = null -let termsAccepted = false - -const port = chrome.runtime.connect({ - name: 'popup.js' -}) - -function slugify(string) { - return string - .toLowerCase() - .replace(/[^a-z0-9-]/g, '-') - .replace(/--+/g, '-') - .replace(/(?:^-|-$)/, '') -} - -function i18n() { - const nodes = document.querySelectorAll('[data-i18n]') - - Array.prototype.forEach.call(nodes, (node) => { - node.innerHTML = browser.i18n.getMessage(node.dataset.i18n) - }) -} - -function replaceDom(domTemplate) { - const container = document.getElementsByClassName('container')[0] - - while (container.firstChild) { - container.removeChild(container.firstChild) - } - - container.appendChild(jsonToDOM(domTemplate, document, {})) - - i18n() - - Array.from( - document.querySelectorAll('.detected__category-pin-wrapper') - ).forEach((pin) => { - pin.addEventListener('click', () => { - const categoryId = parseInt(pin.dataset.categoryId, 10) - - if (categoryId === pinnedCategory) { - pin.className = 'detected__category-pin-wrapper' - - pinnedCategory = null - } else { - const active = document.querySelector( - '.detected__category-pin-wrapper--active' - ) - - if (active) { - active.className = 'detected__category-pin-wrapper' - } - - pin.className = - 'detected__category-pin-wrapper detected__category-pin-wrapper--active' - - pinnedCategory = categoryId - } - - port.postMessage({ - id: 'set_option', - key: 'pinnedCategory', - value: pinnedCategory - }) - }) - }) - - Array.from(document.querySelectorAll('a')).forEach((link) => { - link.addEventListener('click', () => { - browser.tabs.create({ url: link.href }) - - return false - }) - }) -} - -function replaceDomWhenReady(dom) { - if (/complete|interactive|loaded/.test(document.readyState)) { - replaceDom(dom) - } else { - document.addEventListener('DOMContentLoaded', () => { - replaceDom(dom) - }) - } -} - -function appsToDomTemplate(response) { - let template = [] - - if (response.tabCache && Object.keys(response.tabCache.detected).length > 0) { - const categories = {} - - // Group apps by category - for (const appName in response.tabCache.detected) { - response.apps[appName].cats.forEach((cat) => { - categories[cat] = categories[cat] || { - name: response.categories[cat].name, - apps: [] - } - - categories[cat].apps[appName] = appName - }) - } - - for (const cat in categories) { - const apps = [] - - for (const appName in categories[cat].apps) { - const { confidenceTotal, version } = response.tabCache.detected[appName] - - apps.push([ - 'a', - { - class: 'detected__app', - href: `https://www.wappalyzer.com/technologies/${slugify( - categories[cat].name - )}/${slugify(appName)}` - }, - [ - 'img', - { - class: 'detected__app-icon', - src: `../images/icons/${response.apps[appName].icon || - 'default.svg'}` - } - ], - [ - 'span', - { - class: 'detected__app-name' - }, - appName - ], - version - ? [ - 'span', - { - class: 'detected__app-version' - }, - version - ] - : null, - confidenceTotal < 100 - ? [ - 'span', - { - class: 'detected__app-confidence' - }, - `${confidenceTotal}% sure` - ] - : null - ]) - } - - template.push([ - 'div', - { - class: 'detected__category' - }, - [ - 'div', - { - class: 'detected__category-name' - }, - [ - 'a', - { - class: 'detected__category-link', - href: `https://www.wappalyzer.com/categories/${slugify( - response.categories[cat].name - )}` - }, - browser.i18n.getMessage(`categoryName${cat}`) - ], - [ - 'span', - { - class: `detected__category-pin-wrapper${ - parseInt(pinnedCategory, 10) === parseInt(cat, 10) - ? ' detected__category-pin-wrapper--active' - : '' - }`, - 'data-category-id': cat, - title: browser.i18n.getMessage('categoryPin') - }, - [ - 'img', - { - class: 'detected__category-pin detected__category-pin--active', - src: '../images/pin-active.svg' - } - ], - [ - 'img', - { - class: - 'detected__category-pin detected__category-pin--inactive', - src: '../images/pin.svg' - } - ] - ] - ], - [ - 'div', - { - class: 'detected__apps' - }, - apps - ] - ]) - } - - template = [ - 'div', - { - class: 'detected' - }, - template - ] - } else { - template = [ - 'div', - { - class: 'empty' - }, - [ - 'span', - { - class: 'empty__text' - }, - browser.i18n.getMessage('noAppsDetected') - ] - ] - } - - return template -} - -async function getApps() { - try { - const tabs = await browser.tabs.query({ - active: true, - currentWindow: true - }) - - const url = new URL(tabs[0].url) - - document.querySelector( - '.footer__link' - ).href = `https://www.wappalyzer.com/alerts/manage?url=${encodeURIComponent( - `${url.protocol}//${url.hostname}` - )}` - - port.postMessage({ - id: 'get_apps', - tab: tabs[0] - }) - } catch (error) { - console.error(error) // eslint-disable-line no-console - } -} - -/** - * Async function to update body class based on option. - */ -function getThemeMode() { - try { - port.postMessage({ - id: 'update_theme_mode' - }) - } catch (error) { - console.error(error) // eslint-disable-line no-console - } -} - -/** - * Update theme mode based on browser option. - * @param {object} res Response from port listener. - */ -function updateThemeMode(res) { - if (res.hasOwnProperty('themeMode') && res.themeMode !== false) { - document.body.classList.add('theme-mode-sync') - } -} - -function displayApps(response) { - pinnedCategory = response.pinnedCategory // eslint-disable-line prefer-destructuring - termsAccepted = response.termsAccepted // eslint-disable-line prefer-destructuring - - if (termsAccepted) { - replaceDomWhenReady(appsToDomTemplate(response)) - } else { - i18n() - - const wrapper = document.querySelector('.terms__wrapper') - - document.querySelector('.terms__accept').addEventListener('click', () => { - port.postMessage({ - id: 'set_option', - key: 'termsAccepted', - value: true - }) - - wrapper.classList.remove('terms__wrapper--active') - - getApps() - }) - - wrapper.classList.add('terms__wrapper--active') - } -} - -port.onMessage.addListener((message) => { - switch (message.id) { - case 'get_apps': - displayApps(message.response) - - break - case 'update_theme_mode': - updateThemeMode(message.response) - - break - default: - // Do nothing - } -}) - -getThemeMode() -getApps() diff --git a/src/drivers/webextension/js/wappalyzer.bak.js b/src/drivers/webextension/js/wappalyzer.bak.js deleted file mode 100644 index e69399e87..000000000 --- a/src/drivers/webextension/js/wappalyzer.bak.js +++ /dev/null @@ -1,727 +0,0 @@ -const validation = { - hostname: /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/, - hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/ -} - -/** - * Enclose string in array - */ -function asArray(value) { - return Array.isArray(value) ? value : [value] -} - -/** - * - */ -function asyncForEach(iterable, iterator) { - return Promise.all( - (iterable || []).map( - (item) => - new Promise((resolve) => setTimeout(() => resolve(iterator(item)), 1)) - ) - ) -} - -/** - * Mark application as detected, set confidence and version - */ -function addDetected(app, pattern, type, value, key) { - app.detected = true - - // Set confidence level - app.confidence[`${type} ${key ? `${key} ` : ''}${pattern.regex}`] = - pattern.confidence === undefined ? 100 : parseInt(pattern.confidence, 10) - - // Detect version number - if (pattern.version) { - const versions = [] - const matches = pattern.regex.exec(value) - - let { version } = pattern - - if (matches) { - matches.forEach((match, i) => { - // Parse ternary operator - const ternary = new RegExp(`\\\\${i}\\?([^:]+):(.*)$`).exec(version) - - if (ternary && ternary.length === 3) { - version = version.replace(ternary[0], match ? ternary[1] : ternary[2]) - } - - // Replace back references - version = version - .trim() - .replace(new RegExp(`\\\\${i}`, 'g'), match || '') - }) - - if (version && !versions.includes(version)) { - versions.push(version) - } - - if (versions.length) { - // Use the longest detected version number - app.version = versions.reduce((a, b) => (a.length > b.length ? a : b)) - } - } - } -} - -function resolveExcludes(apps, detected) { - const excludes = [] - const detectedApps = Object.assign({}, apps, detected) - - // Exclude app in detected apps only - Object.keys(detectedApps).forEach((appName) => { - const app = detectedApps[appName] - - if (app.props.excludes) { - asArray(app.props.excludes).forEach((excluded) => { - excludes.push(excluded) - }) - } - }) - - // Remove excluded applications - Object.keys(apps).forEach((appName) => { - if (excludes.includes(appName)) { - delete apps[appName] - } - }) -} - -class Application { - constructor(name, props, detected) { - this.confidence = {} - this.confidenceTotal = 0 - this.detected = Boolean(detected) - this.excludes = [] - this.name = name - this.props = props - this.version = '' - } - - /** - * Calculate confidence total - */ - getConfidence() { - let total = 0 - - Object.keys(this.confidence).forEach((id) => { - total += this.confidence[id] - }) - - this.confidenceTotal = Math.min(total, 100) - - return this.confidenceTotal - } -} - -class Wappalyzer { - constructor() { - this.apps = {} - this.categories = {} - this.driver = {} - this.jsPatterns = {} - this.detected = {} - this.hostnameCache = { - expires: Date.now() + 1000 * 60 * 60 * 24, - hostnames: {} - } - this.adCache = [] - - this.config = { - websiteURL: 'https://www.wappalyzer.com/', - twitterURL: 'https://twitter.com/Wappalyzer', - githubURL: 'https://github.com/AliasIO/Wappalyzer' - } - } - - /** - * Log messages to console - */ - log(message, source, type) { - if (this.driver.log) { - this.driver.log(message, source || '', type || 'debug') - } - } - - analyze(url, data, context) { - const apps = {} - const promises = [] - const startTime = new Date() - const { scripts, cookies, headers, js } = data - - let { html } = data - - if (this.detected[url.canonical] === undefined) { - this.detected[url.canonical] = {} - } - - const metaTags = [] - - // Additional information - let language = null - - if (html) { - if (typeof html !== 'string') { - html = '' - } - - let matches = data.html.match( - new RegExp(']*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"', 'i') - ) - - language = matches && matches.length ? matches[1] : data.language || null - - // Meta tags - const regex = /]+>/gi - - do { - matches = regex.exec(html) - - if (!matches) { - break - } - - metaTags.push(matches[0]) - } while (matches) - } - - Object.keys(this.apps).forEach((appName) => { - apps[appName] = - this.detected[url.canonical] && this.detected[url.canonical][appName] - ? this.detected[url.canonical][appName] - : new Application(appName, this.apps[appName]) - - const app = apps[appName] - - promises.push(this.analyzeUrl(app, url)) - - if (html) { - promises.push(this.analyzeHtml(app, html)) - promises.push(this.analyzeMeta(app, metaTags)) - } - - if (scripts) { - promises.push(this.analyzeScripts(app, scripts)) - } - - if (cookies) { - promises.push(this.analyzeCookies(app, cookies)) - } - - if (headers) { - promises.push(this.analyzeHeaders(app, headers)) - } - }) - - if (js) { - Object.keys(js).forEach((appName) => { - if (typeof js[appName] !== 'function') { - promises.push(this.analyzeJs(apps[appName], js[appName])) - } - }) - } - - return new Promise(async (resolve) => { - await Promise.all(promises) - - Object.keys(apps).forEach((appName) => { - const app = apps[appName] - - if (!app.detected || !app.getConfidence()) { - delete apps[app.name] - } - }) - - resolveExcludes(apps, this.detected[url]) - this.resolveImplies(apps, url.canonical) - - this.cacheDetectedApps(apps, url.canonical) - this.trackDetectedApps(apps, url, language) - - this.log( - `Processing ${Object.keys(data).join(', ')} took ${( - (new Date() - startTime) / - 1000 - ).toFixed(2)}s (${url.hostname})`, - 'core' - ) - - if (Object.keys(apps).length) { - this.log( - `Identified ${Object.keys(apps).join(', ')} (${url.hostname})`, - 'core' - ) - } - - this.driver.displayApps( - this.detected[url.canonical], - { language }, - context - ) - - return resolve() - }) - } - - /** - * Cache detected ads - */ - cacheDetectedAds(ad) { - this.adCache.push(ad) - } - - /** - * - */ - robotsTxtAllows(url) { - return new Promise(async (resolve, reject) => { - const parsed = this.parseUrl(url) - - if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { - return reject() - } - - const robotsTxt = await this.driver.getRobotsTxt( - parsed.host, - parsed.protocol === 'https:' - ) - - if ( - robotsTxt.some( - (disallowedPath) => parsed.pathname.indexOf(disallowedPath) === 0 - ) - ) { - return reject() - } - - return resolve() - }) - } - - /** - * Parse a URL - */ - parseUrl(url) { - const a = this.driver.document.createElement('a') - - a.href = url - - a.canonical = `${a.protocol}//${a.host}${a.pathname}` - - return a - } - - /** - * - */ - static parseRobotsTxt(robotsTxt) { - const disallow = [] - - let userAgent - - robotsTxt.split('\n').forEach((line) => { - let matches = /^User-agent:\s*(.+)$/i.exec(line.trim()) - - if (matches) { - userAgent = matches[1].toLowerCase() - } else if (userAgent === '*' || userAgent === 'wappalyzer') { - matches = /^Disallow:\s*(.+)$/i.exec(line.trim()) - - if (matches) { - disallow.push(matches[1]) - } - } - }) - - return disallow - } - - /** - * - */ - ping() { - if ( - !this.hostnameCache.hostnames || - Object.keys(this.hostnameCache.hostnames).length > 50 || - this.hostnameCache.expires < Date.now() - ) { - this.driver.ping(this.hostnameCache) - - this.hostnameCache = { - expires: Date.now() + 1000 * 60 * 60 * 24, - hostnames: {} - } - } - - if (this.adCache.length > 50) { - this.driver.ping(undefined, this.adCache) - - this.adCache = [] - } - } - - /** - * Parse apps.json patterns - */ - parsePatterns(patterns) { - if (!patterns) { - return [] - } - - let parsed = {} - - // Convert string to object containing array containing string - if (typeof patterns === 'string' || Array.isArray(patterns)) { - patterns = { - main: asArray(patterns) - } - } - - Object.keys(patterns).forEach((key) => { - parsed[key] = [] - - asArray(patterns[key]).forEach((pattern) => { - const attrs = {} - - pattern.split('\\;').forEach((attr, i) => { - if (i) { - // Key value pairs - attr = attr.split(':') - - if (attr.length > 1) { - attrs[attr.shift()] = attr.join(':') - } - } else { - attrs.string = attr - - try { - attrs.regex = new RegExp(attr.replace('/', '/'), 'i') // Escape slashes in regular expression - } catch (error) { - attrs.regex = new RegExp() - - this.log(`${error.message}: ${attr}`, 'error', 'core') - } - } - }) - - parsed[key].push(attrs) - }) - }) - - // Convert back to array if the original pattern list was an array (or string) - if ('main' in parsed) { - parsed = parsed.main - } - - return parsed - } - - /** - * Parse JavaScript patterns - */ - parseJsPatterns() { - Object.keys(this.apps).forEach((appName) => { - if (this.apps[appName].js) { - this.jsPatterns[appName] = this.parsePatterns(this.apps[appName].js) - } - }) - } - - resolveImplies(apps, url) { - let checkImplies = true - - const resolve = (appName) => { - const app = apps[appName] - - if (app && app.props.implies) { - asArray(app.props.implies).forEach((implied) => { - ;[implied] = this.parsePatterns(implied) - - if (!this.apps[implied.string]) { - this.log( - `Implied application ${implied.string} does not exist`, - 'core', - 'warn' - ) - - return - } - - if (!(implied.string in apps)) { - apps[implied.string] = - this.detected[url] && this.detected[url][implied.string] - ? this.detected[url][implied.string] - : new Application( - implied.string, - this.apps[implied.string], - true - ) - - checkImplies = true - } - - // Apply app confidence to implied app - Object.keys(app.confidence).forEach((id) => { - apps[implied.string].confidence[`${id} implied by ${appName}`] = - app.confidence[id] * - (implied.confidence === undefined ? 1 : implied.confidence / 100) - }) - }) - } - } - - // Implied applications - // Run several passes as implied apps may imply other apps - while (checkImplies) { - checkImplies = false - - Object.keys(apps).forEach(resolve) - } - } - - /** - * Cache detected applications - */ - cacheDetectedApps(apps, url) { - Object.keys(apps).forEach((appName) => { - const app = apps[appName] - - // Per URL - this.detected[url][appName] = app - - Object.keys(app.confidence).forEach((id) => { - this.detected[url][appName].confidence[id] = app.confidence[id] - }) - }) - - if (this.driver.ping instanceof Function) { - this.ping() - } - } - - /** - * Track detected applications - */ - trackDetectedApps(apps, url, language) { - if (!(this.driver.ping instanceof Function)) { - return - } - - const hostname = `${url.protocol}//${url.hostname}` - - Object.keys(apps).forEach((appName) => { - const app = apps[appName] - - if (this.detected[url.canonical][appName].getConfidence() >= 100) { - if ( - validation.hostname.test(url.hostname) && - !validation.hostnameBlacklist.test(url.hostname) - ) { - if (!(hostname in this.hostnameCache.hostnames)) { - this.hostnameCache.hostnames[hostname] = { - applications: {}, - meta: {} - } - } - - if ( - !(appName in this.hostnameCache.hostnames[hostname].applications) - ) { - this.hostnameCache.hostnames[hostname].applications[appName] = { - hits: 0 - } - } - - this.hostnameCache.hostnames[hostname].applications[appName].hits += 1 - - if (apps[appName].version) { - this.hostnameCache.hostnames[hostname].applications[ - appName - ].version = app.version - } - } - } - }) - - if (hostname in this.hostnameCache.hostnames) { - this.hostnameCache.hostnames[hostname].meta.language = language - } - - this.ping() - } - - /** - * Analyze URL - */ - analyzeUrl(app, url) { - const patterns = this.parsePatterns(app.props.url) - - if (!patterns.length) { - return Promise.resolve() - } - - return asyncForEach(patterns, (pattern) => { - if (pattern.regex.test(url.canonical)) { - addDetected(app, pattern, 'url', url.canonical) - } - }) - } - - /** - * Analyze HTML - */ - analyzeHtml(app, html) { - const patterns = this.parsePatterns(app.props.html) - - if (!patterns.length) { - return Promise.resolve() - } - - return asyncForEach(patterns, (pattern) => { - if (pattern.regex.test(html)) { - addDetected(app, pattern, 'html', html) - } - }) - } - - /** - * Analyze script tag - */ - analyzeScripts(app, scripts) { - const patterns = this.parsePatterns(app.props.script) - - if (!patterns.length) { - return Promise.resolve() - } - - return asyncForEach(patterns, (pattern) => { - scripts.forEach((uri) => { - if (pattern.regex.test(uri)) { - addDetected(app, pattern, 'script', uri) - } - }) - }) - } - - /** - * Analyze meta tag - */ - analyzeMeta(app, metaTags) { - const patterns = this.parsePatterns(app.props.meta) - const promises = [] - - if (!app.props.meta) { - return Promise.resolve() - } - - metaTags.forEach((match) => { - Object.keys(patterns).forEach((meta) => { - const r = new RegExp(`(?:name|property)=["']${meta}["']`, 'i') - - if (r.test(match)) { - const content = match.match(/content=("|')([^"']+)("|')/i) - - promises.push( - asyncForEach(patterns[meta], (pattern) => { - if ( - content && - content.length === 4 && - pattern.regex.test(content[2]) - ) { - addDetected(app, pattern, 'meta', content[2], meta) - } - }) - ) - } - }) - }) - - return Promise.all(promises) - } - - /** - * Analyze response headers - */ - analyzeHeaders(app, headers) { - const patterns = this.parsePatterns(app.props.headers) - const promises = [] - - Object.keys(patterns).forEach((headerName) => { - if (typeof patterns[headerName] !== 'function') { - promises.push( - asyncForEach(patterns[headerName], (pattern) => { - headerName = headerName.toLowerCase() - - if (headerName in headers) { - headers[headerName].forEach((headerValue) => { - if (pattern.regex.test(headerValue)) { - addDetected(app, pattern, 'headers', headerValue, headerName) - } - }) - } - }) - ) - } - }) - - return promises ? Promise.all(promises) : Promise.resolve() - } - - /** - * Analyze cookies - */ - analyzeCookies(app, cookies) { - const patterns = this.parsePatterns(app.props.cookies) - const promises = [] - - Object.keys(patterns).forEach((cookieName) => { - if (typeof patterns[cookieName] !== 'function') { - const cookieNameLower = cookieName.toLowerCase() - - promises.push( - asyncForEach(patterns[cookieName], (pattern) => { - const cookie = cookies.find( - (_cookie) => _cookie.name.toLowerCase() === cookieNameLower - ) - - if (cookie && pattern.regex.test(cookie.value)) { - addDetected(app, pattern, 'cookies', cookie.value, cookieName) - } - }) - ) - } - }) - - return promises ? Promise.all(promises) : Promise.resolve() - } - - /** - * Analyze JavaScript variables - */ - analyzeJs(app, results) { - const promises = [] - - Object.keys(results).forEach((string) => { - if (typeof results[string] !== 'function') { - promises.push( - asyncForEach(Object.keys(results[string]), (index) => { - const pattern = this.jsPatterns[app.name][string][index] - const value = results[string][index] - - if (pattern && pattern.regex.test(value)) { - addDetected(app, pattern, 'js', value, string) - } - }) - ) - } - }) - - return promises ? Promise.all(promises) : Promise.resolve() - } -} - -if (typeof module === 'object') { - module.exports = Wappalyzer -} diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json index e034825ac..e00572b6b 100644 --- a/src/drivers/webextension/manifest.json +++ b/src/drivers/webextension/manifest.json @@ -36,7 +36,6 @@ "https://*/*" ], "js": [ - "node_modules/webextension-polyfill/dist/browser-polyfill.js", "js/content.js" ], "run_at": "document_idle" @@ -46,12 +45,7 @@ "http://*/*", "https://*/*" ], - "exclude_matches": [ - "https://*.modirum.com/*", - "https://www.alphaecommerce.gr/*" - ], "js": [ - "node_modules/webextension-polyfill/dist/browser-polyfill.js", "js/lib/iframe.js" ], "run_at": "document_start", diff --git a/src/wappalyzer.js b/src/wappalyzer.js index e69399e87..e72eb8af7 100644 --- a/src/wappalyzer.js +++ b/src/wappalyzer.js @@ -1,727 +1,331 @@ -const validation = { - hostname: /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/, - hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/ -} - -/** - * Enclose string in array - */ -function asArray(value) { - return Array.isArray(value) ? value : [value] -} - -/** - * - */ -function asyncForEach(iterable, iterator) { - return Promise.all( - (iterable || []).map( - (item) => - new Promise((resolve) => setTimeout(() => resolve(iterator(item)), 1)) - ) - ) -} - -/** - * Mark application as detected, set confidence and version - */ -function addDetected(app, pattern, type, value, key) { - app.detected = true - - // Set confidence level - app.confidence[`${type} ${key ? `${key} ` : ''}${pattern.regex}`] = - pattern.confidence === undefined ? 100 : parseInt(pattern.confidence, 10) - - // Detect version number - if (pattern.version) { - const versions = [] - const matches = pattern.regex.exec(value) - - let { version } = pattern - - if (matches) { - matches.forEach((match, i) => { - // Parse ternary operator - const ternary = new RegExp(`\\\\${i}\\?([^:]+):(.*)$`).exec(version) - - if (ternary && ternary.length === 3) { - version = version.replace(ternary[0], match ? ternary[1] : ternary[2]) - } - - // Replace back references - version = version - .trim() - .replace(new RegExp(`\\\\${i}`, 'g'), match || '') - }) - - if (version && !versions.includes(version)) { - versions.push(version) - } +'use strict' + +const Wappalyzer = { + technologies: [], + categories: [], + + slugify(string) { + return string + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/--+/g, '-') + .replace(/(?:^-|-$)/, '') + }, + + getTechnology(name) { + return Wappalyzer.technologies.find(({ name: _name }) => name === _name) + }, + + getCategory(id) { + return Wappalyzer.categories.find(({ id: _id }) => id === _id) + }, + + resolve(detections = []) { + const resolved = detections.reduce((resolved, { technology }) => { + if ( + resolved.findIndex( + ({ technology: { name } }) => name === technology.name + ) === -1 + ) { + let version = '' + let confidence = 0 + + detections.forEach(({ technology: { name }, pattern, match }) => { + if (name === technology.name) { + const versionValue = Wappalyzer.resolveVersion(pattern, match) + + confidence = Math.min(100, confidence + pattern.confidence) + version = + versionValue.length > version.length && versionValue.length <= 10 + ? versionValue + : version + } + }) - if (versions.length) { - // Use the longest detected version number - app.version = versions.reduce((a, b) => (a.length > b.length ? a : b)) + resolved.push({ technology, confidence, version }) } - } - } -} - -function resolveExcludes(apps, detected) { - const excludes = [] - const detectedApps = Object.assign({}, apps, detected) - - // Exclude app in detected apps only - Object.keys(detectedApps).forEach((appName) => { - const app = detectedApps[appName] - if (app.props.excludes) { - asArray(app.props.excludes).forEach((excluded) => { - excludes.push(excluded) + return resolved + }, []) + + Wappalyzer.resolveExcludes(resolved) + Wappalyzer.resolveImplies(resolved) + + return resolved.map( + ({ + technology: { name, slug, categories, icon, website }, + confidence, + version + }) => ({ + name, + slug, + categories: categories.map((id) => Wappalyzer.getCategory(id)), + confidence, + version, + icon, + website }) - } - }) - - // Remove excluded applications - Object.keys(apps).forEach((appName) => { - if (excludes.includes(appName)) { - delete apps[appName] - } - }) -} - -class Application { - constructor(name, props, detected) { - this.confidence = {} - this.confidenceTotal = 0 - this.detected = Boolean(detected) - this.excludes = [] - this.name = name - this.props = props - this.version = '' - } - - /** - * Calculate confidence total - */ - getConfidence() { - let total = 0 - - Object.keys(this.confidence).forEach((id) => { - total += this.confidence[id] - }) - - this.confidenceTotal = Math.min(total, 100) - - return this.confidenceTotal - } -} - -class Wappalyzer { - constructor() { - this.apps = {} - this.categories = {} - this.driver = {} - this.jsPatterns = {} - this.detected = {} - this.hostnameCache = { - expires: Date.now() + 1000 * 60 * 60 * 24, - hostnames: {} - } - this.adCache = [] - - this.config = { - websiteURL: 'https://www.wappalyzer.com/', - twitterURL: 'https://twitter.com/Wappalyzer', - githubURL: 'https://github.com/AliasIO/Wappalyzer' - } - } - - /** - * Log messages to console - */ - log(message, source, type) { - if (this.driver.log) { - this.driver.log(message, source || '', type || 'debug') - } - } - - analyze(url, data, context) { - const apps = {} - const promises = [] - const startTime = new Date() - const { scripts, cookies, headers, js } = data + ) + }, - let { html } = data + resolveVersion({ version, regex }, match) { + let resolved = version - if (this.detected[url.canonical] === undefined) { - this.detected[url.canonical] = {} - } + if (version) { + const matches = regex.exec(match) - const metaTags = [] + if (matches) { + matches.forEach((match, index) => { + // Parse ternary operator + const ternary = new RegExp(`\\\\${index}\\?([^:]+):(.*)$`).exec( + version + ) - // Additional information - let language = null + if (ternary && ternary.length === 3) { + resolved = version.replace( + ternary[0], + match ? ternary[1] : ternary[2] + ) + } - if (html) { - if (typeof html !== 'string') { - html = '' + // Replace back references + resolved = resolved + .trim() + .replace(new RegExp(`\\\\${index}`, 'g'), match || '') + }) } - - let matches = data.html.match( - new RegExp(']*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"', 'i') - ) - - language = matches && matches.length ? matches[1] : data.language || null - - // Meta tags - const regex = /]+>/gi - - do { - matches = regex.exec(html) - - if (!matches) { - break - } - - metaTags.push(matches[0]) - } while (matches) } - Object.keys(this.apps).forEach((appName) => { - apps[appName] = - this.detected[url.canonical] && this.detected[url.canonical][appName] - ? this.detected[url.canonical][appName] - : new Application(appName, this.apps[appName]) + return resolved + }, - const app = apps[appName] + resolveExcludes(resolved) { + resolved.forEach(({ technology }) => { + technology.excludes.forEach((name) => { + const excluded = Wappalyzer.getTechnology(name) - promises.push(this.analyzeUrl(app, url)) - - if (html) { - promises.push(this.analyzeHtml(app, html)) - promises.push(this.analyzeMeta(app, metaTags)) - } - - if (scripts) { - promises.push(this.analyzeScripts(app, scripts)) - } - - if (cookies) { - promises.push(this.analyzeCookies(app, cookies)) - } - - if (headers) { - promises.push(this.analyzeHeaders(app, headers)) - } - }) - - if (js) { - Object.keys(js).forEach((appName) => { - if (typeof js[appName] !== 'function') { - promises.push(this.analyzeJs(apps[appName], js[appName])) + if (!excluded) { + throw new Error(`Excluded technology does not exist: ${name}`) } - }) - } - - return new Promise(async (resolve) => { - await Promise.all(promises) - Object.keys(apps).forEach((appName) => { - const app = apps[appName] + const index = resolved.findIndex(({ name }) => name === excluded.name) - if (!app.detected || !app.getConfidence()) { - delete apps[app.name] + if (index === -1) { + resolved.splice(index, 1) } }) - - resolveExcludes(apps, this.detected[url]) - this.resolveImplies(apps, url.canonical) - - this.cacheDetectedApps(apps, url.canonical) - this.trackDetectedApps(apps, url, language) - - this.log( - `Processing ${Object.keys(data).join(', ')} took ${( - (new Date() - startTime) / - 1000 - ).toFixed(2)}s (${url.hostname})`, - 'core' - ) - - if (Object.keys(apps).length) { - this.log( - `Identified ${Object.keys(apps).join(', ')} (${url.hostname})`, - 'core' - ) - } - - this.driver.displayApps( - this.detected[url.canonical], - { language }, - context - ) - - return resolve() }) - } + }, - /** - * Cache detected ads - */ - cacheDetectedAds(ad) { - this.adCache.push(ad) - } - - /** - * - */ - robotsTxtAllows(url) { - return new Promise(async (resolve, reject) => { - const parsed = this.parseUrl(url) + resolveImplies(resolved) { + let done = false - if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { - return reject() - } - - const robotsTxt = await this.driver.getRobotsTxt( - parsed.host, - parsed.protocol === 'https:' - ) - - if ( - robotsTxt.some( - (disallowedPath) => parsed.pathname.indexOf(disallowedPath) === 0 - ) - ) { - return reject() - } - - return resolve() - }) - } - - /** - * Parse a URL - */ - parseUrl(url) { - const a = this.driver.document.createElement('a') - - a.href = url - - a.canonical = `${a.protocol}//${a.host}${a.pathname}` - - return a - } - - /** - * - */ - static parseRobotsTxt(robotsTxt) { - const disallow = [] - - let userAgent - - robotsTxt.split('\n').forEach((line) => { - let matches = /^User-agent:\s*(.+)$/i.exec(line.trim()) - - if (matches) { - userAgent = matches[1].toLowerCase() - } else if (userAgent === '*' || userAgent === 'wappalyzer') { - matches = /^Disallow:\s*(.+)$/i.exec(line.trim()) - - if (matches) { - disallow.push(matches[1]) - } - } - }) - - return disallow - } - - /** - * - */ - ping() { - if ( - !this.hostnameCache.hostnames || - Object.keys(this.hostnameCache.hostnames).length > 50 || - this.hostnameCache.expires < Date.now() - ) { - this.driver.ping(this.hostnameCache) - - this.hostnameCache = { - expires: Date.now() + 1000 * 60 * 60 * 24, - hostnames: {} - } - } + while (resolved.length && !done) { + resolved.forEach(({ technology, confidence }) => { + done = true - if (this.adCache.length > 50) { - this.driver.ping(undefined, this.adCache) + technology.implies.forEach((name) => { + const implied = Wappalyzer.getTechnology(name) - this.adCache = [] - } - } - - /** - * Parse apps.json patterns - */ - parsePatterns(patterns) { - if (!patterns) { - return [] - } - - let parsed = {} - - // Convert string to object containing array containing string - if (typeof patterns === 'string' || Array.isArray(patterns)) { - patterns = { - main: asArray(patterns) - } - } - - Object.keys(patterns).forEach((key) => { - parsed[key] = [] - - asArray(patterns[key]).forEach((pattern) => { - const attrs = {} - - pattern.split('\\;').forEach((attr, i) => { - if (i) { - // Key value pairs - attr = attr.split(':') - - if (attr.length > 1) { - attrs[attr.shift()] = attr.join(':') - } - } else { - attrs.string = attr + if (!implied) { + throw new Error(`Implied technology does not exist: ${name}`) + } - try { - attrs.regex = new RegExp(attr.replace('/', '/'), 'i') // Escape slashes in regular expression - } catch (error) { - attrs.regex = new RegExp() + if ( + resolved.findIndex( + ({ technology: { name } }) => name === implied.name + ) === -1 + ) { + resolved.push({ technology: implied, confidence, version: '' }) - this.log(`${error.message}: ${attr}`, 'error', 'core') - } + done = false } }) - - parsed[key].push(attrs) }) - }) - - // Convert back to array if the original pattern list was an array (or string) - if ('main' in parsed) { - parsed = parsed.main } - - return parsed - } - - /** - * Parse JavaScript patterns - */ - parseJsPatterns() { - Object.keys(this.apps).forEach((appName) => { - if (this.apps[appName].js) { - this.jsPatterns[appName] = this.parsePatterns(this.apps[appName].js) - } - }) - } - - resolveImplies(apps, url) { - let checkImplies = true - - const resolve = (appName) => { - const app = apps[appName] - - if (app && app.props.implies) { - asArray(app.props.implies).forEach((implied) => { - ;[implied] = this.parsePatterns(implied) - - if (!this.apps[implied.string]) { - this.log( - `Implied application ${implied.string} does not exist`, - 'core', - 'warn' + }, + + async analyze(url, { html, meta, headers, cookies, scripts }) { + const oo = Wappalyzer.analyzeOneToOne + const om = Wappalyzer.analyzeOneToMany + const mm = Wappalyzer.analyzeManyToMany + + const flatten = (array) => Array.prototype.concat.apply([], array) + + try { + const detections = flatten( + flatten( + await Promise.all( + Wappalyzer.technologies.map((technology) => + Promise.all([ + oo(technology, 'url', url), + oo(technology, 'html', html), + om(technology, 'meta', meta), + mm(technology, 'headers', headers), + om(technology, 'cookies', cookies), + om(technology, 'scripts', scripts) + ]) ) + ) + ) + ).filter((technology) => technology) - return - } - - if (!(implied.string in apps)) { - apps[implied.string] = - this.detected[url] && this.detected[url][implied.string] - ? this.detected[url][implied.string] - : new Application( - implied.string, - this.apps[implied.string], - true - ) - - checkImplies = true - } - - // Apply app confidence to implied app - Object.keys(app.confidence).forEach((id) => { - apps[implied.string].confidence[`${id} implied by ${appName}`] = - app.confidence[id] * - (implied.confidence === undefined ? 1 : implied.confidence / 100) - }) - }) - } - } - - // Implied applications - // Run several passes as implied apps may imply other apps - while (checkImplies) { - checkImplies = false - - Object.keys(apps).forEach(resolve) + return detections + } catch (error) { + throw new Error(error.message || error.toString()) } - } + }, + + setTechnologies(data) { + const transform = Wappalyzer.transformPatterns + + Wappalyzer.technologies = Object.keys(data).reduce((technologies, name) => { + const { + cats, + url, + html, + meta, + headers, + cookies, + script, + js, + implies, + excludes, + icon, + website + } = data[name] + + technologies.push({ + name, + categories: cats || [], + slug: Wappalyzer.slugify(name), + url: transform(url), + headers: transform( + Object.keys(headers || {}).reduce( + (lcHeaders, header) => ({ + ...lcHeaders, + [header.toLowerCase()]: headers[header] + }), + {} + ) + ), + cookies: transform(cookies), + html: transform(html), + meta: transform(meta), + scripts: transform(script), + js: transform(js), + implies: typeof implies === 'string' ? [implies] : implies || [], + excludes: typeof excludes === 'string' ? [excludes] : excludes || [], + icon: icon || 'default.svg', + website: website || '' + }) - /** - * Cache detected applications - */ - cacheDetectedApps(apps, url) { - Object.keys(apps).forEach((appName) => { - const app = apps[appName] + return technologies + }, []) + }, - // Per URL - this.detected[url][appName] = app + setCategories(data) { + Wappalyzer.categories = Object.keys(data) + .reduce((categories, id) => { + const category = data[id] - Object.keys(app.confidence).forEach((id) => { - this.detected[url][appName].confidence[id] = app.confidence[id] - }) - }) + categories.push({ + id: parseInt(id, 10), + slug: Wappalyzer.slugify(category.name), + ...category + }) - if (this.driver.ping instanceof Function) { - this.ping() - } - } + return categories + }, []) + .sort(({ priority: a }, { priority: b }) => (a > b ? -1 : 0)) + }, - /** - * Track detected applications - */ - trackDetectedApps(apps, url, language) { - if (!(this.driver.ping instanceof Function)) { - return + transformPatterns(patterns) { + if (!patterns) { + return [] } - const hostname = `${url.protocol}//${url.hostname}` + const toArray = (value) => (Array.isArray(value) ? value : [value]) - Object.keys(apps).forEach((appName) => { - const app = apps[appName] - - if (this.detected[url.canonical][appName].getConfidence() >= 100) { - if ( - validation.hostname.test(url.hostname) && - !validation.hostnameBlacklist.test(url.hostname) - ) { - if (!(hostname in this.hostnameCache.hostnames)) { - this.hostnameCache.hostnames[hostname] = { - applications: {}, - meta: {} - } - } + if (typeof patterns === 'string' || Array.isArray(patterns)) { + patterns = { main: patterns } + } - if ( - !(appName in this.hostnameCache.hostnames[hostname].applications) - ) { - this.hostnameCache.hostnames[hostname].applications[appName] = { - hits: 0 + const parsed = Object.keys(patterns).reduce((parsed, key) => { + parsed[key] = toArray(patterns[key]).map((pattern) => { + const { regex, confidence, version } = pattern + .split('\\;') + .reduce((attrs, attr, i) => { + if (i) { + // Key value pairs + attr = attr.split(':') + + if (attr.length > 1) { + attrs[attr.shift()] = attr.join(':') + } + } else { + // Escape slashes in regular expression + attrs.regex = new RegExp(attr.replace(/\//g, '\\/'), 'i') } - } - this.hostnameCache.hostnames[hostname].applications[appName].hits += 1 + return attrs + }, {}) - if (apps[appName].version) { - this.hostnameCache.hostnames[hostname].applications[ - appName - ].version = app.version - } + return { + regex, + confidence: parseInt(confidence || 100, 10), + version: version || '' } - } - }) - - if (hostname in this.hostnameCache.hostnames) { - this.hostnameCache.hostnames[hostname].meta.language = language - } - - this.ping() - } - - /** - * Analyze URL - */ - analyzeUrl(app, url) { - const patterns = this.parsePatterns(app.props.url) - - if (!patterns.length) { - return Promise.resolve() - } - - return asyncForEach(patterns, (pattern) => { - if (pattern.regex.test(url.canonical)) { - addDetected(app, pattern, 'url', url.canonical) - } - }) - } + }) - /** - * Analyze HTML - */ - analyzeHtml(app, html) { - const patterns = this.parsePatterns(app.props.html) + return parsed + }, {}) - if (!patterns.length) { - return Promise.resolve() - } + return 'main' in parsed ? parsed.main : parsed + }, - return asyncForEach(patterns, (pattern) => { - if (pattern.regex.test(html)) { - addDetected(app, pattern, 'html', html) + analyzeOneToOne(technology, type, value) { + return technology[type].reduce((technologies, pattern) => { + if (pattern.regex.test(value)) { + technologies.push({ technology, pattern, match: value }) } - }) - } - /** - * Analyze script tag - */ - analyzeScripts(app, scripts) { - const patterns = this.parsePatterns(app.props.script) + return technologies + }, []) + }, - if (!patterns.length) { - return Promise.resolve() - } + analyzeOneToMany(technology, type, items = []) { + return items.reduce((technologies, { key, value }) => { + const patterns = technology[type][key] || [] - return asyncForEach(patterns, (pattern) => { - scripts.forEach((uri) => { - if (pattern.regex.test(uri)) { - addDetected(app, pattern, 'script', uri) + patterns.forEach((pattern) => { + if (pattern.regex.test(value)) { + technologies.push({ technology, pattern, match: value }) } }) - }) - } - /** - * Analyze meta tag - */ - analyzeMeta(app, metaTags) { - const patterns = this.parsePatterns(app.props.meta) - const promises = [] + return technologies + }, []) + }, - if (!app.props.meta) { - return Promise.resolve() - } + analyzeManyToMany(technology, type, items = {}) { + return Object.keys(technology[type]).reduce((technologies, key) => { + const patterns = technology[type][key] || [] + const values = items[key] || [] - metaTags.forEach((match) => { - Object.keys(patterns).forEach((meta) => { - const r = new RegExp(`(?:name|property)=["']${meta}["']`, 'i') - - if (r.test(match)) { - const content = match.match(/content=("|')([^"']+)("|')/i) - - promises.push( - asyncForEach(patterns[meta], (pattern) => { - if ( - content && - content.length === 4 && - pattern.regex.test(content[2]) - ) { - addDetected(app, pattern, 'meta', content[2], meta) - } - }) - ) - } + patterns.forEach((pattern) => { + values.forEach((value) => { + if (pattern.regex.test(value)) { + technologies.push({ technology, pattern, match: value }) + } + }) }) - }) - - return Promise.all(promises) - } - - /** - * Analyze response headers - */ - analyzeHeaders(app, headers) { - const patterns = this.parsePatterns(app.props.headers) - const promises = [] - - Object.keys(patterns).forEach((headerName) => { - if (typeof patterns[headerName] !== 'function') { - promises.push( - asyncForEach(patterns[headerName], (pattern) => { - headerName = headerName.toLowerCase() - - if (headerName in headers) { - headers[headerName].forEach((headerValue) => { - if (pattern.regex.test(headerValue)) { - addDetected(app, pattern, 'headers', headerValue, headerName) - } - }) - } - }) - ) - } - }) - - return promises ? Promise.all(promises) : Promise.resolve() - } - - /** - * Analyze cookies - */ - analyzeCookies(app, cookies) { - const patterns = this.parsePatterns(app.props.cookies) - const promises = [] - - Object.keys(patterns).forEach((cookieName) => { - if (typeof patterns[cookieName] !== 'function') { - const cookieNameLower = cookieName.toLowerCase() - - promises.push( - asyncForEach(patterns[cookieName], (pattern) => { - const cookie = cookies.find( - (_cookie) => _cookie.name.toLowerCase() === cookieNameLower - ) - - if (cookie && pattern.regex.test(cookie.value)) { - addDetected(app, pattern, 'cookies', cookie.value, cookieName) - } - }) - ) - } - }) - - return promises ? Promise.all(promises) : Promise.resolve() - } - - /** - * Analyze JavaScript variables - */ - analyzeJs(app, results) { - const promises = [] - - Object.keys(results).forEach((string) => { - if (typeof results[string] !== 'function') { - promises.push( - asyncForEach(Object.keys(results[string]), (index) => { - const pattern = this.jsPatterns[app.name][string][index] - const value = results[string][index] - - if (pattern && pattern.regex.test(value)) { - addDetected(app, pattern, 'js', value, string) - } - }) - ) - } - }) - return promises ? Promise.all(promises) : Promise.resolve() + return technologies + }, []) } } -if (typeof module === 'object') { +if (typeof module !== 'undefined') { module.exports = Wappalyzer }