-
-
+
+
+
+
+
+
+
+
+
+
+
-
+
-
-
+
+
+
+
+
diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js
index 66d620f4a..a1deeb4a6 100644
--- a/src/drivers/webextension/js/content.js
+++ b/src/drivers/webextension/js/content.js
@@ -1,20 +1,22 @@
-/** global: browser */
-/** global: XMLSerializer */
-
-/* global browser */
+'use strict'
/* eslint-env browser */
+/* globals chrome */
-const port = browser.runtime.connect({
- name: 'content.js'
-})
-
-;(async function() {
- if (typeof browser !== 'undefined' && typeof document.body !== 'undefined') {
+const Content = {
+ async init() {
await new Promise((resolve) => setTimeout(resolve, 1000))
- try {
- port.postMessage({ id: 'init' })
+ Content.port = chrome.runtime.connect({ name: 'content.js' })
+ Content.port.onMessage.addListener(({ func, args }) => {
+ const onFunc = `on${func.charAt(0).toUpperCase() + func.slice(1)}`
+
+ if (Content[onFunc]) {
+ Content[onFunc](args)
+ }
+ })
+
+ try {
// HTML
let html = new XMLSerializer().serializeToString(document)
@@ -23,9 +25,7 @@ const port = browser.runtime.connect({
const maxRows = 3000
const rows = html.length / maxCols
- let i
-
- for (i = 0; i < rows; i += 1) {
+ for (let i = 0; i < rows; i += 1) {
if (i < maxRows / 2 || i > rows - maxRows / 2) {
chunks.push(html.slice(i * maxCols, (i + 1) * maxCols))
}
@@ -33,62 +33,83 @@ const port = browser.runtime.connect({
html = chunks.join('\n')
- // Scripts
- const scripts = Array.prototype.slice
- .apply(document.scripts)
- .filter((script) => script.src)
- .map((script) => script.src)
+ const language =
+ document.documentElement.getAttribute('lang') ||
+ document.documentElement.getAttribute('xml:lang') ||
+ (await new Promise((resolve) =>
+ chrome.i18n.detectLanguage(html, ({ languages }) =>
+ resolve(
+ languages
+ .filter(({ percentage }) => percentage >= 75)
+ .map(({ language: lang }) => lang)[0]
+ )
+ )
+ ))
+
+ // Script tags
+ const scripts = Array.from(document.scripts)
+ .filter(({ src }) => src)
+ .map(({ src }) => src)
.filter((script) => script.indexOf('data:text/javascript;') !== 0)
- port.postMessage({ id: 'analyze', subject: { html, scripts } })
-
- // JavaScript variables
- const script = document.createElement('script')
+ // Meta tags
+ const meta = Array.from(document.querySelectorAll('meta'))
+ .map((meta) => ({
+ key: meta.getAttribute('name') || meta.getAttribute('property'),
+ value: meta.getAttribute('content')
+ }))
+ .filter(({ value }) => value)
- script.onload = () => {
- const onMessage = (event) => {
- if (event.data.id !== 'js') {
- return
- }
+ Content.port.postMessage({
+ func: 'onContentLoad',
+ args: [location.href, { html, scripts, meta }, language]
+ })
- window.removeEventListener('message', onMessage)
+ Content.port.postMessage({ func: 'getTechnologies' })
+ } catch (error) {
+ Content.port.postMessage({ func: 'error', args: [error, 'content.js'] })
+ }
+ },
- port.postMessage({ id: 'analyze', subject: { js: event.data.js } })
+ onGetTechnologies(technologies) {
+ const script = document.createElement('script')
- script.remove()
+ script.onload = () => {
+ const onMessage = ({ data }) => {
+ if (!data.wappalyzer || !data.wappalyzer.js) {
+ return
}
- window.addEventListener('message', onMessage)
+ window.removeEventListener('message', onMessage)
- port.postMessage({ id: 'get_js_patterns' })
+ Content.port.postMessage({
+ func: 'analyzeJs',
+ args: [location.href, data.wappalyzer.js]
+ })
+
+ script.remove()
}
- script.setAttribute('src', browser.extension.getURL('js/inject.js'))
+ window.addEventListener('message', onMessage)
- document.body.appendChild(script)
- } catch (error) {
- port.postMessage({ id: 'log', subject: error })
+ window.postMessage({
+ wappalyzer: {
+ technologies: technologies
+ .filter(({ js }) => Object.keys(js).length)
+ .filter(({ name }) => name === 'jQuery')
+ .map(({ name, js }) => ({ name, chains: Object.keys(js) }))
+ }
+ })
}
+
+ script.setAttribute('src', chrome.extension.getURL('js/inject.js'))
+
+ document.body.appendChild(script)
}
-})()
-
-port.onMessage.addListener((message) => {
- switch (message.id) {
- case 'get_js_patterns':
- postMessage(
- {
- id: 'patterns',
- patterns: message.response.patterns
- },
- window.location.href
- )
-
- break
- default:
- // Do nothing
- }
-})
+}
-// https://stackoverflow.com/a/44774834
-// https://developer.mozilla.org/en-US/Add-ons/WebExtensions/API/tabs/executeScript#Return_value
-undefined // eslint-disable-line no-unused-expressions
+if (/complete|interactive|loaded/.test(document.readyState)) {
+ Content.init()
+} else {
+ document.addEventListener('DOMContentLoaded', Content.init)
+}
diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js
index 9742f8906..d83d12b94 100644
--- a/src/drivers/webextension/js/driver.js
+++ b/src/drivers/webextension/js/driver.js
@@ -1,445 +1,483 @@
-/**
- * WebExtension driver
- */
-
+'use strict'
/* eslint-env browser */
-/* global browser, chrome, Wappalyzer */
-
-/** global: browser */
-/** global: chrome */
-/** global: fetch */
-/** global: Wappalyzer */
-
-const wappalyzer = new Wappalyzer()
-
-const tabCache = {}
-const robotsTxtQueue = {}
-
-let categoryOrder = []
-
-browser.tabs.onRemoved.addListener((tabId) => {
- tabCache[tabId] = null
-})
+/* globals chrome, Wappalyzer, Utils */
+
+const {
+ setTechnologies,
+ setCategories,
+ analyze,
+ analyzeManyToMany,
+ resolve
+} = Wappalyzer
+const { agent, promisify, getOption, setOption } = Utils
+
+const expiry = 1000 * 60 * 60 * 24
+
+const Driver = {
+ lastPing: Date.now(),
+
+ async init() {
+ chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect)
+
+ await Driver.loadTechnologies()
+
+ const hostnameCache = (await getOption('hostnames')) || {}
+
+ Driver.cache = {
+ hostnames: Object.keys(hostnameCache).reduce(
+ (cache, hostname) => ({
+ ...cache,
+ [hostname]: {
+ ...hostnameCache[hostname],
+ detections: hostnameCache[hostname].detections.map(
+ ({
+ pattern: { regex, confidence, version },
+ match,
+ technology: name
+ }) => ({
+ pattern: {
+ regex: new RegExp(regex, 'i'),
+ confidence,
+ version
+ },
+ match,
+ technology: Wappalyzer.technologies.find(
+ ({ name: _name }) => name === _name
+ )
+ })
+ )
+ }
+ }),
+ {}
+ ),
+ tabs: {},
+ robots: (await getOption('robots')) || {},
+ ads: (await getOption('ads')) || []
+ }
-function userAgent() {
- const url = chrome.extension.getURL('/')
+ chrome.webRequest.onCompleted.addListener(
+ Driver.onWebRequestComplete,
+ { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] },
+ ['responseHeaders']
+ )
+ chrome.tabs.onRemoved.addListener((id) => (Driver.cache.tabs[id] = null))
+ },
- if (url.startsWith('moz-')) {
- return 'firefox'
- }
+ log(message, source = 'driver', type = 'log') {
+ // eslint-disable-next-line no-console
+ console[type](`wappalyzer | ${source} |`, message)
+ },
- if (url.startsWith('ms-browser')) {
- return 'edge'
- }
+ warn(message, source = 'driver') {
+ Driver.log(message, source, 'warn')
+ },
- return 'chrome'
-}
+ error(error, source = 'driver') {
+ Driver.log(error, source, 'error')
+ },
-/**
- * Get a value from localStorage
- */
-function getOption(name, defaultValue = null) {
- return new Promise(async (resolve, reject) => {
- let value = defaultValue
+ open(url, active = true) {
+ chrome.tabs.create({ url, active })
+ },
+ async loadTechnologies() {
try {
- const option = await browser.storage.local.get(name)
+ const { apps: technologies, categories } = await (
+ await fetch(chrome.extension.getURL('apps.json'))
+ ).json()
- if (option[name] !== undefined) {
- value = option[name]
- }
+ setTechnologies(technologies)
+ setCategories(categories)
} catch (error) {
- wappalyzer.log(error.message, 'driver', 'error')
-
- return reject(error.message)
+ Driver.error(error)
}
+ },
- return resolve(value)
- })
-}
-
-/**
- * Set a value in localStorage
- */
-function setOption(name, value) {
- return new Promise(async (resolve, reject) => {
+ post(url, body) {
try {
- await browser.storage.local.set({ [name]: value })
+ return fetch(url, {
+ method: 'POST',
+ body: JSON.stringify(body)
+ })
} catch (error) {
- wappalyzer.log(error.message, 'driver', 'error')
-
- return reject(error.message)
+ throw new Error(error.message || error.toString())
}
+ },
- return resolve()
- })
-}
+ async analyzeJs(href, js) {
+ const url = new URL(href)
+
+ await Driver.onDetect(
+ url,
+ Array.prototype.concat.apply(
+ [],
+ await Promise.all(
+ js.map(({ name, chain, value }) =>
+ analyzeManyToMany(
+ Wappalyzer.technologies.find(({ name: _name }) => name === _name),
+ 'js',
+ { [chain]: [value] }
+ )
+ )
+ )
+ )
+ )
+ },
-/**
- * Open a tab
- */
-function openTab(args) {
- browser.tabs.create({
- url: args.url,
- active: args.background === undefined || !args.background
- })
-}
+ onRuntimeConnect(port) {
+ Driver.log(`Connected to ${port.name}`)
-/**
- * Make a POST request
- */
-async function post(url, body) {
- try {
- const response = await fetch(url, {
- method: 'POST',
- body: JSON.stringify(body)
- })
+ port.onMessage.addListener(async ({ func, args }) => {
+ if (!func) {
+ return
+ }
- wappalyzer.log(`POST ${url}: ${response.status}`, 'driver')
- } catch (error) {
- wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error')
- }
-}
+ Driver.log({ port: port.name, func, args })
-// Capture response headers
-browser.webRequest.onCompleted.addListener(
- async (request) => {
- const headers = {}
+ if (!Driver[func]) {
+ Driver.error(new Error(`Method does not exist: Driver.${func}`))
- if (request.responseHeaders) {
- const url = wappalyzer.parseUrl(request.url)
+ return
+ }
+
+ port.postMessage({
+ func,
+ args: await Driver[func].call(port.sender, ...(args || []))
+ })
+ })
+ },
- let tab
+ async onWebRequestComplete(request) {
+ if (request.responseHeaders) {
+ const headers = {}
try {
- ;[tab] = await browser.tabs.query({ url: [url.href] })
- } catch (error) {
- wappalyzer.log(error, 'driver', 'error')
- }
+ const url = new URL(request.url)
- if (tab) {
- request.responseHeaders.forEach((header) => {
- const name = header.name.toLowerCase()
+ const [tab] = await promisify(chrome.tabs, 'query', { url: [url.href] })
- headers[name] = headers[name] || []
+ if (tab) {
+ request.responseHeaders.forEach((header) => {
+ const name = header.name.toLowerCase()
- headers[name].push(
- (header.value || header.binaryValue || '').toString()
- )
- })
+ headers[name] = headers[name] || []
+
+ headers[name].push(
+ (header.value || header.binaryValue || '').toString()
+ )
+ })
- if (
- headers['content-type'] &&
- /\/x?html/.test(headers['content-type'][0])
- ) {
- wappalyzer.analyze(url, { headers }, { tab })
+ if (
+ headers['content-type'] &&
+ /\/x?html/.test(headers['content-type'][0])
+ ) {
+ await Driver.onDetect(
+ url,
+ await analyze(url.href, { headers }, { tab })
+ )
+ }
}
+ } catch (error) {
+ Driver.error(error)
}
}
},
- { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] },
- ['responseHeaders']
-)
-browser.runtime.onConnect.addListener((port) => {
- port.onMessage.addListener(async (message) => {
- if (message.id === undefined) {
- return
- }
+ async onContentLoad(href, items, language) {
+ try {
+ const url = new URL(href)
- if (message.id !== 'log') {
- wappalyzer.log(`Message from ${port.name}: ${message.id}`, 'driver')
- }
+ items.cookies = await promisify(chrome.cookies, 'getAll', {
+ domain: `.${url.hostname}`
+ })
- const pinnedCategory = await getOption('pinnedCategory')
+ await Driver.onDetect(url, await analyze(href, items), language, true)
+ } catch (error) {
+ Driver.error(error)
+ }
+ },
- const url = wappalyzer.parseUrl(port.sender.tab ? port.sender.tab.url : '')
+ getTechnologies() {
+ return Wappalyzer.technologies
+ },
- const cookies = await browser.cookies.getAll({
- domain: `.${url.hostname}`
- })
+ async onDetect(url, detections = [], language, incrementHits = false) {
+ if (!detections.length) {
+ return
+ }
- let response
+ const { hostname, href } = url
- switch (message.id) {
- case 'log':
- wappalyzer.log(message.subject, message.source)
+ // Cache detections
+ const cache = (Driver.cache.hostnames[hostname] = {
+ ...(Driver.cache.hostnames[hostname] || {
+ detections: [],
+ hits: 0
+ }),
+ dateTime: Date.now()
+ })
- break
- case 'init':
- wappalyzer.analyze(url, { cookies }, { tab: port.sender.tab })
+ // Remove duplicates
+ cache.detections = cache.detections = cache.detections.concat(detections)
- break
- case 'analyze':
- if (message.subject.html) {
- browser.i18n
- .detectLanguage(message.subject.html)
- .then(({ languages }) => {
- const language = languages
- .filter(({ percentage }) => percentage >= 75)
- .map(({ language: lang }) => lang)[0]
+ cache.detections.filter(
+ ({ technology: { name }, pattern: { regex } }, index) =>
+ cache.detections.findIndex(
+ ({ technology: { name: _name }, pattern: { regex: _regex } }) =>
+ name === _name && (!regex || regex.toString() === _regex.toString())
+ ) === index
+ )
- message.subject.language = language
+ cache.hits += incrementHits ? 1 : 0
+ cache.language = cache.language || language
- wappalyzer.analyze(url, message.subject, { tab: port.sender.tab })
- })
- } else {
- wappalyzer.analyze(url, message.subject, { tab: port.sender.tab })
- }
+ // Expire cache
+ Driver.cache.hostnames = Object.keys(Driver.cache.hostnames).reduce(
+ (hostnames, hostname) => {
+ const cache = Driver.cache.hostnames[hostname]
- await setOption('hostnameCache', wappalyzer.hostnameCache)
-
- break
- case 'ad_log':
- wappalyzer.cacheDetectedAds(message.subject)
-
- break
- case 'get_apps':
- response = {
- tabCache: tabCache[message.tab.id],
- apps: wappalyzer.apps,
- categories: wappalyzer.categories,
- pinnedCategory,
- termsAccepted:
- userAgent() === 'chrome' ||
- (await getOption('termsAccepted', false))
+ if (cache.dateTime > Date.now() - expiry) {
+ hostnames[hostname] = cache
}
- break
- case 'set_option':
- await setOption(message.key, message.value)
-
- break
- case 'get_js_patterns':
- response = {
- patterns: wappalyzer.jsPatterns
- }
+ return hostnames
+ },
+ {}
+ )
- break
- case 'update_theme_mode':
- // Sync theme mode to popup.
- response = {
- themeMode: await getOption('themeMode', false)
- }
+ await setOption(
+ 'hostnames',
+ Object.keys(Driver.cache.hostnames).reduce(
+ (cache, hostname) => ({
+ ...cache,
+ [hostname]: {
+ ...Driver.cache.hostnames[hostname],
+ detections: Driver.cache.hostnames[hostname].detections.map(
+ ({
+ pattern: { regex, confidence, version },
+ match,
+ technology: { name: technology }
+ }) => ({
+ technology,
+ pattern: {
+ regex: regex.source,
+ confidence,
+ version
+ },
+ match
+ })
+ )
+ }
+ }),
+ {}
+ )
+ )
- break
- default:
- // Do nothing
- }
+ const resolved = resolve(Driver.cache.hostnames[hostname].detections)
- if (response) {
- port.postMessage({
- id: message.id,
- response
- })
- }
- })
-})
+ await Driver.setIcon(url, resolved)
-wappalyzer.driver.document = document
+ const tabs = await promisify(chrome.tabs, 'query', { url: [href] })
-/**
- * Log messages to console
- */
-wappalyzer.driver.log = (message, source, type) => {
- const log = ['warn', 'error'].includes(type) ? type : 'log'
+ tabs.forEach(({ id }) => (Driver.cache.tabs[id] = resolved))
- console[log](`[wappalyzer ${type}]`, `[${source}]`, message) // eslint-disable-line no-console
-}
+ Driver.log({ hostname, technologies: resolved })
-/**
- * Display apps
- */
-wappalyzer.driver.displayApps = async (detected, meta, context) => {
- const { tab } = context
+ await Driver.ping()
+ },
- if (tab === undefined) {
- return
- }
+ async onAd(ad) {
+ Driver.cache.ads.push(ad)
- tabCache[tab.id] = tabCache[tab.id] || {
- detected: []
- }
+ await setOption('ads', Driver.cache.ads)
+ },
- tabCache[tab.id].detected = detected
+ async setIcon(url, technologies) {
+ const dynamicIcon = await getOption('dynamicIcon', true)
- const pinnedCategory = await getOption('pinnedCategory')
- const dynamicIcon = await getOption('dynamicIcon', true)
+ let icon = 'default.svg'
- let found = false
+ if (dynamicIcon) {
+ const pinnedCategory = parseInt(await getOption('pinnedCategory'), 10)
- // Find the main application to display
- ;[pinnedCategory].concat(categoryOrder).forEach((match) => {
- Object.keys(detected).forEach((appName) => {
- const app = detected[appName]
+ const pinned = technologies.find(({ categories }) =>
+ categories.some(({ id }) => id === pinnedCategory)
+ )
- app.props.cats.forEach((category) => {
- if (category === match && !found) {
- let icon =
- app.props.icon && dynamicIcon ? app.props.icon : 'default.svg'
+ ;({ icon } = pinned ||
+ technologies.sort(({ categories: a }, { categories: b }) => {
+ const max = (value) =>
+ value.reduce((max, { priority }) => Math.max(max, priority))
- if (/\.svg$/i.test(icon)) {
- icon = `converted/${icon.replace(/\.svg$/, '.png')}`
- }
+ return max(a) > max(b) ? -1 : 1
+ })[0] || { icon })
+ }
- try {
- browser.pageAction.setIcon({
- tabId: tab.id,
- path: `../images/icons/${icon}`
- })
- } catch (e) {
- // Firefox for Android does not support setIcon see https://bugzilla.mozilla.org/show_bug.cgi?id=1331746
- }
+ const tabs = await promisify(chrome.tabs, 'query', { url: [url.href] })
+
+ await Promise.all(
+ tabs.map(async ({ id: tabId }) => {
+ await promisify(chrome.pageAction, 'setIcon', {
+ tabId,
+ path: chrome.extension.getURL(
+ `../images/icons/${
+ /\.svg$/i.test(icon)
+ ? `converted/${icon.replace(/\.svg$/, '.png')}`
+ : icon
+ }`
+ )
+ })
- found = true
- }
+ chrome.pageAction.show(tabId)
})
- })
- })
-
- browser.pageAction.show(tab.id)
-}
+ )
+ },
-/**
- * Fetch and cache robots.txt for host
- */
-wappalyzer.driver.getRobotsTxt = async (host, secure = false) => {
- if (robotsTxtQueue[host]) {
- return robotsTxtQueue[host]
- }
+ async getDetections() {
+ const [{ id }] = await promisify(chrome.tabs, 'query', {
+ active: true,
+ currentWindow: true
+ })
- const tracking = await getOption('tracking', true)
- const robotsTxtCache = await getOption('robotsTxtCache', {})
+ return Driver.cache.tabs[id]
+ },
- robotsTxtQueue[host] = new Promise(async (resolve) => {
- if (!tracking) {
- return resolve([])
+ async getRobots(hostname, secure = false) {
+ if (!(await getOption('tracking', true))) {
+ return
}
- if (host in robotsTxtCache) {
- return resolve(robotsTxtCache[host])
+ if (typeof Driver.cache.robots[hostname] !== 'undefined') {
+ return Driver.cache.robots[hostname]
}
- const timeout = setTimeout(() => resolve([]), 3000)
-
- let response
-
try {
- response = await fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, {
- redirect: 'follow',
- mode: 'no-cors'
- })
- } catch (error) {
- wappalyzer.log(error, 'driver', 'error')
-
- return resolve([])
- }
-
- clearTimeout(timeout)
-
- const robotsTxt = response.ok ? await response.text() : ''
+ Driver.cache.robots[hostname] = await Promise.race([
+ new Promise(async (resolve) => {
+ const response = await fetch(
+ `http${secure ? 's' : ''}://${hostname}/robots.txt`,
+ {
+ redirect: 'follow',
+ mode: 'no-cors'
+ }
+ )
- robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt)
+ if (!response.ok) {
+ Driver.error(new Error(response.statusText))
- await setOption('robotsTxtCache', robotsTxtCache)
+ resolve('')
+ }
- delete robotsTxtQueue[host]
+ let agent
- return resolve(robotsTxtCache[host])
- })
+ resolve(
+ (await response.text()).split('\n').reduce((disallows, line) => {
+ let matches = /^User-agent:\s*(.+)$/i.exec(line.trim())
- return robotsTxtQueue[host]
-}
+ if (matches) {
+ agent = matches[1].toLowerCase()
+ } else if (agent === '*' || agent === 'wappalyzer') {
+ matches = /^Disallow:\s*(.+)$/i.exec(line.trim())
-/**
- * Anonymously track detected applications for research purposes
- */
-wappalyzer.driver.ping = async (
- hostnameCache = { expires: 0, hostnames: {} },
- adCache = []
-) => {
- const tracking = await getOption('tracking', true)
- const termsAccepted =
- userAgent() === 'chrome' || (await getOption('termsAccepted', false))
-
- if (tracking && termsAccepted) {
- if (
- hostnameCache.hostnames &&
- Object.keys(hostnameCache.hostnames).length
- ) {
- post('https://api.wappalyzer.com/ping/v1/', hostnameCache.hostnames)
- }
+ if (matches) {
+ disallows.push(matches[1])
+ }
+ }
- if (adCache.length) {
- post('https://ad.wappalyzer.com/log/wp/', adCache)
+ return disallows
+ }, [])
+ )
+ }),
+ new Promise((resolve) => setTimeout(() => resolve(''), 5000))
+ ])
+
+ Driver.cache.robots = Object.keys(Driver.cache.robots)
+ .slice(-50)
+ .reduce(
+ (cache, hostname) => ({
+ ...cache,
+ [hostname]: Driver.cache.robots[hostname]
+ }),
+ {}
+ )
+
+ await setOption('robots', Driver.cache.robots)
+
+ return Driver.cache.robots[hostname]
+ } catch (error) {
+ Driver.error(error)
}
+ },
- await setOption('robotsTxtCache', {})
- }
-}
-
-// Init
-;(async () => {
- // Technologies
- try {
- const response = await fetch('../apps.json')
- const json = await response.json()
-
- wappalyzer.apps = json.apps
- wappalyzer.categories = json.categories
- } catch (error) {
- wappalyzer.log(`GET apps.json: ${error.message}`, 'driver', 'error')
- }
+ async checkRobots(href) {
+ const url = new URL(href)
- wappalyzer.parseJsPatterns()
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+ throw new Error('Invalid protocol')
+ }
- categoryOrder = Object.keys(wappalyzer.categories)
- .map((categoryId) => parseInt(categoryId, 10))
- .sort(
- (a, b) =>
- wappalyzer.categories[a].priority - wappalyzer.categories[b].priority
+ const robots = await Driver.getRobots(
+ url.hostname,
+ url.protocol === 'https:'
)
- // Version check
- const { version } = browser.runtime.getManifest()
- const previousVersion = await getOption('version')
- const upgradeMessage = await getOption('upgradeMessage', true)
-
- if (previousVersion === null) {
- openTab({
- url: `${wappalyzer.config.websiteURL}installed`
- })
- } else if (version !== previousVersion && upgradeMessage) {
- // openTab({
- // url: `${wappalyzer.config.websiteURL}upgraded?v${version}`,
- // background: true
- // })
- }
-
- await setOption('version', version)
+ if (robots.some((disallowed) => url.pathname.indexOf(disallowed) === 0)) {
+ throw new Error('Disallowed')
+ }
+ },
- // Hostname cache
- wappalyzer.hostnameCache = await getOption('hostnameCache', {
- expires: Date.now() + 1000 * 60 * 60 * 24,
- hostnames: {}
- })
+ async ping() {
+ const tracking = await getOption('tracking', true)
+ const termsAccepted =
+ agent === 'chrome' || (await getOption('termsAccepted', false))
+
+ if (tracking && termsAccepted) {
+ const count = Object.keys(Driver.cache.hostnames).length
+
+ if (count && (count >= 50 || Driver.lastPing < Date.now() - expiry)) {
+ await Driver.post(
+ 'https://api.wappalyzer.com/ping/v1/',
+ Object.keys(Driver.cache.hostnames).reduce((hostnames, hostname) => {
+ // eslint-disable-next-line standard/computed-property-even-spacing
+ const { language, detections, hits } = Driver.cache.hostnames[
+ hostname
+ ]
+
+ hostnames[hostname] = hostnames[hostname] || {
+ applications: resolve(detections).reduce(
+ (technologies, { name, confidence, version }) => {
+ if (confidence === 100) {
+ technologies[name] = {
+ version,
+ hits
+ }
+
+ return technologies
+ }
+ },
+ {}
+ ),
+ meta: {
+ language
+ }
+ }
+
+ return hostnames
+ }, {})
+ )
+
+ await setOption('hostnames', (Driver.cache.hostnames = {}))
+
+ Driver.lastPing = Date.now()
+ }
- // Run content script on all tabs
- try {
- const tabs = await browser.tabs.query({
- url: ['http://*/*', 'https://*/*']
- })
+ if (Driver.cache.ads.length > 50) {
+ await Driver.post('https://ad.wappalyzer.com/log/wp/', Driver.cache.ads)
- tabs.forEach(async (tab) => {
- try {
- await browser.tabs.executeScript(tab.id, {
- file: '../js/content.js'
- })
- } catch (error) {
- //
+ await setOption('ads', (Driver.cache.ads = []))
}
- })
- } catch (error) {
- wappalyzer.log(error, 'driver', 'error')
+ }
}
-})()
+}
+
+Driver.init()
diff --git a/src/drivers/webextension/js/inject.js b/src/drivers/webextension/js/inject.js
index e23cb48b9..b0807124e 100644
--- a/src/drivers/webextension/js/inject.js
+++ b/src/drivers/webextension/js/inject.js
@@ -1,62 +1,44 @@
/* eslint-env browser */
-/* eslint-disable no-restricted-globals, no-prototype-builtins */
-;(() => {
+;(function() {
try {
- const detectJs = (chain) => {
- const properties = chain.split('.')
-
- let value = properties.length ? window : null
-
- for (let i = 0; i < properties.length; i += 1) {
- const property = properties[i]
-
- if (value && value.hasOwnProperty(property)) {
- value = value[property]
- } else {
- value = null
-
- break
- }
- }
-
- return typeof value === 'string' || typeof value === 'number'
- ? value
- : !!value
- }
-
- const onMessage = (event) => {
- if (event.data.id !== 'patterns') {
+ const onMessage = ({ data }) => {
+ if (!data.wappalyzer) {
return
}
- removeEventListener('message', onMessage)
-
- const patterns = event.data.patterns || {}
-
- const js = {}
-
- for (const appName in patterns) {
- if (patterns.hasOwnProperty(appName)) {
- js[appName] = {}
+ const { technologies } = data.wappalyzer || {}
- for (const chain in patterns[appName]) {
- if (patterns[appName].hasOwnProperty(chain)) {
- js[appName][chain] = {}
-
- for (const index in patterns[appName][chain]) {
- const value = detectJs(chain)
+ removeEventListener('message', onMessage)
- if (value && patterns[appName][chain].hasOwnProperty(index)) {
- js[appName][chain][index] = value
- }
- }
- }
- }
+ postMessage({
+ wappalyzer: {
+ js: technologies.reduce((results, { name, chains }) => {
+ chains.forEach((chain) => {
+ const value = chain
+ .split('.')
+ .reduce(
+ (value, method) =>
+ value && value.hasOwnProperty(method)
+ ? value[method]
+ : undefined,
+ window
+ )
+
+ technologies.push({
+ name,
+ chain,
+ value:
+ typeof value === 'string' || typeof value === 'number'
+ ? value
+ : !!value
+ })
+ })
+
+ return technologies
+ }, [])
}
- }
-
- postMessage({ id: 'js', js }, window.location.href)
+ })
}
addEventListener('message', onMessage)
diff --git a/src/drivers/webextension/js/lib/iframe.js b/src/drivers/webextension/js/lib/iframe.js
index e132433c8..9fd152494 100644
--- a/src/drivers/webextension/js/lib/iframe.js
+++ b/src/drivers/webextension/js/lib/iframe.js
@@ -120,16 +120,16 @@ var exports = {};
return dict;
},
sendToBackground: function(message, event, responseMessage) {
- if ( typeof browser !== 'undefined' || typeof chrome !== 'undefined' ) {
- var port = browser.runtime.connect({name:"adparser"});
+ if ( typeof chrome !== 'undefined' ) {
+ var port = chrome.runtime.connect({name:"adparser"});
port.onMessage.addListener((message) => {
- if ( message && message.tracking_enabled ) {
-
- utilCallback();
- } else {
-
- utilElseCallback();
+ if ( message && typeof message.tracking_enabled !== 'undefined' ) {
+ if (message.tracking_enabled) {
+ utilCallback();
+ } else {
+ utilElseCallback();
+ }
}
});
@@ -1088,8 +1088,8 @@ var exports = {};
}
function addBackgroundListener(event, callback) {
- if ( typeof browser !== 'undefined' || typeof chrome !== 'undefined' ) {
- browser.runtime.onMessage.addListener(function(msg) {
+ if ( typeof chrome !== 'undefined' ) {
+ chrome.runtime.onMessage.addListener(function(msg) {
if ( msg.event === event ) {
callback(msg);
}
@@ -1173,7 +1173,7 @@ if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
})(window);
(function(adparser, pageUrl) {
function onAdFound(log) {
- adparser.sendToBackground({ id: 'ad_log', subject: log }, 'ad_log', '', function(){});
+ adparser.sendToBackground({ func: 'onAd', args: [log] }, 'onAd', '', function(){});
}
if ( adparser && adparser.inWindowTop ) {
diff --git a/src/drivers/webextension/js/lib/jsontodom.js b/src/drivers/webextension/js/lib/jsontodom.js
deleted file mode 100644
index 24d9e4c29..000000000
--- a/src/drivers/webextension/js/lib/jsontodom.js
+++ /dev/null
@@ -1,63 +0,0 @@
-jsonToDOM.namespaces = {
- html: 'http://www.w3.org/1999/xhtml',
- xul: 'http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul',
-};
-
-jsonToDOM.defaultNamespace = jsonToDOM.namespaces.html;
-
-function jsonToDOM(jsonTemplate, doc, nodes) {
- function namespace(name) {
- const reElemNameParts = /^(?:(.*):)?(.*)$/.exec(name);
- return { namespace: jsonToDOM.namespaces[reElemNameParts[1]], shortName: reElemNameParts[2] };
- }
-
- // Note that 'elemNameOrArray' is: either the full element name (eg. [html:]div) or an array of elements in JSON notation
- function tag(elemNameOrArray, elemAttr) {
- // Array of elements? Parse each one...
- if (Array.isArray(elemNameOrArray)) {
- const frag = doc.createDocumentFragment();
- Array.prototype.forEach.call(arguments, (thisElem) => {
- frag.appendChild(tag(...thisElem));
- });
- return frag;
- }
-
- // Single element? Parse element namespace prefix (if none exists, default to defaultNamespace), and create element
- const elemNs = namespace(elemNameOrArray);
- const elem = doc.createElementNS(elemNs.namespace || jsonToDOM.defaultNamespace, elemNs.shortName);
-
- // Set element's attributes and/or callback functions (eg. onclick)
- for (const key in elemAttr) {
- const val = elemAttr[key];
- if (nodes && key == 'key') {
- nodes[val] = elem;
- continue;
- }
-
- const attrNs = namespace(key);
- if (typeof val === 'function') {
- // Special case for function attributes; don't just add them as 'on...' attributes, but as events, using addEventListener
- elem.addEventListener(key.replace(/^on/, ''), val, false);
- } else {
- // Note that the default namespace for XML attributes is, and should be, blank (ie. they're not in any namespace)
- elem.setAttributeNS(attrNs.namespace || '', attrNs.shortName, val);
- }
- }
-
- // Create and append this element's children
- const childElems = Array.prototype.slice.call(arguments, 2);
- childElems.forEach((childElem) => {
- if (childElem != null) {
- elem.appendChild(
- childElem instanceof doc.defaultView.Node ? childElem
- : Array.isArray(childElem) ? tag(...childElem)
- : doc.createTextNode(childElem),
- );
- }
- });
-
- return elem;
- }
-
- return tag(...jsonTemplate);
-}
diff --git a/src/drivers/webextension/js/lib/network.js b/src/drivers/webextension/js/lib/network.js
index 7c46fcf9f..f9f58d843 100644
--- a/src/drivers/webextension/js/lib/network.js
+++ b/src/drivers/webextension/js/lib/network.js
@@ -1,19 +1,6 @@
-(function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i
= MIN_FF_MAJOR_VERSION) {
- callback();
- } else {
- elseCallback();
- }
- });
- } catch (err) {
-
- elseCallback();
- }
- } else {
- elseCallback();
- }
+ chrome.webNavigation.getFrame(getFrameDetails, callback);
}
function ifTrackingEnabled(details, ifCallback, elseCallback) {
@@ -112,24 +72,19 @@
allowedByRobotsTxt(details, ifCallback, elseCallback);
};
- browser.storage.local.get('tracking').then(function(item) {
-
- if ( item.hasOwnProperty('tracking') ) {
- if ( item.tracking ) {
- fullIfCallback();
- } else {
- elseCallback();
- }
- } else {
- fullIfCallback();
- }
+ Utils.getOption('tracking', true).then(function(tracking) {
+ if ( tracking ) {
+ fullIfCallback();
+ } else {
+ elseCallback();
+ }
});
}
function allowedByRobotsTxt(details, ifCallback, elseCallback) {
if ( details.url && !details.url.startsWith('chrome://') ) {
- robotsTxtAllows(details.url).then(ifCallback, elseCallback);
+ Driver.checkRobots(details.url, details.url.startsWith('https:')).then(ifCallback).catch(elseCallback);
} else {
elseCallback();
}
@@ -268,7 +223,7 @@
PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() {
var logMessage = Array.from(arguments).join(' ');
var message = {message: logMessage, event: 'console-log-message'};
- browser.tabs.sendMessage(this.tabId, message);
+ chrome.tabs.sendMessage(this.tabId, message);
};
PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, adTrackingEvent) {
@@ -298,7 +253,7 @@
msg.origUrl = curPageUrl;
msg.displayAdFound = this.displayAdFound;
- browser.tabs.sendMessage(this.tabId, msg);
+ chrome.tabs.sendMessage(this.tabId, msg);
};
PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) {
@@ -615,7 +570,7 @@
var _this = this,
origPageUrl, msgAssetReq;
msgAssetReq = this.msgsBeingSent[msgKey];
- browser.tabs.get(this.tabId).then(function(tab) {
+ chrome.tabs.get(this.tabId, function(tab) {
origPageUrl = tab.url;
});
@@ -697,110 +652,85 @@
function registerListeners() {
- browser.webRequest.onBeforeRequest.addListener(
+ chrome.webRequest.onBeforeRequest.addListener(
onBeforeRequestListener,
{urls: ['http://*/*', 'https://*/*']},
[]
);
- browser.webRequest.onSendHeaders.addListener(
+ chrome.webRequest.onSendHeaders.addListener(
onSendHeadersListener,
{urls: ['http://*/*', 'https://*/*']},
['requestHeaders']
);
- browser.webRequest.onHeadersReceived.addListener(
+ chrome.webRequest.onHeadersReceived.addListener(
onHeadersReceivedListener,
{urls: ['http://*/*', 'https://*/*']},
['responseHeaders']
);
- browser.webRequest.onBeforeRedirect.addListener(
+ chrome.webRequest.onBeforeRedirect.addListener(
onBeforeRedirectListener,
{urls: ['http://*/*', 'https://*/*']},
[]
);
- browser.webRequest.onResponseStarted.addListener(
+ chrome.webRequest.onResponseStarted.addListener(
onResponseStartedListener,
{urls: ['http://*/*', 'https://*/*']},
['responseHeaders']
);
- browser.webNavigation.onCommitted.addListener(onCommittedListener);
- browser.webNavigation.onCompleted.addListener(onCompletedListener);
- browser.tabs.onRemoved.addListener(onRemovedListener);
- browser.runtime.onMessage.addListener(onMessageListener);
+ chrome.webNavigation.onCommitted.addListener(onCommittedListener);
+ chrome.webNavigation.onCompleted.addListener(onCompletedListener);
+ chrome.tabs.onRemoved.addListener(onRemovedListener);
+ chrome.runtime.onMessage.addListener(onMessageListener);
areListenersRegistered = true;
}
function unregisterListeners() {
- browser.webRequest.onBeforeRequest.removeListener(
+ chrome.webRequest.onBeforeRequest.removeListener(
onBeforeRequestListener
);
- browser.webRequest.onSendHeaders.removeListener(
+ chrome.webRequest.onSendHeaders.removeListener(
onSendHeadersListener
);
- browser.webRequest.onHeadersReceived.removeListener(
+ chrome.webRequest.onHeadersReceived.removeListener(
onHeadersReceivedListener
);
- browser.webRequest.onBeforeRedirect.removeListener(
+ chrome.webRequest.onBeforeRedirect.removeListener(
onBeforeRedirectListener
);
- browser.webRequest.onResponseStarted.removeListener(
+ chrome.webRequest.onResponseStarted.removeListener(
onResponseStartedListener
);
- browser.webNavigation.onCommitted.removeListener(onCommittedListener);
- browser.webNavigation.onCompleted.removeListener(onCompletedListener);
- browser.tabs.onRemoved.removeListener(onRemovedListener);
- browser.runtime.onMessage.removeListener(onMessageListener);
+ chrome.webNavigation.onCommitted.removeListener(onCommittedListener);
+ chrome.webNavigation.onCompleted.removeListener(onCompletedListener);
+ chrome.tabs.onRemoved.removeListener(onRemovedListener);
+ chrome.runtime.onMessage.removeListener(onMessageListener);
areListenersRegistered = false;
}
- function areRequiredBrowserApisAvailable() {
- return requiredBrowserApis.every(function(api) {
- return typeof api !== 'undefined';
- });
- }
-
- if ( areRequiredBrowserApisAvailable() ) {
- ifBrowserValid(
- function() {
- browser.webNavigation.onBeforeNavigate.addListener(
- function(details) {
- if ( details.frameId === 0 ) {
- globalPageContainer.onNewNavigation(details);
- }
- },
- {
- url: [{urlMatches: 'http://*/*'}, {urlMatches: 'https://*/*'}]
- }
- );
- }, function() {
-
- }
- );
- }
-
- browser.runtime.onConnect.addListener((port) => {
- port.onMessage.addListener((message) => {
- if ( message === 'is_browser_valid' ) {
- ifBrowserValid(
- port.postMessage({'browser_valid': true}),
- port.postMessage({'browser_valid': false})
- );
- }
- });
- });
-
- browser.runtime.onConnect.addListener((port) => {
+ chrome.webNavigation.onBeforeNavigate.addListener(
+ function(details) {
+ if ( details.frameId === 0 ) {
+ globalPageContainer.onNewNavigation(details);
+ }
+ },
+ {
+ url: [{urlMatches: 'http://*/*'}, {urlMatches: 'https://*/*'}]
+ }
+ );
+
+ chrome.runtime.onConnect.addListener((port) => {
port.onMessage.addListener((message) => {
if ( message === 'is_tracking_enabled' ) {
ifTrackingEnabled(
@@ -816,7 +746,4 @@
return true;
});
});
-
})();
-
-},{}]},{},[1]);
diff --git a/src/drivers/webextension/js/options.js b/src/drivers/webextension/js/options.js
index 29d11d5d4..505117b52 100644
--- a/src/drivers/webextension/js/options.js
+++ b/src/drivers/webextension/js/options.js
@@ -1,109 +1,44 @@
-/** global: browser */
-/** global: Wappalyzer */
-/* globals browser Wappalyzer */
+'use strict'
/* eslint-env browser */
+/* globals Utils */
-const wappalyzer = new Wappalyzer()
+const { i18n, getOption, setOption } = Utils
-/**
- * Get a value from localStorage
- */
-function getOption(name, defaultValue = null) {
- return new Promise(async (resolve, reject) => {
- let value = defaultValue
+const Options = {
+ async init() {
+ // Theme mode
+ const themeMode = await getOption('themeMode', false)
- try {
- const option = await browser.storage.local.get(name)
-
- if (option[name] !== undefined) {
- value = option[name]
- }
- } catch (error) {
- wappalyzer.log(error.message, 'driver', 'error')
-
- return reject(error.message)
+ if (themeMode) {
+ document.querySelector('body').classList.add('theme-mode')
}
- return resolve(value)
- })
+ ;[
+ ['upgradeMessage', true],
+ ['dynamicIcon', true],
+ ['tracking', true],
+ ['themeMode', false]
+ ].map(async ([option, defaultValue]) => {
+ const el = document
+ .querySelector(
+ `[data-i18n="option${option.charAt(0).toUpperCase() +
+ option.slice(1)}"]`
+ )
+ .parentNode.querySelector('input')
+
+ el.checked = !!(await getOption(option, defaultValue))
+
+ el.addEventListener('click', async () => {
+ await setOption(option, !!el.checked)
+ })
+ })
+
+ i18n()
+ }
}
-/**
- * Set a value in localStorage
- */
-function setOption(name, value) {
- return new Promise(async (resolve, reject) => {
- try {
- await browser.storage.local.set({ [name]: value })
- } catch (error) {
- wappalyzer.log(error.message, 'driver', 'error')
-
- return reject(error.message)
- }
-
- return resolve()
- })
+if (/complete|interactive|loaded/.test(document.readyState)) {
+ Options.init()
+} else {
+ document.addEventListener('DOMContentLoaded', Options.init)
}
-
-document.addEventListener('DOMContentLoaded', async () => {
- const nodes = document.querySelectorAll('[data-i18n]')
-
- Array.prototype.forEach.call(nodes, (node) => {
- node.childNodes[0].nodeValue = browser.i18n.getMessage(node.dataset.i18n)
- })
-
- document.querySelector('#github').addEventListener('click', () => {
- window.open(wappalyzer.config.githubURL)
- })
-
- document.querySelector('#twitter').addEventListener('click', () => {
- window.open(wappalyzer.config.twitterURL)
- })
-
- document.querySelector('#wappalyzer').addEventListener('click', () => {
- window.open(wappalyzer.config.websiteURL)
- })
-
- let el
- let value
-
- // Upgrade message
- value = await getOption('upgradeMessage', true)
-
- el = document.querySelector('#option-upgrade-message')
-
- el.checked = value
-
- el.addEventListener('change', (e) =>
- setOption('upgradeMessage', e.target.checked)
- )
-
- // Dynamic icon
- value = await getOption('dynamicIcon', true)
-
- el = document.querySelector('#option-dynamic-icon')
-
- el.checked = value
-
- el.addEventListener('change', (e) =>
- setOption('dynamicIcon', e.target.checked)
- )
-
- // Tracking
- value = await getOption('tracking', true)
-
- el = document.querySelector('#option-tracking')
-
- el.checked = value
-
- el.addEventListener('change', (e) => setOption('tracking', e.target.checked))
-
- // Theme Mode
- value = await getOption('themeMode', false)
-
- el = document.querySelector('#option-theme-mode')
-
- el.checked = value
-
- el.addEventListener('change', (e) => setOption('themeMode', e.target.checked))
-})
diff --git a/src/drivers/webextension/js/popup.js b/src/drivers/webextension/js/popup.js
index 3aa561403..5852af745 100644
--- a/src/drivers/webextension/js/popup.js
+++ b/src/drivers/webextension/js/popup.js
@@ -1,335 +1,188 @@
+'use strict'
/* eslint-env browser */
-/* global browser, jsonToDOM */
+/* globals chrome, Utils */
-/** global: browser */
-/** global: jsonToDOM */
+const { agent, i18n, getOption, setOption, promisify } = Utils
-let pinnedCategory = null
-let termsAccepted = false
+const Popup = {
+ port: chrome.runtime.connect({ name: 'popup.js' }),
-const port = browser.runtime.connect({
- name: 'popup.js'
-})
-
-function slugify(string) {
- return string
- .toLowerCase()
- .replace(/[^a-z0-9-]/g, '-')
- .replace(/--+/g, '-')
- .replace(/(?:^-|-$)/, '')
-}
-
-function i18n() {
- const nodes = document.querySelectorAll('[data-i18n]')
+ async init() {
+ // Templates
+ Popup.templates = Array.from(
+ document.querySelectorAll('[data-template]')
+ ).reduce((templates, template) => {
+ templates[template.dataset.template] = template.cloneNode(true)
- Array.prototype.forEach.call(nodes, (node) => {
- node.innerHTML = browser.i18n.getMessage(node.dataset.i18n)
- })
-}
-
-function replaceDom(domTemplate) {
- const container = document.getElementsByClassName('container')[0]
-
- while (container.firstChild) {
- container.removeChild(container.firstChild)
- }
+ template.remove()
- container.appendChild(jsonToDOM(domTemplate, document, {}))
+ return templates
+ }, {})
- i18n()
+ // Theme mode
+ const themeMode = await getOption('themeMode', false)
- Array.from(
- document.querySelectorAll('.detected__category-pin-wrapper')
- ).forEach((pin) => {
- pin.addEventListener('click', () => {
- const categoryId = parseInt(pin.dataset.categoryId, 10)
-
- if (categoryId === pinnedCategory) {
- pin.className = 'detected__category-pin-wrapper'
+ if (themeMode) {
+ document.querySelector('body').classList.add('theme-mode')
+ }
- pinnedCategory = null
- } else {
- const active = document.querySelector(
- '.detected__category-pin-wrapper--active'
- )
+ // Terms
+ const termsAccepted =
+ agent === 'chrome' || (await getOption('termsAccepted', false))
- if (active) {
- active.className = 'detected__category-pin-wrapper'
- }
+ if (termsAccepted) {
+ document.querySelector('.terms').style.display = 'none'
- pin.className =
- 'detected__category-pin-wrapper detected__category-pin-wrapper--active'
+ Popup.driver('getDetections')
+ } else {
+ document.querySelector('.detections').style.display = 'none'
- pinnedCategory = categoryId
- }
+ i18n()
+ }
- port.postMessage({
- id: 'set_option',
- key: 'pinnedCategory',
- value: pinnedCategory
- })
+ // Alert
+ const [{ url }] = await promisify(chrome.tabs, 'query', {
+ active: true,
+ currentWindow: true
})
- })
- Array.from(document.querySelectorAll('a')).forEach((link) => {
- link.addEventListener('click', () => {
- browser.tabs.create({ url: link.href })
+ document.querySelector(
+ '.alerts__link'
+ ).href = `https://www.wappalyzer.com/alerts/manage?url=${encodeURIComponent(
+ `${url}`
+ )}`
- return false
- })
- })
-}
+ document
+ .querySelector('.footer__settings')
+ .addEventListener('click', () => chrome.runtime.openOptionsPage())
+ },
+
+ driver(func, ...args) {
+ Popup.port.postMessage({ func, args })
+ },
+
+ log(message) {
+ Popup.driver('log', message, 'popup.js')
+ },
+
+ categorise(technologies) {
+ return Object.values(
+ technologies.reduce((categories, technology) => {
+ technology.categories.forEach((category) => {
+ categories[category.id] = categories[category.id] || {
+ ...category,
+ technologies: []
+ }
+
+ categories[category.id].technologies.push(technology)
+ })
+
+ return categories
+ }, {})
+ )
+ },
+
+ async onGetDetections(detections) {
+ const pinnedCategory = await getOption('pinnedCategory')
+
+ if (detections.length) {
+ document.querySelector('.empty').remove()
+ }
-function replaceDomWhenReady(dom) {
- if (/complete|interactive|loaded/.test(document.readyState)) {
- replaceDom(dom)
- } else {
- document.addEventListener('DOMContentLoaded', () => {
- replaceDom(dom)
- })
- }
-}
+ Popup.categorise(detections).forEach(
+ ({ id, name, slug: categorySlug, technologies }) => {
+ const categoryNode = Popup.templates.category.cloneNode(true)
+
+ const link = categoryNode.querySelector('.category__link')
-function appsToDomTemplate(response) {
- let template = []
+ link.href = `https://www.wappalyzer.com/technologies/${categorySlug}`
+ link.textContent = name
- if (response.tabCache && Object.keys(response.tabCache.detected).length > 0) {
- const categories = {}
+ const pins = categoryNode.querySelectorAll('.category__pin')
- // Group apps by category
- for (const appName in response.tabCache.detected) {
- response.apps[appName].cats.forEach((cat) => {
- categories[cat] = categories[cat] || {
- name: response.categories[cat].name,
- apps: []
+ if (pinnedCategory === id) {
+ pins.forEach((pin) => pin.classList.add('category__pin--active'))
}
- categories[cat].apps[appName] = appName
- })
- }
+ pins.forEach((pin) =>
+ pin.addEventListener('click', async () => {
+ const pinnedCategory = await getOption('pinnedCategory')
- for (const cat in categories) {
- const apps = []
-
- for (const appName in categories[cat].apps) {
- const { confidenceTotal, version } = response.tabCache.detected[appName]
-
- apps.push([
- 'a',
- {
- class: 'detected__app',
- href: `https://www.wappalyzer.com/technologies/${slugify(
- categories[cat].name
- )}/${slugify(appName)}`
- },
- [
- 'img',
- {
- class: 'detected__app-icon',
- src: `../images/icons/${response.apps[appName].icon ||
- 'default.svg'}`
- }
- ],
- [
- 'span',
- {
- class: 'detected__app-name'
- },
- appName
- ],
- version
- ? [
- 'span',
- {
- class: 'detected__app-version'
- },
- version
- ]
- : null,
- confidenceTotal < 100
- ? [
- 'span',
- {
- class: 'detected__app-confidence'
- },
- `${confidenceTotal}% sure`
- ]
- : null
- ])
- }
+ Array.from(
+ document.querySelectorAll('.category__pin--active')
+ ).forEach((pin) => pin.classList.remove('category__pin--active'))
- template.push([
- 'div',
- {
- class: 'detected__category'
- },
- [
- 'div',
- {
- class: 'detected__category-name'
- },
- [
- 'a',
- {
- class: 'detected__category-link',
- href: `https://www.wappalyzer.com/categories/${slugify(
- response.categories[cat].name
- )}`
- },
- browser.i18n.getMessage(`categoryName${cat}`)
- ],
- [
- 'span',
- {
- class: `detected__category-pin-wrapper${
- parseInt(pinnedCategory, 10) === parseInt(cat, 10)
- ? ' detected__category-pin-wrapper--active'
- : ''
- }`,
- 'data-category-id': cat,
- title: browser.i18n.getMessage('categoryPin')
- },
- [
- 'img',
- {
- class: 'detected__category-pin detected__category-pin--active',
- src: '../images/pin-active.svg'
- }
- ],
- [
- 'img',
- {
- class:
- 'detected__category-pin detected__category-pin--inactive',
- src: '../images/pin.svg'
- }
- ]
- ]
- ],
- [
- 'div',
- {
- class: 'detected__apps'
- },
- apps
- ]
- ])
- }
-
- template = [
- 'div',
- {
- class: 'detected'
- },
- template
- ]
- } else {
- template = [
- 'div',
- {
- class: 'empty'
- },
- [
- 'span',
- {
- class: 'empty__text'
- },
- browser.i18n.getMessage('noAppsDetected')
- ]
- ]
- }
+ if (pinnedCategory === id) {
+ await setOption('pinnedCategory', null)
+ } else {
+ await setOption('pinnedCategory', id)
- return template
-}
+ pins.forEach((pin) => pin.classList.add('category__pin--active'))
+ }
+ })
+ )
-async function getApps() {
- try {
- const tabs = await browser.tabs.query({
- active: true,
- currentWindow: true
- })
+ technologies
+ .filter(({ confidence }) => confidence)
+ .forEach(({ name, slug, confidence, version, icon, website }) => {
+ const technologyNode = Popup.templates.technology.cloneNode(true)
- const url = new URL(tabs[0].url)
+ const image = technologyNode.querySelector('.technology__icon')
- document.querySelector(
- '.footer__link'
- ).href = `https://www.wappalyzer.com/alerts/manage?url=${encodeURIComponent(
- `${url.protocol}//${url.hostname}`
- )}`
+ image.src = `../images/icons/${icon}`
- port.postMessage({
- id: 'get_apps',
- tab: tabs[0]
- })
- } catch (error) {
- console.error(error) // eslint-disable-line no-console
- }
-}
+ const link = technologyNode.querySelector('.technology__link')
-/**
- * Async function to update body class based on option.
- */
-function getThemeMode() {
- try {
- port.postMessage({
- id: 'update_theme_mode'
- })
- } catch (error) {
- console.error(error) // eslint-disable-line no-console
- }
-}
+ link.href = `https://www.wappalyzer.com/technologies/${categorySlug}/${slug}`
+ link.textContent = name
-/**
- * Update theme mode based on browser option.
- * @param {object} res Response from port listener.
- */
-function updateThemeMode(res) {
- if (res.hasOwnProperty('themeMode') && res.themeMode !== false) {
- document.body.classList.add('theme-mode-sync')
- }
-}
+ const confidenceNode = technologyNode.querySelector(
+ '.technology__confidence'
+ )
-function displayApps(response) {
- pinnedCategory = response.pinnedCategory // eslint-disable-line prefer-destructuring
- termsAccepted = response.termsAccepted // eslint-disable-line prefer-destructuring
+ if (confidence < 100) {
+ confidenceNode.textContent = `${confidence}% sure`
+ } else {
+ confidenceNode.remove()
+ }
- if (termsAccepted) {
- replaceDomWhenReady(appsToDomTemplate(response))
- } else {
- i18n()
+ const versionNode = technologyNode.querySelector(
+ '.technology__version'
+ )
- const wrapper = document.querySelector('.terms__wrapper')
+ if (version) {
+ versionNode.textContent = version
+ } else {
+ versionNode.remove()
+ }
- document.querySelector('.terms__accept').addEventListener('click', () => {
- port.postMessage({
- id: 'set_option',
- key: 'termsAccepted',
- value: true
- })
+ categoryNode
+ .querySelector('.technologies')
+ .appendChild(technologyNode)
+ })
- wrapper.classList.remove('terms__wrapper--active')
+ document.querySelector('.detections').appendChild(categoryNode)
+ }
+ )
- getApps()
- })
+ Array.from(document.querySelectorAll('a')).forEach((a) =>
+ a.addEventListener('click', () => Popup.driver('open', a.href))
+ )
- wrapper.classList.add('terms__wrapper--active')
+ i18n()
}
}
-port.onMessage.addListener((message) => {
- switch (message.id) {
- case 'get_apps':
- displayApps(message.response)
-
- break
- case 'update_theme_mode':
- updateThemeMode(message.response)
+Popup.port.onMessage.addListener(({ func, args }) => {
+ const onFunc = `on${func.charAt(0).toUpperCase() + func.slice(1)}`
- break
- default:
- // Do nothing
+ if (Popup[onFunc]) {
+ Popup[onFunc](args)
}
})
-getThemeMode()
-getApps()
+if (/complete|interactive|loaded/.test(document.readyState)) {
+ Popup.init()
+} else {
+ document.addEventListener('DOMContentLoaded', Popup.init)
+}
diff --git a/src/drivers/webextension/js/utils.js b/src/drivers/webextension/js/utils.js
new file mode 100644
index 000000000..3f83449d1
--- /dev/null
+++ b/src/drivers/webextension/js/utils.js
@@ -0,0 +1,53 @@
+'use strict'
+/* eslint-env browser */
+/* globals chrome */
+
+const Utils = {
+ agent: chrome.extension.getURL('/').startsWith('moz-') ? 'firefox' : 'chrome',
+
+ promisify(context, method, ...args) {
+ return new Promise((resolve, reject) => {
+ context[method](...args, (...args) => {
+ if (chrome.runtime.lastError) {
+ return reject(chrome.runtime.lastError)
+ }
+
+ resolve(...args)
+ })
+ })
+ },
+
+ open(url, active = true) {
+ chrome.tabs.create({ url, active })
+ },
+
+ async getOption(name, defaultValue = null) {
+ try {
+ const option = await Utils.promisify(chrome.storage.local, 'get', name)
+
+ if (option[name] !== undefined) {
+ return option[name]
+ }
+
+ return defaultValue
+ } catch (error) {
+ throw new Error(error.message || error.toString())
+ }
+ },
+
+ async setOption(name, value) {
+ try {
+ await Utils.promisify(chrome.storage.local, 'set', {
+ [name]: value
+ })
+ } catch (error) {
+ throw new Error(error.message || error.toString())
+ }
+ },
+
+ i18n() {
+ Array.from(document.querySelectorAll('[data-i18n]')).forEach(
+ (node) => (node.innerHTML = chrome.i18n.getMessage(node.dataset.i18n))
+ )
+ }
+}
diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json
index e034825ac..e00572b6b 100644
--- a/src/drivers/webextension/manifest.json
+++ b/src/drivers/webextension/manifest.json
@@ -36,7 +36,6 @@
"https://*/*"
],
"js": [
- "node_modules/webextension-polyfill/dist/browser-polyfill.js",
"js/content.js"
],
"run_at": "document_idle"
@@ -46,12 +45,7 @@
"http://*/*",
"https://*/*"
],
- "exclude_matches": [
- "https://*.modirum.com/*",
- "https://www.alphaecommerce.gr/*"
- ],
"js": [
- "node_modules/webextension-polyfill/dist/browser-polyfill.js",
"js/lib/iframe.js"
],
"run_at": "document_start",
diff --git a/src/wappalyzer.js b/src/wappalyzer.js
index 043975167..e72eb8af7 100644
--- a/src/wappalyzer.js
+++ b/src/wappalyzer.js
@@ -1,735 +1,331 @@
-/**
- * Wappalyzer v5
- *
- * Created by Elbert Alias
- *
- * License: GPLv3 http://www.gnu.org/licenses/gpl-3.0.txt
- */
-
-const validation = {
- hostname: /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/,
- hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/
-}
-
-/**
- * Enclose string in array
- */
-function asArray(value) {
- return Array.isArray(value) ? value : [value]
-}
-
-/**
- *
- */
-function asyncForEach(iterable, iterator) {
- return Promise.all(
- (iterable || []).map(
- (item) =>
- new Promise((resolve) => setTimeout(() => resolve(iterator(item)), 1))
- )
- )
-}
-
-/**
- * Mark application as detected, set confidence and version
- */
-function addDetected(app, pattern, type, value, key) {
- app.detected = true
-
- // Set confidence level
- app.confidence[`${type} ${key ? `${key} ` : ''}${pattern.regex}`] =
- pattern.confidence === undefined ? 100 : parseInt(pattern.confidence, 10)
-
- // Detect version number
- if (pattern.version) {
- const versions = []
- const matches = pattern.regex.exec(value)
-
- let { version } = pattern
-
- if (matches) {
- matches.forEach((match, i) => {
- // Parse ternary operator
- const ternary = new RegExp(`\\\\${i}\\?([^:]+):(.*)$`).exec(version)
-
- if (ternary && ternary.length === 3) {
- version = version.replace(ternary[0], match ? ternary[1] : ternary[2])
- }
-
- // Replace back references
- version = version
- .trim()
- .replace(new RegExp(`\\\\${i}`, 'g'), match || '')
- })
-
- if (version && !versions.includes(version)) {
- versions.push(version)
- }
+'use strict'
+
+const Wappalyzer = {
+ technologies: [],
+ categories: [],
+
+ slugify(string) {
+ return string
+ .toLowerCase()
+ .replace(/[^a-z0-9-]/g, '-')
+ .replace(/--+/g, '-')
+ .replace(/(?:^-|-$)/, '')
+ },
+
+ getTechnology(name) {
+ return Wappalyzer.technologies.find(({ name: _name }) => name === _name)
+ },
+
+ getCategory(id) {
+ return Wappalyzer.categories.find(({ id: _id }) => id === _id)
+ },
+
+ resolve(detections = []) {
+ const resolved = detections.reduce((resolved, { technology }) => {
+ if (
+ resolved.findIndex(
+ ({ technology: { name } }) => name === technology.name
+ ) === -1
+ ) {
+ let version = ''
+ let confidence = 0
+
+ detections.forEach(({ technology: { name }, pattern, match }) => {
+ if (name === technology.name) {
+ const versionValue = Wappalyzer.resolveVersion(pattern, match)
+
+ confidence = Math.min(100, confidence + pattern.confidence)
+ version =
+ versionValue.length > version.length && versionValue.length <= 10
+ ? versionValue
+ : version
+ }
+ })
- if (versions.length) {
- // Use the longest detected version number
- app.version = versions.reduce((a, b) => (a.length > b.length ? a : b))
+ resolved.push({ technology, confidence, version })
}
- }
- }
-}
-
-function resolveExcludes(apps, detected) {
- const excludes = []
- const detectedApps = Object.assign({}, apps, detected)
-
- // Exclude app in detected apps only
- Object.keys(detectedApps).forEach((appName) => {
- const app = detectedApps[appName]
- if (app.props.excludes) {
- asArray(app.props.excludes).forEach((excluded) => {
- excludes.push(excluded)
+ return resolved
+ }, [])
+
+ Wappalyzer.resolveExcludes(resolved)
+ Wappalyzer.resolveImplies(resolved)
+
+ return resolved.map(
+ ({
+ technology: { name, slug, categories, icon, website },
+ confidence,
+ version
+ }) => ({
+ name,
+ slug,
+ categories: categories.map((id) => Wappalyzer.getCategory(id)),
+ confidence,
+ version,
+ icon,
+ website
})
- }
- })
-
- // Remove excluded applications
- Object.keys(apps).forEach((appName) => {
- if (excludes.includes(appName)) {
- delete apps[appName]
- }
- })
-}
-
-class Application {
- constructor(name, props, detected) {
- this.confidence = {}
- this.confidenceTotal = 0
- this.detected = Boolean(detected)
- this.excludes = []
- this.name = name
- this.props = props
- this.version = ''
- }
-
- /**
- * Calculate confidence total
- */
- getConfidence() {
- let total = 0
-
- Object.keys(this.confidence).forEach((id) => {
- total += this.confidence[id]
- })
-
- this.confidenceTotal = Math.min(total, 100)
-
- return this.confidenceTotal
- }
-}
-
-class Wappalyzer {
- constructor() {
- this.apps = {}
- this.categories = {}
- this.driver = {}
- this.jsPatterns = {}
- this.detected = {}
- this.hostnameCache = {
- expires: Date.now() + 1000 * 60 * 60 * 24,
- hostnames: {}
- }
- this.adCache = []
-
- this.config = {
- websiteURL: 'https://www.wappalyzer.com/',
- twitterURL: 'https://twitter.com/Wappalyzer',
- githubURL: 'https://github.com/AliasIO/Wappalyzer'
- }
- }
-
- /**
- * Log messages to console
- */
- log(message, source, type) {
- if (this.driver.log) {
- this.driver.log(message, source || '', type || 'debug')
- }
- }
-
- analyze(url, data, context) {
- const apps = {}
- const promises = []
- const startTime = new Date()
- const { scripts, cookies, headers, js } = data
+ )
+ },
- let { html } = data
+ resolveVersion({ version, regex }, match) {
+ let resolved = version
- if (this.detected[url.canonical] === undefined) {
- this.detected[url.canonical] = {}
- }
+ if (version) {
+ const matches = regex.exec(match)
- const metaTags = []
+ if (matches) {
+ matches.forEach((match, index) => {
+ // Parse ternary operator
+ const ternary = new RegExp(`\\\\${index}\\?([^:]+):(.*)$`).exec(
+ version
+ )
- // Additional information
- let language = null
+ if (ternary && ternary.length === 3) {
+ resolved = version.replace(
+ ternary[0],
+ match ? ternary[1] : ternary[2]
+ )
+ }
- if (html) {
- if (typeof html !== 'string') {
- html = ''
+ // Replace back references
+ resolved = resolved
+ .trim()
+ .replace(new RegExp(`\\\\${index}`, 'g'), match || '')
+ })
}
-
- let matches = data.html.match(
- new RegExp(']*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"', 'i')
- )
-
- language = matches && matches.length ? matches[1] : data.language || null
-
- // Meta tags
- const regex = /]+>/gi
-
- do {
- matches = regex.exec(html)
-
- if (!matches) {
- break
- }
-
- metaTags.push(matches[0])
- } while (matches)
}
- Object.keys(this.apps).forEach((appName) => {
- apps[appName] =
- this.detected[url.canonical] && this.detected[url.canonical][appName]
- ? this.detected[url.canonical][appName]
- : new Application(appName, this.apps[appName])
+ return resolved
+ },
- const app = apps[appName]
+ resolveExcludes(resolved) {
+ resolved.forEach(({ technology }) => {
+ technology.excludes.forEach((name) => {
+ const excluded = Wappalyzer.getTechnology(name)
- promises.push(this.analyzeUrl(app, url))
-
- if (html) {
- promises.push(this.analyzeHtml(app, html))
- promises.push(this.analyzeMeta(app, metaTags))
- }
-
- if (scripts) {
- promises.push(this.analyzeScripts(app, scripts))
- }
-
- if (cookies) {
- promises.push(this.analyzeCookies(app, cookies))
- }
-
- if (headers) {
- promises.push(this.analyzeHeaders(app, headers))
- }
- })
-
- if (js) {
- Object.keys(js).forEach((appName) => {
- if (typeof js[appName] !== 'function') {
- promises.push(this.analyzeJs(apps[appName], js[appName]))
+ if (!excluded) {
+ throw new Error(`Excluded technology does not exist: ${name}`)
}
- })
- }
-
- return new Promise(async (resolve) => {
- await Promise.all(promises)
- Object.keys(apps).forEach((appName) => {
- const app = apps[appName]
+ const index = resolved.findIndex(({ name }) => name === excluded.name)
- if (!app.detected || !app.getConfidence()) {
- delete apps[app.name]
+ if (index === -1) {
+ resolved.splice(index, 1)
}
})
-
- resolveExcludes(apps, this.detected[url])
- this.resolveImplies(apps, url.canonical)
-
- this.cacheDetectedApps(apps, url.canonical)
- this.trackDetectedApps(apps, url, language)
-
- this.log(
- `Processing ${Object.keys(data).join(', ')} took ${(
- (new Date() - startTime) /
- 1000
- ).toFixed(2)}s (${url.hostname})`,
- 'core'
- )
-
- if (Object.keys(apps).length) {
- this.log(
- `Identified ${Object.keys(apps).join(', ')} (${url.hostname})`,
- 'core'
- )
- }
-
- this.driver.displayApps(
- this.detected[url.canonical],
- { language },
- context
- )
-
- return resolve()
})
- }
+ },
- /**
- * Cache detected ads
- */
- cacheDetectedAds(ad) {
- this.adCache.push(ad)
- }
-
- /**
- *
- */
- robotsTxtAllows(url) {
- return new Promise(async (resolve, reject) => {
- const parsed = this.parseUrl(url)
+ resolveImplies(resolved) {
+ let done = false
- if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
- return reject()
- }
-
- const robotsTxt = await this.driver.getRobotsTxt(
- parsed.host,
- parsed.protocol === 'https:'
- )
-
- if (
- robotsTxt.some(
- (disallowedPath) => parsed.pathname.indexOf(disallowedPath) === 0
- )
- ) {
- return reject()
- }
-
- return resolve()
- })
- }
-
- /**
- * Parse a URL
- */
- parseUrl(url) {
- const a = this.driver.document.createElement('a')
-
- a.href = url
-
- a.canonical = `${a.protocol}//${a.host}${a.pathname}`
-
- return a
- }
-
- /**
- *
- */
- static parseRobotsTxt(robotsTxt) {
- const disallow = []
-
- let userAgent
-
- robotsTxt.split('\n').forEach((line) => {
- let matches = /^User-agent:\s*(.+)$/i.exec(line.trim())
-
- if (matches) {
- userAgent = matches[1].toLowerCase()
- } else if (userAgent === '*' || userAgent === 'wappalyzer') {
- matches = /^Disallow:\s*(.+)$/i.exec(line.trim())
-
- if (matches) {
- disallow.push(matches[1])
- }
- }
- })
-
- return disallow
- }
-
- /**
- *
- */
- ping() {
- if (
- !this.hostnameCache.hostnames ||
- Object.keys(this.hostnameCache.hostnames).length > 50 ||
- this.hostnameCache.expires < Date.now()
- ) {
- this.driver.ping(this.hostnameCache)
-
- this.hostnameCache = {
- expires: Date.now() + 1000 * 60 * 60 * 24,
- hostnames: {}
- }
- }
+ while (resolved.length && !done) {
+ resolved.forEach(({ technology, confidence }) => {
+ done = true
- if (this.adCache.length > 50) {
- this.driver.ping(undefined, this.adCache)
+ technology.implies.forEach((name) => {
+ const implied = Wappalyzer.getTechnology(name)
- this.adCache = []
- }
- }
-
- /**
- * Parse apps.json patterns
- */
- parsePatterns(patterns) {
- if (!patterns) {
- return []
- }
-
- let parsed = {}
-
- // Convert string to object containing array containing string
- if (typeof patterns === 'string' || Array.isArray(patterns)) {
- patterns = {
- main: asArray(patterns)
- }
- }
-
- Object.keys(patterns).forEach((key) => {
- parsed[key] = []
-
- asArray(patterns[key]).forEach((pattern) => {
- const attrs = {}
-
- pattern.split('\\;').forEach((attr, i) => {
- if (i) {
- // Key value pairs
- attr = attr.split(':')
-
- if (attr.length > 1) {
- attrs[attr.shift()] = attr.join(':')
- }
- } else {
- attrs.string = attr
+ if (!implied) {
+ throw new Error(`Implied technology does not exist: ${name}`)
+ }
- try {
- attrs.regex = new RegExp(attr.replace('/', '/'), 'i') // Escape slashes in regular expression
- } catch (error) {
- attrs.regex = new RegExp()
+ if (
+ resolved.findIndex(
+ ({ technology: { name } }) => name === implied.name
+ ) === -1
+ ) {
+ resolved.push({ technology: implied, confidence, version: '' })
- this.log(`${error.message}: ${attr}`, 'error', 'core')
- }
+ done = false
}
})
-
- parsed[key].push(attrs)
})
- })
-
- // Convert back to array if the original pattern list was an array (or string)
- if ('main' in parsed) {
- parsed = parsed.main
}
-
- return parsed
- }
-
- /**
- * Parse JavaScript patterns
- */
- parseJsPatterns() {
- Object.keys(this.apps).forEach((appName) => {
- if (this.apps[appName].js) {
- this.jsPatterns[appName] = this.parsePatterns(this.apps[appName].js)
- }
- })
- }
-
- resolveImplies(apps, url) {
- let checkImplies = true
-
- const resolve = (appName) => {
- const app = apps[appName]
-
- if (app && app.props.implies) {
- asArray(app.props.implies).forEach((implied) => {
- ;[implied] = this.parsePatterns(implied)
-
- if (!this.apps[implied.string]) {
- this.log(
- `Implied application ${implied.string} does not exist`,
- 'core',
- 'warn'
+ },
+
+ async analyze(url, { html, meta, headers, cookies, scripts }) {
+ const oo = Wappalyzer.analyzeOneToOne
+ const om = Wappalyzer.analyzeOneToMany
+ const mm = Wappalyzer.analyzeManyToMany
+
+ const flatten = (array) => Array.prototype.concat.apply([], array)
+
+ try {
+ const detections = flatten(
+ flatten(
+ await Promise.all(
+ Wappalyzer.technologies.map((technology) =>
+ Promise.all([
+ oo(technology, 'url', url),
+ oo(technology, 'html', html),
+ om(technology, 'meta', meta),
+ mm(technology, 'headers', headers),
+ om(technology, 'cookies', cookies),
+ om(technology, 'scripts', scripts)
+ ])
)
+ )
+ )
+ ).filter((technology) => technology)
- return
- }
-
- if (!(implied.string in apps)) {
- apps[implied.string] =
- this.detected[url] && this.detected[url][implied.string]
- ? this.detected[url][implied.string]
- : new Application(
- implied.string,
- this.apps[implied.string],
- true
- )
-
- checkImplies = true
- }
-
- // Apply app confidence to implied app
- Object.keys(app.confidence).forEach((id) => {
- apps[implied.string].confidence[`${id} implied by ${appName}`] =
- app.confidence[id] *
- (implied.confidence === undefined ? 1 : implied.confidence / 100)
- })
- })
- }
- }
-
- // Implied applications
- // Run several passes as implied apps may imply other apps
- while (checkImplies) {
- checkImplies = false
-
- Object.keys(apps).forEach(resolve)
+ return detections
+ } catch (error) {
+ throw new Error(error.message || error.toString())
}
- }
+ },
+
+ setTechnologies(data) {
+ const transform = Wappalyzer.transformPatterns
+
+ Wappalyzer.technologies = Object.keys(data).reduce((technologies, name) => {
+ const {
+ cats,
+ url,
+ html,
+ meta,
+ headers,
+ cookies,
+ script,
+ js,
+ implies,
+ excludes,
+ icon,
+ website
+ } = data[name]
+
+ technologies.push({
+ name,
+ categories: cats || [],
+ slug: Wappalyzer.slugify(name),
+ url: transform(url),
+ headers: transform(
+ Object.keys(headers || {}).reduce(
+ (lcHeaders, header) => ({
+ ...lcHeaders,
+ [header.toLowerCase()]: headers[header]
+ }),
+ {}
+ )
+ ),
+ cookies: transform(cookies),
+ html: transform(html),
+ meta: transform(meta),
+ scripts: transform(script),
+ js: transform(js),
+ implies: typeof implies === 'string' ? [implies] : implies || [],
+ excludes: typeof excludes === 'string' ? [excludes] : excludes || [],
+ icon: icon || 'default.svg',
+ website: website || ''
+ })
- /**
- * Cache detected applications
- */
- cacheDetectedApps(apps, url) {
- Object.keys(apps).forEach((appName) => {
- const app = apps[appName]
+ return technologies
+ }, [])
+ },
- // Per URL
- this.detected[url][appName] = app
+ setCategories(data) {
+ Wappalyzer.categories = Object.keys(data)
+ .reduce((categories, id) => {
+ const category = data[id]
- Object.keys(app.confidence).forEach((id) => {
- this.detected[url][appName].confidence[id] = app.confidence[id]
- })
- })
+ categories.push({
+ id: parseInt(id, 10),
+ slug: Wappalyzer.slugify(category.name),
+ ...category
+ })
- if (this.driver.ping instanceof Function) {
- this.ping()
- }
- }
+ return categories
+ }, [])
+ .sort(({ priority: a }, { priority: b }) => (a > b ? -1 : 0))
+ },
- /**
- * Track detected applications
- */
- trackDetectedApps(apps, url, language) {
- if (!(this.driver.ping instanceof Function)) {
- return
+ transformPatterns(patterns) {
+ if (!patterns) {
+ return []
}
- const hostname = `${url.protocol}//${url.hostname}`
+ const toArray = (value) => (Array.isArray(value) ? value : [value])
- Object.keys(apps).forEach((appName) => {
- const app = apps[appName]
-
- if (this.detected[url.canonical][appName].getConfidence() >= 100) {
- if (
- validation.hostname.test(url.hostname) &&
- !validation.hostnameBlacklist.test(url.hostname)
- ) {
- if (!(hostname in this.hostnameCache.hostnames)) {
- this.hostnameCache.hostnames[hostname] = {
- applications: {},
- meta: {}
- }
- }
+ if (typeof patterns === 'string' || Array.isArray(patterns)) {
+ patterns = { main: patterns }
+ }
- if (
- !(appName in this.hostnameCache.hostnames[hostname].applications)
- ) {
- this.hostnameCache.hostnames[hostname].applications[appName] = {
- hits: 0
+ const parsed = Object.keys(patterns).reduce((parsed, key) => {
+ parsed[key] = toArray(patterns[key]).map((pattern) => {
+ const { regex, confidence, version } = pattern
+ .split('\\;')
+ .reduce((attrs, attr, i) => {
+ if (i) {
+ // Key value pairs
+ attr = attr.split(':')
+
+ if (attr.length > 1) {
+ attrs[attr.shift()] = attr.join(':')
+ }
+ } else {
+ // Escape slashes in regular expression
+ attrs.regex = new RegExp(attr.replace(/\//g, '\\/'), 'i')
}
- }
- this.hostnameCache.hostnames[hostname].applications[appName].hits += 1
+ return attrs
+ }, {})
- if (apps[appName].version) {
- this.hostnameCache.hostnames[hostname].applications[
- appName
- ].version = app.version
- }
+ return {
+ regex,
+ confidence: parseInt(confidence || 100, 10),
+ version: version || ''
}
- }
- })
-
- if (hostname in this.hostnameCache.hostnames) {
- this.hostnameCache.hostnames[hostname].meta.language = language
- }
-
- this.ping()
- }
-
- /**
- * Analyze URL
- */
- analyzeUrl(app, url) {
- const patterns = this.parsePatterns(app.props.url)
-
- if (!patterns.length) {
- return Promise.resolve()
- }
-
- return asyncForEach(patterns, (pattern) => {
- if (pattern.regex.test(url.canonical)) {
- addDetected(app, pattern, 'url', url.canonical)
- }
- })
- }
+ })
- /**
- * Analyze HTML
- */
- analyzeHtml(app, html) {
- const patterns = this.parsePatterns(app.props.html)
+ return parsed
+ }, {})
- if (!patterns.length) {
- return Promise.resolve()
- }
+ return 'main' in parsed ? parsed.main : parsed
+ },
- return asyncForEach(patterns, (pattern) => {
- if (pattern.regex.test(html)) {
- addDetected(app, pattern, 'html', html)
+ analyzeOneToOne(technology, type, value) {
+ return technology[type].reduce((technologies, pattern) => {
+ if (pattern.regex.test(value)) {
+ technologies.push({ technology, pattern, match: value })
}
- })
- }
- /**
- * Analyze script tag
- */
- analyzeScripts(app, scripts) {
- const patterns = this.parsePatterns(app.props.script)
+ return technologies
+ }, [])
+ },
- if (!patterns.length) {
- return Promise.resolve()
- }
+ analyzeOneToMany(technology, type, items = []) {
+ return items.reduce((technologies, { key, value }) => {
+ const patterns = technology[type][key] || []
- return asyncForEach(patterns, (pattern) => {
- scripts.forEach((uri) => {
- if (pattern.regex.test(uri)) {
- addDetected(app, pattern, 'script', uri)
+ patterns.forEach((pattern) => {
+ if (pattern.regex.test(value)) {
+ technologies.push({ technology, pattern, match: value })
}
})
- })
- }
- /**
- * Analyze meta tag
- */
- analyzeMeta(app, metaTags) {
- const patterns = this.parsePatterns(app.props.meta)
- const promises = []
+ return technologies
+ }, [])
+ },
- if (!app.props.meta) {
- return Promise.resolve()
- }
+ analyzeManyToMany(technology, type, items = {}) {
+ return Object.keys(technology[type]).reduce((technologies, key) => {
+ const patterns = technology[type][key] || []
+ const values = items[key] || []
- metaTags.forEach((match) => {
- Object.keys(patterns).forEach((meta) => {
- const r = new RegExp(`(?:name|property)=["']${meta}["']`, 'i')
-
- if (r.test(match)) {
- const content = match.match(/content=("|')([^"']+)("|')/i)
-
- promises.push(
- asyncForEach(patterns[meta], (pattern) => {
- if (
- content &&
- content.length === 4 &&
- pattern.regex.test(content[2])
- ) {
- addDetected(app, pattern, 'meta', content[2], meta)
- }
- })
- )
- }
+ patterns.forEach((pattern) => {
+ values.forEach((value) => {
+ if (pattern.regex.test(value)) {
+ technologies.push({ technology, pattern, match: value })
+ }
+ })
})
- })
-
- return Promise.all(promises)
- }
-
- /**
- * Analyze response headers
- */
- analyzeHeaders(app, headers) {
- const patterns = this.parsePatterns(app.props.headers)
- const promises = []
-
- Object.keys(patterns).forEach((headerName) => {
- if (typeof patterns[headerName] !== 'function') {
- promises.push(
- asyncForEach(patterns[headerName], (pattern) => {
- headerName = headerName.toLowerCase()
-
- if (headerName in headers) {
- headers[headerName].forEach((headerValue) => {
- if (pattern.regex.test(headerValue)) {
- addDetected(app, pattern, 'headers', headerValue, headerName)
- }
- })
- }
- })
- )
- }
- })
-
- return promises ? Promise.all(promises) : Promise.resolve()
- }
-
- /**
- * Analyze cookies
- */
- analyzeCookies(app, cookies) {
- const patterns = this.parsePatterns(app.props.cookies)
- const promises = []
-
- Object.keys(patterns).forEach((cookieName) => {
- if (typeof patterns[cookieName] !== 'function') {
- const cookieNameLower = cookieName.toLowerCase()
-
- promises.push(
- asyncForEach(patterns[cookieName], (pattern) => {
- const cookie = cookies.find(
- (_cookie) => _cookie.name.toLowerCase() === cookieNameLower
- )
-
- if (cookie && pattern.regex.test(cookie.value)) {
- addDetected(app, pattern, 'cookies', cookie.value, cookieName)
- }
- })
- )
- }
- })
-
- return promises ? Promise.all(promises) : Promise.resolve()
- }
-
- /**
- * Analyze JavaScript variables
- */
- analyzeJs(app, results) {
- const promises = []
-
- Object.keys(results).forEach((string) => {
- if (typeof results[string] !== 'function') {
- promises.push(
- asyncForEach(Object.keys(results[string]), (index) => {
- const pattern = this.jsPatterns[app.name][string][index]
- const value = results[string][index]
-
- if (pattern && pattern.regex.test(value)) {
- addDetected(app, pattern, 'js', value, string)
- }
- })
- )
- }
- })
- return promises ? Promise.all(promises) : Promise.resolve()
+ return technologies
+ }, [])
}
}
-if (typeof module === 'object') {
+if (typeof module !== 'undefined') {
module.exports = Wappalyzer
}
diff --git a/src/wappalyzer.spec.js b/src/wappalyzer.spec.js
index f51d7fb10..0df917d63 100644
--- a/src/wappalyzer.spec.js
+++ b/src/wappalyzer.spec.js
@@ -1,51 +1,50 @@
/* eslint-env mocha */
-const { assert, expect } = require('chai');
-const Wappalyzer = require('../src/wappalyzer');
+const { assert, expect } = require('chai')
+const Wappalyzer = require('../src/wappalyzer')
const appsJson = {
appUrl: {
- url: 'test',
+ url: 'test'
},
appCookies: {
cookies: {
- test: 'test',
- },
+ test: 'test'
+ }
},
appUppercaseCookies: {
cookies: {
- Test: 'Test',
- },
+ Test: 'Test'
+ }
},
appHeaders: {
headers: {
- 'X-Powered-By': 'test',
- },
+ 'X-Powered-By': 'test'
+ }
},
appHtml: {
html: 'test v(\\d)\\;confidence:50\\;version:\\1',
implies: 'appImplies',
- excludes: 'appExcludes',
+ excludes: 'appExcludes'
},
appMeta: {
meta: {
- generator: 'test',
- },
+ generator: 'test'
+ }
},
appScript: {
- script: 'test',
+ script: 'test'
},
appJs: {
js: {
- key: 'value',
- },
- },
- appImplies: {
+ key: 'value'
+ }
},
+ appImplies: {},
appExcludes: {
- html: 'test',
- },
-};
+ html: 'test'
+ }
+}
const driverData = {
cookies: [
@@ -53,92 +52,86 @@ const driverData = {
name: 'test',
value: 'test',
domain: '',
- path: '',
- },
+ path: ''
+ }
],
headers: {
- 'x-powered-by': [
- 'test',
- ],
+ 'x-powered-by': ['test']
},
html: ' html test v1',
- scripts: [
- 'test',
- ],
+ scripts: ['test'],
js: {
appJs: {
- key: [
- 'value',
- ],
- },
- },
-};
+ key: ['value']
+ }
+ }
+}
describe('Wappalyzer', () => {
describe('#analyze()', () => {
- let apps;
+ let apps
before(async () => {
- const wappalyzer = new Wappalyzer();
+ const wappalyzer = new Wappalyzer()
- wappalyzer.apps = appsJson;
+ wappalyzer.apps = appsJson
- wappalyzer.parseJsPatterns();
+ wappalyzer.parseJsPatterns()
wappalyzer.driver.displayApps = (detected) => {
- apps = detected;
- };
+ apps = detected
+ }
- await wappalyzer.analyze({ canonical: 'test' }, driverData);
- });
+ await wappalyzer.analyze({ canonical: 'test' }, driverData)
+ })
it('should identify technologies using URLs', () => {
- expect(apps).to.have.any.keys('appUrl');
- });
+ expect(apps).to.have.any.keys('appUrl')
+ })
it('should identify technologies using HTML', () => {
- expect(apps).to.have.any.keys('appHtml');
- });
+ expect(apps).to.have.any.keys('appHtml')
+ })
it('should identify technologies using meta tags', () => {
- expect(apps).to.have.any.keys('appMeta');
- });
+ expect(apps).to.have.any.keys('appMeta')
+ })
it('should identify technologies using script URLs', () => {
- expect(apps).to.have.any.keys('appScript');
- });
+ expect(apps).to.have.any.keys('appScript')
+ })
it('should identify technologies using headers', () => {
- expect(apps).to.have.any.keys('appHeaders');
- });
+ expect(apps).to.have.any.keys('appHeaders')
+ })
it('should identify technologies using cookies', () => {
- expect(apps).to.have.any.keys('appCookies');
- });
+ expect(apps).to.have.any.keys('appCookies')
+ })
it('should identify technologies using uppercase named cookies', () => {
- expect(apps).to.have.any.keys('appUppercaseCookies');
- });
+ expect(apps).to.have.any.keys('appUppercaseCookies')
+ })
it('should identify technologies using JavaScript', () => {
- expect(apps).to.have.any.keys('appJs');
- });
+ expect(apps).to.have.any.keys('appJs')
+ })
it('should return the implied technology', () => {
- expect(apps).to.have.any.keys('appImplies');
- });
+ expect(apps).to.have.any.keys('appImplies')
+ })
it('should not return the excluded technology', () => {
- expect(apps).to.not.have.any.keys('appExcludes');
- });
+ expect(apps).to.not.have.any.keys('appExcludes')
+ })
it('should return the confidence value', () => {
- assert.equal(apps.appHtml.confidenceTotal, 50);
- });
+ assert.equal(apps.appHtml.confidenceTotal, 50)
+ })
it('should return the version number', () => {
- assert.equal(apps.appHtml.version, '1');
- });
+ assert.equal(apps.appHtml.version, '1')
+ })
it('should analyze html', async () => {
const html = `
@@ -156,123 +149,125 @@ describe('Wappalyzer', () => {
-