Refactoring

src/drivers/webextension/css/popup.css

@@ -1,30 +1,47 @@
+:root {
+  --color-primary: #4608ad;
+  --color-secondary: #e0e0e0;
+  --color-text: #4a4a4a;
+}
+
 body {
 	background: #fff;
   direction: __MSG_@@bidi_dir__;
 	font-family: Helvetica, Arial, sans-serif;
-  font-size: .8rem;
+  font-size: .9rem;
+  line-height: 1.5rem;
 	margin: 0;
   min-width: 30rem;
 }
 
+a {
+  color: var(--color-primary);
+  outline: none;
+  text-decoration: none;
+}
+
+a:focus {
+  outline: none;
+}
+
+a:hover {
+  text-decoration: underline;
+}
+
 .header {
   align-items: center;
-  border-bottom: 1px solid #dbdbdb;
+  border-bottom: 1px solid var(--color-secondary);
   height: 4rem;
   display: flex;
 }
 
-.header__link:focus {
-  outline: none;
-}
-
 .header__logo {
   display: inline-block;
   margin: .2rem 1.5rem 0 1.5rem;
   -webkit-backface-visibility: hidden;
   -webkit-transform: translateZ(0) scale(1.0, 1.0);
   transform: translateZ(0);
-  height: 2rem;
+  height: 2.5rem;
 }
 
 .header__logo--dark {
@@ -33,144 +50,37 @@ body {
 
 .footer {
   align-items: center;
-  border-top: 1px solid #dbdbdb;
+  border-top: 1px solid var(--color-secondary);
   height: 3rem;
   display: flex;
   padding: 0 1.5rem;
 }
 
-.footer__link {
+.detections {
-  color: #4608ad;
-  text-decoration: none;
-}
-
-.footer__link:hover, .footer__link:active {
-  color: #4608ad;
-  text-decoration: underline;
-}
-
-.container {
-  min-height: 5rem;
-  padding: 1rem 1.5rem 0rem 1.5rem;
-}
-
-.detected {
   columns: 2;
   column-gap: 1.5rem;
-  line-height: 1.4rem;
+  padding: 1.5rem;
 }
 
-.detected__category {
+.category {
   page-break-inside: avoid;
   break-inside: avoid-column;
   padding-bottom: 1rem;
 }
 
-.detected__category-name {
+.category__link {
-  display: block;
-}
-
-.detected__category-link {
-  color: #4608ad;
   font-weight: bold;
   line-height: 2rem;
   text-decoration: none;
 }
 
-.detected__category-link:hover {
+.technology {
-  color: #4a4a4a;
-}
-
-.detected__category-pin-wrapper {
-  margin-left: .2rem;
-  margin-right: .2rem;
-}
-
-.detected__category-pin {
-  cursor: pointer;
-  display: none;
-  height: 16px;
-  margin-left: .2rem;
-  width: 16px;
-  vertical-align: middle;
-}
-
-.detected__category:hover .detected__category-pin--inactive {
-  display: inline-block;
-}
-
-.detected__category-pin-wrapper--active .detected__category-pin--inactive,
-.detected__category-pin-wrapper:hover .detected__category-pin--inactive {
-  display: none !important;
-}
-
-.detected__category-pin-wrapper--active .detected__category-pin--active,
-.detected__category-pin-wrapper:hover .detected__category-pin--active {
-  display: inline-block;
-}
-
-.detected__app {
-  color: #4a4a4a;
   display: block;
   line-height: 1.7rem;
-  text-decoration: none;
-}
-
-.detected__app:focus {
-  display: block;
-  outline: 0;
-}
-
-.detected__app-icon {
-	display: inline-block;
-	height: 16px;
-	margin-inline-end: .5rem;
-	vertical-align: -.2rem;
-	width: 16px;
-}
-
-.detected__app-name {
-}
-
-.detected__app-version, .detected__app-confidence {
-  background: #eee;
-  border-radius: 3px;
-  font-size: .7rem;
-  margin-left: .3rem;
-  padding: .1rem .2rem;
 }
 
-.detected__app:hover .detected__app-name {
+.technology__link {
-  border-bottom: 1px solid #4a4a4a;
+  color: var(--color-text)
-}
-
-.detected__app:hover .detected__app-version,
-.detected__app:hover .detected__app-confidence {
-  border-bottom: 1px solid white;
-}
-
-.detected-app {
-	padding: 7px 0;
-}
-
-.detected-app:first-child {
-	padding-top: 0;
-}
-
-.detected-app:last-child {
-	border: none;
-	padding-bottom: 0;
-}
-
-.empty {
-  display: flex;
-  height: 5rem;
-  margin-bottom: 1rem;
-  align-items: center;
-  justify-content: center;
-}
-
-.empty__text {
 }
 
 .terms {
@@ -216,72 +126,5 @@ body {
 }
 
 .terms__privacy {
-  color: #4608ad;
   margin-top: 1rem;
 }
-
-@media (prefers-color-scheme: dark) {
-  /* Add alternative color palette for Dark mode theme. */
-  body.theme-mode-sync  {
-    background: linear-gradient(160deg, #32067c, #150233);
-  }
-
-  .theme-mode-sync .header {
-    border-bottom: 1px solid rgba(255, 255, 255, .2);
-  }
-
-  .theme-mode-sync .header__logo--dark {
-    display: inline-block;
-  }
-
-  .theme-mode-sync .header__logo--light {
-    display: none;
-  }
-
-  .theme-mode-sync .footer {
-    border-top: 1px solid rgba(255, 255, 255, .2);
-  }
-
-  .theme-mode-sync .footer__link {
-    color: rgba(255, 255, 255, .8);
-  }
-
-  .theme-mode-sync .footer__link:hover, .theme-mode-sync .footer__link:active {
-    color: rgba(255, 255, 255, .8);
-  }
-
-  .theme-mode-sync .container {
-    color: white;
-  }
-
-  .theme-mode-sync .detected__category-link {
-    color: #fff;
-  }
-
-  .theme-mode-sync .detected__app {
-    color: rgba(255, 255, 255, .8);
-  }
-
-  .theme-mode-sync .detected__category-link:hover {
-    color: white;
-    border-bottom: 1px solid white;
-  }
-
-  .theme-mode-sync .detected__app-version, .theme-mode-sync .detected__app-confidence {
-    background-color: #4608ad;
-  }
-
-  .theme-mode-sync .detected__app:hover .detected__app-name {
-    border-bottom: 1px solid white;
-  }
-
-  .theme-mode-sync .detected__app:hover .theme-mode-sync .detected__app-version,
-  .theme-mode-sync .detected__app:hover .theme-mode-sync .detected__app-confidence {
-    border-bottom: none;
-  }
-
-  .theme-mode-sync .terms__accept,
-  .theme-mode-sync .terms__privacy {
-    color: white;
-  }
-}

src/drivers/webextension/html/background.html

@@ -4,8 +4,9 @@
 	<head>
 		<meta charset="utf-8">
 
-		<script src="../js/wappalyzer2.js"></script>
+		<script src="../js/wappalyzer.js"></script>
-		<script src="../js/driver2.js"></script>
+		<script src="../js/utils.js"></script>
+		<script src="../js/driver.js"></script>
 		<script src="../js/lib/network.js"></script>
 	</head>
 	<body>

src/drivers/webextension/html/options.html

@@ -12,7 +12,7 @@
     <link rel="stylesheet" href="../css/options.css">
 
     <script src="../node_modules/webextension-polyfill/dist/browser-polyfill.js"></script>
-    <script src="../js/wappalyzer.js"></script>
+    <script src="../js/utils.js"></script>
     <script src="../js/options.js"></script>
   </head>
 

src/drivers/webextension/html/popup.html

@@ -1,5 +1,4 @@
 <!DOCTYPE html>
-
 <html>
 	<head>
 		<meta charset="utf-8">
@@ -7,28 +6,35 @@
 
 		<link rel="stylesheet" href="../css/popup.css">
 
-		<script src="../node_modules/webextension-polyfill/dist/browser-polyfill.js"></script>
+		<script src="../js/utils.js"></script>
-		<script src="../js/lib/jsontodom.js"></script>
 		<script src="../js/popup.js"></script>
 	</head>
 	<body>
     <div class="header">
-      <a href="https://www.wappalyzer.com/" class="header__link" target="_blank">
+      <a href="https://www.wappalyzer.com/" class="header__link">
         <img alt="" class="header__logo header__logo--light" src="../images/logo-purple.svg">
         <img alt="" class="header__logo header__logo--dark" src="../images/logo-white.svg">
       </a>
     </div>
 
-    <div class="container">
+    <div class="terms">
-      <div class="terms__wrapper">
+      <div class="terms__content" data-i18n="termsContent"></div>
-        <div class="terms">
+
-          <div class="terms__content" data-i18n="termsContent"></div>
+      <button class="terms__accept" data-i18n="termsAccept">&nbsp;</button>
+
+      <a class="terms__privacy" href="https://www.wappalyzer.com/privacy" data-i18n="privacyPolicy"></a>
+    </div>
 
-          <button class="terms__accept" data-i18n="termsAccept" />
+    <div class="detections"></div>
+
+    <div data-template="category" class="category">
+      <a class="category__link" href="#"></a>
+
+      <div class="technologies"></div>
+    </div>
 
-          <a class="terms__privacy" href="https://www.wappalyzer.com/privacy" data-i18n="privacyPolicy"></a>
+    <div data-template="technology" class="technology">
-        </div>
+      <a class="technology__link" href="#"></a>
-      </div>
     </div>
 
     <div class="footer">

src/drivers/webextension/js/driver.bak.js

@@ -0,0 +1,445 @@
+/**
+ * WebExtension driver
+ */
+
+/* eslint-env browser */
+/* global browser, chrome, Wappalyzer */
+
+/** global: browser */
+/** global: chrome */
+/** global: fetch */
+/** global: Wappalyzer */
+
+const wappalyzer = new Wappalyzer()
+
+const tabCache = {}
+const robotsTxtQueue = {}
+
+let categoryOrder = []
+
+browser.tabs.onRemoved.addListener((tabId) => {
+  tabCache[tabId] = null
+})
+
+function userAgent() {
+  const url = chrome.extension.getURL('/')
+
+  if (url.startsWith('moz-')) {
+    return 'firefox'
+  }
+
+  if (url.startsWith('ms-browser')) {
+    return 'edge'
+  }
+
+  return 'chrome'
+}
+
+/**
+ * Get a value from localStorage
+ */
+function getOption(name, defaultValue = null) {
+  return new Promise(async (resolve, reject) => {
+    let value = defaultValue
+
+    try {
+      const option = await browser.storage.local.get(name)
+
+      if (option[name] !== undefined) {
+        value = option[name]
+      }
+    } catch (error) {
+      wappalyzer.log(error.message, 'driver', 'error')
+
+      return reject(error.message)
+    }
+
+    return resolve(value)
+  })
+}
+
+/**
+ * Set a value in localStorage
+ */
+function setOption(name, value) {
+  return new Promise(async (resolve, reject) => {
+    try {
+      await browser.storage.local.set({ [name]: value })
+    } catch (error) {
+      wappalyzer.log(error.message, 'driver', 'error')
+
+      return reject(error.message)
+    }
+
+    return resolve()
+  })
+}
+
+/**
+ * Open a tab
+ */
+function openTab(args) {
+  browser.tabs.create({
+    url: args.url,
+    active: args.background === undefined || !args.background
+  })
+}
+
+/**
+ * Make a POST request
+ */
+async function post(url, body) {
+  try {
+    const response = await fetch(url, {
+      method: 'POST',
+      body: JSON.stringify(body)
+    })
+
+    wappalyzer.log(`POST ${url}: ${response.status}`, 'driver')
+  } catch (error) {
+    wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error')
+  }
+}
+
+// Capture response headers
+browser.webRequest.onCompleted.addListener(
+  async (request) => {
+    const headers = {}
+
+    if (request.responseHeaders) {
+      const url = wappalyzer.parseUrl(request.url)
+
+      let tab
+
+      try {
+        ;[tab] = await browser.tabs.query({ url: [url.href] })
+      } catch (error) {
+        wappalyzer.log(error, 'driver', 'error')
+      }
+
+      if (tab) {
+        request.responseHeaders.forEach((header) => {
+          const name = header.name.toLowerCase()
+
+          headers[name] = headers[name] || []
+
+          headers[name].push(
+            (header.value || header.binaryValue || '').toString()
+          )
+        })
+
+        if (
+          headers['content-type'] &&
+          /\/x?html/.test(headers['content-type'][0])
+        ) {
+          wappalyzer.analyze(url, { headers }, { tab })
+        }
+      }
+    }
+  },
+  { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] },
+  ['responseHeaders']
+)
+
+browser.runtime.onConnect.addListener((port) => {
+  port.onMessage.addListener(async (message) => {
+    if (message.id === undefined) {
+      return
+    }
+
+    if (message.id !== 'log') {
+      wappalyzer.log(`Message from ${port.name}: ${message.id}`, 'driver')
+    }
+
+    const pinnedCategory = await getOption('pinnedCategory')
+
+    const url = wappalyzer.parseUrl(port.sender.tab ? port.sender.tab.url : '')
+
+    const cookies = await browser.cookies.getAll({
+      domain: `.${url.hostname}`
+    })
+
+    let response
+
+    switch (message.id) {
+      case 'log':
+        wappalyzer.log(message.subject, message.source)
+
+        break
+      case 'init':
+        wappalyzer.analyze(url, { cookies }, { tab: port.sender.tab })
+
+        break
+      case 'analyze':
+        if (message.subject.html) {
+          browser.i18n
+            .detectLanguage(message.subject.html)
+            .then(({ languages }) => {
+              const language = languages
+                .filter(({ percentage }) => percentage >= 75)
+                .map(({ language: lang }) => lang)[0]
+
+              message.subject.language = language
+
+              wappalyzer.analyze(url, message.subject, { tab: port.sender.tab })
+            })
+        } else {
+          wappalyzer.analyze(url, message.subject, { tab: port.sender.tab })
+        }
+
+        await setOption('hostnameCache', wappalyzer.hostnameCache)
+
+        break
+      case 'ad_log':
+        wappalyzer.cacheDetectedAds(message.subject)
+
+        break
+      case 'get_apps':
+        response = {
+          tabCache: tabCache[message.tab.id],
+          apps: wappalyzer.apps,
+          categories: wappalyzer.categories,
+          pinnedCategory,
+          termsAccepted:
+            userAgent() === 'chrome' ||
+            (await getOption('termsAccepted', false))
+        }
+
+        break
+      case 'set_option':
+        await setOption(message.key, message.value)
+
+        break
+      case 'get_js_patterns':
+        response = {
+          patterns: wappalyzer.jsPatterns
+        }
+
+        break
+      case 'update_theme_mode':
+        // Sync theme mode to popup.
+        response = {
+          themeMode: await getOption('themeMode', false)
+        }
+
+        break
+      default:
+      // Do nothing
+    }
+
+    if (response) {
+      port.postMessage({
+        id: message.id,
+        response
+      })
+    }
+  })
+})
+
+wappalyzer.driver.document = document
+
+/**
+ * Log messages to console
+ */
+wappalyzer.driver.log = (message, source, type) => {
+  const log = ['warn', 'error'].includes(type) ? type : 'log'
+
+  console[log](`[wappalyzer ${type}]`, `[${source}]`, message) // eslint-disable-line no-console
+}
+
+/**
+ * Display apps
+ */
+wappalyzer.driver.displayApps = async (detected, meta, context) => {
+  const { tab } = context
+
+  if (tab === undefined) {
+    return
+  }
+
+  tabCache[tab.id] = tabCache[tab.id] || {
+    detected: []
+  }
+
+  tabCache[tab.id].detected = detected
+
+  const pinnedCategory = await getOption('pinnedCategory')
+  const dynamicIcon = await getOption('dynamicIcon', true)
+
+  let found = false
+
+  // Find the main application to display
+  ;[pinnedCategory].concat(categoryOrder).forEach((match) => {
+    Object.keys(detected).forEach((appName) => {
+      const app = detected[appName]
+
+      app.props.cats.forEach((category) => {
+        if (category === match && !found) {
+          let icon =
+            app.props.icon && dynamicIcon ? app.props.icon : 'default.svg'
+
+          if (/\.svg$/i.test(icon)) {
+            icon = `converted/${icon.replace(/\.svg$/, '.png')}`
+          }
+
+          try {
+            browser.pageAction.setIcon({
+              tabId: tab.id,
+              path: `../images/icons/${icon}`
+            })
+          } catch (e) {
+            // Firefox for Android does not support setIcon see https://bugzilla.mozilla.org/show_bug.cgi?id=1331746
+          }
+
+          found = true
+        }
+      })
+    })
+  })
+
+  browser.pageAction.show(tab.id)
+}
+
+/**
+ * Fetch and cache robots.txt for host
+ */
+wappalyzer.driver.getRobotsTxt = async (host, secure = false) => {
+  if (robotsTxtQueue[host]) {
+    return robotsTxtQueue[host]
+  }
+
+  const tracking = await getOption('tracking', true)
+  const robotsTxtCache = await getOption('robotsTxtCache', {})
+
+  robotsTxtQueue[host] = new Promise(async (resolve) => {
+    if (!tracking) {
+      return resolve([])
+    }
+
+    if (host in robotsTxtCache) {
+      return resolve(robotsTxtCache[host])
+    }
+
+    const timeout = setTimeout(() => resolve([]), 3000)
+
+    let response
+
+    try {
+      response = await fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, {
+        redirect: 'follow',
+        mode: 'no-cors'
+      })
+    } catch (error) {
+      wappalyzer.log(error, 'driver', 'error')
+
+      return resolve([])
+    }
+
+    clearTimeout(timeout)
+
+    const robotsTxt = response.ok ? await response.text() : ''
+
+    robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt)
+
+    await setOption('robotsTxtCache', robotsTxtCache)
+
+    delete robotsTxtQueue[host]
+
+    return resolve(robotsTxtCache[host])
+  })
+
+  return robotsTxtQueue[host]
+}
+
+/**
+ * Anonymously track detected applications for research purposes
+ */
+wappalyzer.driver.ping = async (
+  hostnameCache = { expires: 0, hostnames: {} },
+  adCache = []
+) => {
+  const tracking = await getOption('tracking', true)
+  const termsAccepted =
+    userAgent() === 'chrome' || (await getOption('termsAccepted', false))
+
+  if (tracking && termsAccepted) {
+    if (
+      hostnameCache.hostnames &&
+      Object.keys(hostnameCache.hostnames).length
+    ) {
+      post('https://api.wappalyzer.com/ping/v1/', hostnameCache.hostnames)
+    }
+
+    if (adCache.length) {
+      post('https://ad.wappalyzer.com/log/wp/', adCache)
+    }
+
+    await setOption('robotsTxtCache', {})
+  }
+}
+
+// Init
+;(async () => {
+  // Technologies
+  try {
+    const response = await fetch('../apps.json')
+    const json = await response.json()
+
+    wappalyzer.apps = json.apps
+    wappalyzer.categories = json.categories
+  } catch (error) {
+    wappalyzer.log(`GET apps.json: ${error.message}`, 'driver', 'error')
+  }
+
+  wappalyzer.parseJsPatterns()
+
+  categoryOrder = Object.keys(wappalyzer.categories)
+    .map((categoryId) => parseInt(categoryId, 10))
+    .sort(
+      (a, b) =>
+        wappalyzer.categories[a].priority - wappalyzer.categories[b].priority
+    )
+
+  // Version check
+  const { version } = browser.runtime.getManifest()
+  const previousVersion = await getOption('version')
+  const upgradeMessage = await getOption('upgradeMessage', true)
+
+  if (previousVersion === null) {
+    openTab({
+      url: `${wappalyzer.config.websiteURL}installed`
+    })
+  } else if (version !== previousVersion && upgradeMessage) {
+    openTab({
+      url: `${wappalyzer.config.websiteURL}upgraded?v${version}`,
+      background: true
+    })
+  }
+
+  await setOption('version', version)
+
+  // Hostname cache
+  wappalyzer.hostnameCache = await getOption('hostnameCache', {
+    expires: Date.now() + 1000 * 60 * 60 * 24,
+    hostnames: {}
+  })
+
+  // Run content script on all tabs
+  try {
+    const tabs = await browser.tabs.query({
+      url: ['http://*/*', 'https://*/*']
+    })
+
+    tabs.forEach(async (tab) => {
+      try {
+        await browser.tabs.executeScript(tab.id, {
+          file: '../js/content.js'
+        })
+      } catch (error) {
+        //
+      }
+    })
+  } catch (error) {
+    wappalyzer.log(error, 'driver', 'error')
+  }
+})()

src/drivers/webextension/js/driver.js

@@ -1,445 +1,269 @@
-/**
+'use strict'
- * WebExtension driver
- */
-
 /* eslint-env browser */
-/* global browser, chrome, Wappalyzer */
+/* globals chrome, Wappalyzer, Utils */
-
-/** global: browser */
-/** global: chrome */
-/** global: fetch */
-/** global: Wappalyzer */
-
-const wappalyzer = new Wappalyzer()
-
-const tabCache = {}
-const robotsTxtQueue = {}
 
-let categoryOrder = []
+const { setTechnologies, setCategories, analyze, resolve, unique } = Wappalyzer
+const { promisify, getOption } = Utils
 
-browser.tabs.onRemoved.addListener((tabId) => {
+const Driver = {
-  tabCache[tabId] = null
+  cache: {
-})
+    hostnames: {},
-
+    robots: {}
-function userAgent() {
+  },
-  const url = chrome.extension.getURL('/')
 
-  if (url.startsWith('moz-')) {
+  log(message, source = 'driver', type = 'log') {
-    return 'firefox'
+    // eslint-disable-next-line no-console
-  }
+    console[type](`wappalyzer | ${source} |`, message)
+  },
 
-  if (url.startsWith('ms-browser')) {
+  warn(message, source = 'driver') {
-    return 'edge'
+    Driver.log(message, source, 'warn')
-  }
+  },
 
-  return 'chrome'
+  error(error, source = 'driver') {
-}
+    Driver.log(error, source, 'error')
+  },
 
-/**
+  open(url, active = true) {
- * Get a value from localStorage
+    chrome.tabs.create({ url, active })
- */
+  },
-function getOption(name, defaultValue = null) {
-  return new Promise(async (resolve, reject) => {
-    let value = defaultValue
 
+  async loadTechnologies() {
     try {
-      const option = await browser.storage.local.get(name)
+      const { apps: technologies, categories } = await (
+        await fetch(chrome.extension.getURL('apps.json'))
+      ).json()
 
-      if (option[name] !== undefined) {
+      setTechnologies(technologies)
-        value = option[name]
+      setCategories(categories)
-      }
     } catch (error) {
-      wappalyzer.log(error.message, 'driver', 'error')
+      Driver.error(error)
-
-      return reject(error.message)
     }
+  },
 
-    return resolve(value)
+  post(url, body) {
-  })
-}
-
-/**
- * Set a value in localStorage
- */
-function setOption(name, value) {
-  return new Promise(async (resolve, reject) => {
     try {
-      await browser.storage.local.set({ [name]: value })
+      return fetch(url, {
+        method: 'POST',
+        body: JSON.stringify(body)
+      })
     } catch (error) {
-      wappalyzer.log(error.message, 'driver', 'error')
+      throw new Error(error.message || error.toString())
-
-      return reject(error.message)
-    }
-
-    return resolve()
-  })
-}
-
-/**
- * Open a tab
- */
-function openTab(args) {
-  browser.tabs.create({
-    url: args.url,
-    active: args.background === undefined || !args.background
-  })
-}
-
-/**
- * Make a POST request
- */
-async function post(url, body) {
-  try {
-    const response = await fetch(url, {
-      method: 'POST',
-      body: JSON.stringify(body)
-    })
-
-    wappalyzer.log(`POST ${url}: ${response.status}`, 'driver')
-  } catch (error) {
-    wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error')
-  }
-}
-
-// Capture response headers
-browser.webRequest.onCompleted.addListener(
-  async (request) => {
-    const headers = {}
-
-    if (request.responseHeaders) {
-      const url = wappalyzer.parseUrl(request.url)
-
-      let tab
-
-      try {
-        ;[tab] = await browser.tabs.query({ url: [url.href] })
-      } catch (error) {
-        wappalyzer.log(error, 'driver', 'error')
-      }
-
-      if (tab) {
-        request.responseHeaders.forEach((header) => {
-          const name = header.name.toLowerCase()
-
-          headers[name] = headers[name] || []
-
-          headers[name].push(
-            (header.value || header.binaryValue || '').toString()
-          )
-        })
-
-        if (
-          headers['content-type'] &&
-          /\/x?html/.test(headers['content-type'][0])
-        ) {
-          wappalyzer.analyze(url, { headers }, { tab })
-        }
-      }
     }
   },
-  { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] },
-  ['responseHeaders']
-)
-
-browser.runtime.onConnect.addListener((port) => {
-  port.onMessage.addListener(async (message) => {
-    if (message.id === undefined) {
-      return
-    }
-
-    if (message.id !== 'log') {
-      wappalyzer.log(`Message from ${port.name}: ${message.id}`, 'driver')
-    }
 
-    const pinnedCategory = await getOption('pinnedCategory')
+  onRuntimeConnect(port) {
-
+    port.onMessage.addListener(async ({ func, args }) => {
-    const url = wappalyzer.parseUrl(port.sender.tab ? port.sender.tab.url : '')
+      if (!func) {
-
+        return
-    const cookies = await browser.cookies.getAll({
+      }
-      domain: `.${url.hostname}`
-    })
-
-    let response
-
-    switch (message.id) {
-      case 'log':
-        wappalyzer.log(message.subject, message.source)
-
-        break
-      case 'init':
-        wappalyzer.analyze(url, { cookies }, { tab: port.sender.tab })
-
-        break
-      case 'analyze':
-        if (message.subject.html) {
-          browser.i18n
-            .detectLanguage(message.subject.html)
-            .then(({ languages }) => {
-              const language = languages
-                .filter(({ percentage }) => percentage >= 75)
-                .map(({ language: lang }) => lang)[0]
-
-              message.subject.language = language
-
-              wappalyzer.analyze(url, message.subject, { tab: port.sender.tab })
-            })
-        } else {
-          wappalyzer.analyze(url, message.subject, { tab: port.sender.tab })
-        }
-
-        await setOption('hostnameCache', wappalyzer.hostnameCache)
-
-        break
-      case 'ad_log':
-        wappalyzer.cacheDetectedAds(message.subject)
-
-        break
-      case 'get_apps':
-        response = {
-          tabCache: tabCache[message.tab.id],
-          apps: wappalyzer.apps,
-          categories: wappalyzer.categories,
-          pinnedCategory,
-          termsAccepted:
-            userAgent() === 'chrome' ||
-            (await getOption('termsAccepted', false))
-        }
-
-        break
-      case 'set_option':
-        await setOption(message.key, message.value)
-
-        break
-      case 'get_js_patterns':
-        response = {
-          patterns: wappalyzer.jsPatterns
-        }
-
-        break
-      case 'update_theme_mode':
-        // Sync theme mode to popup.
-        response = {
-          themeMode: await getOption('themeMode', false)
-        }
 
-        break
+      Driver.log({ port: port.name, func, args })
-      default:
-      // Do nothing
-    }
 
-    if (response) {
       port.postMessage({
-        id: message.id,
+        func,
-        response
+        args: await Driver[func].call(port.sender, ...(args || []))
       })
-    }
-  })
-})
-
-wappalyzer.driver.document = document
-
-/**
- * Log messages to console
- */
-wappalyzer.driver.log = (message, source, type) => {
-  const log = ['warn', 'error'].includes(type) ? type : 'log'
-
-  console[log](`[wappalyzer ${type}]`, `[${source}]`, message) // eslint-disable-line no-console
-}
-
-/**
- * Display apps
- */
-wappalyzer.driver.displayApps = async (detected, meta, context) => {
-  const { tab } = context
-
-  if (tab === undefined) {
-    return
-  }
 
-  tabCache[tab.id] = tabCache[tab.id] || {
+      /*
-    detected: []
+      const pinnedCategory = await getOption('pinnedCategory')
-  }
 
-  tabCache[tab.id].detected = detected
+      const url = new URL(port.sender.tab.url)
 
-  const pinnedCategory = await getOption('pinnedCategory')
+      const cookies = await browser.cookies.getAll({
-  const dynamicIcon = await getOption('dynamicIcon', true)
+        domain: `.${url.hostname}`
+      })
 
-  let found = false
+      let response
+
+      switch (message.id) {
+        case 'log':
+          wappalyzer.log(message.subject, message.source)
+
+          break
+        case 'analyze':
+          if (message.subject.html) {
+            browser.i18n
+              .detectLanguage(message.subject.html)
+              .then(({ languages }) => {
+                const language = languages
+                  .filter(({ percentage }) => percentage >= 75)
+                  .map(({ language: lang }) => lang)[0]
+
+                message.subject.language = language
+
+                wappalyzer.analyze(url, message.subject, {
+                  tab: port.sender.tab
+                })
+              })
+          } else {
+            wappalyzer.analyze(url, message.subject, { tab: port.sender.tab })
+          }
 
-  // Find the main application to display
+          await setOption('hostnameCache', wappalyzer.hostnameCache)
-  ;[pinnedCategory].concat(categoryOrder).forEach((match) => {
+
-    Object.keys(detected).forEach((appName) => {
+          break
-      const app = detected[appName]
+        case 'ad_log':
+          wappalyzer.cacheDetectedAds(message.subject)
+
+          break
+        case 'get_apps':
+          response = {
+            tabCache: tabCache[message.tab.id],
+            apps: wappalyzer.apps,
+            categories: wappalyzer.categories,
+            pinnedCategory,
+            termsAccepted:
+              userAgent() === 'chrome' ||
+              (await getOption('termsAccepted', false))
+          }
 
-      app.props.cats.forEach((category) => {
+          break
-        if (category === match && !found) {
+        case 'set_option':
-          let icon =
+          await setOption(message.key, message.value)
-            app.props.icon && dynamicIcon ? app.props.icon : 'default.svg'
 
-          if (/\.svg$/i.test(icon)) {
+          break
-            icon = `converted/${icon.replace(/\.svg$/, '.png')}`
+        case 'get_js_patterns':
+          response = {
+            patterns: wappalyzer.jsPatterns
           }
 
-          try {
+          break
-            browser.pageAction.setIcon({
+        case 'update_theme_mode':
-              tabId: tab.id,
+          // Sync theme mode to popup.
-              path: `../images/icons/${icon}`
+          response = {
-            })
+            themeMode: await getOption('themeMode', false)
-          } catch (e) {
-            // Firefox for Android does not support setIcon see https://bugzilla.mozilla.org/show_bug.cgi?id=1331746
           }
 
-          found = true
+          break
-        }
+        default:
-      })
+        // Do nothing
+      }
+
+      if (response) {
+        port.postMessage({
+          id: message.id,
+          response
+        })
+      }
     })
-  })
+    */
+    })
+  },
 
-  browser.pageAction.show(tab.id)
+  async onWebRequestComplete(request) {
-}
+    if (request.responseHeaders) {
+      const headers = {}
 
-/**
+      try {
- * Fetch and cache robots.txt for host
+        const url = new URL(request.url)
- */
-wappalyzer.driver.getRobotsTxt = async (host, secure = false) => {
-  if (robotsTxtQueue[host]) {
-    return robotsTxtQueue[host]
-  }
 
-  const tracking = await getOption('tracking', true)
+        const [tab] = await promisify(chrome.tabs, 'query', { url: [url.href] })
-  const robotsTxtCache = await getOption('robotsTxtCache', {})
 
-  robotsTxtQueue[host] = new Promise(async (resolve) => {
+        if (tab) {
-    if (!tracking) {
+          request.responseHeaders.forEach((header) => {
-      return resolve([])
+            const name = header.name.toLowerCase()
-    }
 
-    if (host in robotsTxtCache) {
+            headers[name] = headers[name] || []
-      return resolve(robotsTxtCache[host])
-    }
 
-    const timeout = setTimeout(() => resolve([]), 3000)
+            headers[name].push(
+              (header.value || header.binaryValue || '').toString()
+            )
+          })
 
-    let response
+          if (
+            headers['content-type'] &&
+            /\/x?html/.test(headers['content-type'][0])
+          ) {
+            await Driver.onDetect(url, await analyze(url, { headers }, { tab }))
+          }
+        }
+      } catch (error) {
+        Driver.error(error)
+      }
+    }
+  },
 
+  async onContentLoad(href, items) {
     try {
-      response = await fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, {
+      const url = new URL(href)
-        redirect: 'follow',
+
-        mode: 'no-cors'
+      items.cookies = await promisify(chrome.cookies, 'getAll', {
+        domain: `.${url.hostname}`
       })
-    } catch (error) {
-      wappalyzer.log(error, 'driver', 'error')
 
-      return resolve([])
+      await Driver.onDetect(url, await analyze(url, items))
+    } catch (error) {
+      Driver.error(error)
     }
+  },
 
-    clearTimeout(timeout)
+  async onDetect(url, detections = []) {
+    Driver.cache.hostnames[url.hostname] = unique([
+      ...(Driver.cache.hostnames[url.hostname] || []),
+      ...detections
+    ])
 
-    const robotsTxt = response.ok ? await response.text() : ''
+    const resolved = resolve(Driver.cache.hostnames[url.hostname])
 
-    robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt)
+    await Driver.setIcon(url, resolved)
+  },
 
-    await setOption('robotsTxtCache', robotsTxtCache)
+  async setIcon(url, technologies) {
+    const dynamicIcon = await getOption('dynamicIcon', true)
 
-    delete robotsTxtQueue[host]
+    let icon = 'default.svg'
 
-    return resolve(robotsTxtCache[host])
+    if (dynamicIcon) {
-  })
+      const pinnedCategory = parseInt(await getOption('pinnedCategory'), 10)
 
-  return robotsTxtQueue[host]
+      const pinned = technologies.find(({ categories }) =>
-}
+        categories.some(({ id }) => id === pinnedCategory)
+      )
 
-/**
+      ;({ icon } =
- * Anonymously track detected applications for research purposes
+        pinned ||
- */
+        technologies.sort(({ categories: a }, { categories: b }) => {
-wappalyzer.driver.ping = async (
+          const max = (value) =>
-  hostnameCache = { expires: 0, hostnames: {} },
+            value.reduce((max, { priority }) => Math.max(max, priority))
-  adCache = []
-) => {
-  const tracking = await getOption('tracking', true)
-  const termsAccepted =
-    userAgent() === 'chrome' || (await getOption('termsAccepted', false))
-
-  if (tracking && termsAccepted) {
-    if (
-      hostnameCache.hostnames &&
-      Object.keys(hostnameCache.hostnames).length
-    ) {
-      post('https://api.wappalyzer.com/ping/v1/', hostnameCache.hostnames)
-    }
 
-    if (adCache.length) {
+          return max(a) > max(b) ? -1 : 1
-      post('https://ad.wappalyzer.com/log/wp/', adCache)
+        })[0])
     }
 
-    await setOption('robotsTxtCache', {})
+    const tabs = await promisify(chrome.tabs, 'query', { url: [url.href] })
-  }
-}
-
-// Init
-;(async () => {
-  // Technologies
-  try {
-    const response = await fetch('../apps.json')
-    const json = await response.json()
-
-    wappalyzer.apps = json.apps
-    wappalyzer.categories = json.categories
-  } catch (error) {
-    wappalyzer.log(`GET apps.json: ${error.message}`, 'driver', 'error')
-  }
 
-  wappalyzer.parseJsPatterns()
+    await Promise.all(
+      tabs.map(async ({ id: tabId }) => {
+        await promisify(chrome.pageAction, 'setIcon', {
+          tabId,
+          path: chrome.extension.getURL(`../images/icons/${icon}`)
+        })
 
-  categoryOrder = Object.keys(wappalyzer.categories)
+        chrome.pageAction.show(tabId)
-    .map((categoryId) => parseInt(categoryId, 10))
+      })
-    .sort(
-      (a, b) =>
-        wappalyzer.categories[a].priority - wappalyzer.categories[b].priority
     )
+  },
 
-  // Version check
+  async getDetections() {
-  const { version } = browser.runtime.getManifest()
+    const [{ url: href }] = await promisify(chrome.tabs, 'query', {
-  const previousVersion = await getOption('version')
+      active: true,
-  const upgradeMessage = await getOption('upgradeMessage', true)
+      currentWindow: true
-
-  if (previousVersion === null) {
-    openTab({
-      url: `${wappalyzer.config.websiteURL}installed`
     })
-  } else if (version !== previousVersion && upgradeMessage) {
-    openTab({
-      url: `${wappalyzer.config.websiteURL}upgraded?v${version}`,
-      background: true
-    })
-  }
 
-  await setOption('version', version)
+    const url = new URL(href)
 
-  // Hostname cache
+    return resolve(Driver.cache.hostnames[url.hostname])
-  wappalyzer.hostnameCache = await getOption('hostnameCache', {
+  }
-    expires: Date.now() + 1000 * 60 * 60 * 24,
+}
-    hostnames: {}
-  })
 
-  // Run content script on all tabs
+;(async function() {
-  try {
+  await Driver.loadTechnologies()
-    const tabs = await browser.tabs.query({
-      url: ['http://*/*', 'https://*/*']
-    })
 
-    tabs.forEach(async (tab) => {
+  chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect)
-      try {
+  chrome.webRequest.onCompleted.addListener(
-        await browser.tabs.executeScript(tab.id, {
+    Driver.onWebRequestComplete,
-          file: '../js/content.js'
+    { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] },
-        })
+    ['responseHeaders']
-      } catch (error) {
+  )
-        //
-      }
-    })
-  } catch (error) {
-    wappalyzer.log(error, 'driver', 'error')
-  }
 })()

src/drivers/webextension/js/driver2.js

@@ -1,285 +0,0 @@
-'use strict'
-/* eslint-env browser */
-/* globals chrome, Wappalyzer */
-
-const { setTechnologies, setCategories, analyze, resolve, unique } = Wappalyzer
-
-function promisify(context, method, ...args) {
-  return new Promise((resolve, reject) => {
-    context[method](...args, (...args) => {
-      if (chrome.runtime.lastError) {
-        return reject(chrome.runtime.lastError)
-      }
-
-      resolve(...args)
-    })
-  })
-}
-
-const Driver = {
-  cache: {
-    hostnames: {},
-    robots: {}
-  },
-
-  agent: chrome.extension.getURL('/').startsWith('moz-') ? 'firefox' : 'chrome',
-
-  log(message, source = 'driver', type = 'log') {
-    // eslint-disable-next-line no-console
-    console[type](`wappalyzer | ${source} |`, message)
-  },
-
-  warn(message, source = 'driver') {
-    Driver.log(message, source, 'warn')
-  },
-
-  error(error, source = 'driver') {
-    Driver.log(error, source, 'error')
-  },
-
-  open(url, active = true) {
-    chrome.tabs.create({ url, active })
-  },
-
-  async loadTechnologies() {
-    try {
-      const { apps: technologies, categories } = await (
-        await fetch(chrome.extension.getURL('apps.json'))
-      ).json()
-
-      setTechnologies(technologies)
-      setCategories(categories)
-    } catch (error) {
-      Driver.error(error)
-    }
-  },
-
-  post(url, body) {
-    try {
-      return fetch(url, {
-        method: 'POST',
-        body: JSON.stringify(body)
-      })
-    } catch (error) {
-      throw new Error(error.message || error.toString())
-    }
-  },
-
-  async getOption(name, defaultValue = null) {
-    try {
-      const option = await promisify(chrome.storage.local, 'get', name)
-
-      if (option[name] !== undefined) {
-        return option[name]
-      }
-
-      return defaultValue
-    } catch (error) {
-      throw new Error(error.message || error.toString())
-    }
-  },
-
-  async setOption(name, value) {
-    try {
-      await promisify(chrome.storage.local, 'set', {
-        [name]: value
-      })
-    } catch (error) {
-      throw new Error(error.message || error.toString())
-    }
-  },
-
-  onRuntimeConnect(port) {
-    port.onMessage.addListener(async (message) => {
-      const { func, args } = message
-
-      if (!func || !port.sender.tab) {
-        return
-      }
-
-      Driver.log(`Message received from ${port.name}: ${func}`)
-
-      await Driver[func](...args)
-
-      /*
-      const pinnedCategory = await getOption('pinnedCategory')
-
-      const url = new URL(port.sender.tab.url)
-
-      const cookies = await browser.cookies.getAll({
-        domain: `.${url.hostname}`
-      })
-
-      let response
-
-      switch (message.id) {
-        case 'log':
-          wappalyzer.log(message.subject, message.source)
-
-          break
-        case 'analyze':
-          if (message.subject.html) {
-            browser.i18n
-              .detectLanguage(message.subject.html)
-              .then(({ languages }) => {
-                const language = languages
-                  .filter(({ percentage }) => percentage >= 75)
-                  .map(({ language: lang }) => lang)[0]
-
-                message.subject.language = language
-
-                wappalyzer.analyze(url, message.subject, {
-                  tab: port.sender.tab
-                })
-              })
-          } else {
-            wappalyzer.analyze(url, message.subject, { tab: port.sender.tab })
-          }
-
-          await setOption('hostnameCache', wappalyzer.hostnameCache)
-
-          break
-        case 'ad_log':
-          wappalyzer.cacheDetectedAds(message.subject)
-
-          break
-        case 'get_apps':
-          response = {
-            tabCache: tabCache[message.tab.id],
-            apps: wappalyzer.apps,
-            categories: wappalyzer.categories,
-            pinnedCategory,
-            termsAccepted:
-              userAgent() === 'chrome' ||
-              (await getOption('termsAccepted', false))
-          }
-
-          break
-        case 'set_option':
-          await setOption(message.key, message.value)
-
-          break
-        case 'get_js_patterns':
-          response = {
-            patterns: wappalyzer.jsPatterns
-          }
-
-          break
-        case 'update_theme_mode':
-          // Sync theme mode to popup.
-          response = {
-            themeMode: await getOption('themeMode', false)
-          }
-
-          break
-        default:
-        // Do nothing
-      }
-
-      if (response) {
-        port.postMessage({
-          id: message.id,
-          response
-        })
-      }
-    })
-    */
-    })
-  },
-
-  async onWebRequestComplete(request) {
-    if (request.responseHeaders) {
-      const headers = {}
-
-      try {
-        const url = new URL(request.url)
-
-        const [tab] = await promisify(chrome.tabs, 'query', { url: [url.href] })
-
-        if (tab) {
-          request.responseHeaders.forEach((header) => {
-            const name = header.name.toLowerCase()
-
-            headers[name] = headers[name] || []
-
-            headers[name].push(
-              (header.value || header.binaryValue || '').toString()
-            )
-          })
-
-          if (
-            headers['content-type'] &&
-            /\/x?html/.test(headers['content-type'][0])
-          ) {
-            await Driver.onDetect(url, await analyze(url, { headers }, { tab }))
-          }
-        }
-      } catch (error) {
-        Driver.error(error)
-      }
-    }
-  },
-
-  async onContentLoad(href, items) {
-    try {
-      const url = new URL(href)
-
-      items.cookies = await promisify(chrome.cookies, 'getAll', {
-        domain: `.${url.hostname}`
-      })
-
-      await Driver.onDetect(url, await analyze(url, items))
-    } catch (error) {
-      Driver.error(error)
-    }
-  },
-
-  async onDetect(url, detections = []) {
-    Driver.cache.hostnames[url.hostname] = unique([
-      ...(Driver.cache.hostnames[url.hostname] || []),
-      ...detections
-    ])
-
-    const resolved = resolve(Driver.cache.hostnames[url.hostname])
-
-    const pinnedCategory = parseInt(
-      await Driver.getOption('pinnedCategory'),
-      10
-    )
-
-    const pinned = resolved.find(({ categories }) =>
-      categories.some(({ id }) => id === pinnedCategory)
-    )
-
-    const { icon } =
-      pinned ||
-      resolved.sort(({ categories: a }, { categories: b }) => {
-        const max = (value) =>
-          value.reduce((max, { priority }) => Math.max(max, priority))
-
-        return max(a) > max(b) ? -1 : 1
-      })[0]
-
-    const tabs = await promisify(chrome.tabs, 'query', { url: [url.href] })
-
-    await Promise.all(
-      tabs.map(({ id: tabId }) =>
-        promisify(chrome.pageAction, 'setIcon', {
-          tabId,
-          path: chrome.extension.getURL(`../images/icons/${icon}`)
-        })
-      )
-    )
-  }
-}
-
-;(async function() {
-  await Driver.loadTechnologies()
-
-  chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect)
-  chrome.webRequest.onCompleted.addListener(
-    Driver.onWebRequestComplete,
-    { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] },
-    ['responseHeaders']
-  )
-})()

src/drivers/webextension/js/popup.bak.js

@@ -0,0 +1,333 @@
+'use strict'
+/* eslint-env browser */
+/* globals chrome */
+
+let pinnedCategory = null
+let termsAccepted = false
+
+const port = chrome.runtime.connect({
+  name: 'popup.js'
+})
+
+function slugify(string) {
+  return string
+    .toLowerCase()
+    .replace(/[^a-z0-9-]/g, '-')
+    .replace(/--+/g, '-')
+    .replace(/(?:^-|-$)/, '')
+}
+
+function i18n() {
+  const nodes = document.querySelectorAll('[data-i18n]')
+
+  Array.prototype.forEach.call(nodes, (node) => {
+    node.innerHTML = browser.i18n.getMessage(node.dataset.i18n)
+  })
+}
+
+function replaceDom(domTemplate) {
+  const container = document.getElementsByClassName('container')[0]
+
+  while (container.firstChild) {
+    container.removeChild(container.firstChild)
+  }
+
+  container.appendChild(jsonToDOM(domTemplate, document, {}))
+
+  i18n()
+
+  Array.from(
+    document.querySelectorAll('.detected__category-pin-wrapper')
+  ).forEach((pin) => {
+    pin.addEventListener('click', () => {
+      const categoryId = parseInt(pin.dataset.categoryId, 10)
+
+      if (categoryId === pinnedCategory) {
+        pin.className = 'detected__category-pin-wrapper'
+
+        pinnedCategory = null
+      } else {
+        const active = document.querySelector(
+          '.detected__category-pin-wrapper--active'
+        )
+
+        if (active) {
+          active.className = 'detected__category-pin-wrapper'
+        }
+
+        pin.className =
+          'detected__category-pin-wrapper detected__category-pin-wrapper--active'
+
+        pinnedCategory = categoryId
+      }
+
+      port.postMessage({
+        id: 'set_option',
+        key: 'pinnedCategory',
+        value: pinnedCategory
+      })
+    })
+  })
+
+  Array.from(document.querySelectorAll('a')).forEach((link) => {
+    link.addEventListener('click', () => {
+      browser.tabs.create({ url: link.href })
+
+      return false
+    })
+  })
+}
+
+function replaceDomWhenReady(dom) {
+  if (/complete|interactive|loaded/.test(document.readyState)) {
+    replaceDom(dom)
+  } else {
+    document.addEventListener('DOMContentLoaded', () => {
+      replaceDom(dom)
+    })
+  }
+}
+
+function appsToDomTemplate(response) {
+  let template = []
+
+  if (response.tabCache && Object.keys(response.tabCache.detected).length > 0) {
+    const categories = {}
+
+    // Group apps by category
+    for (const appName in response.tabCache.detected) {
+      response.apps[appName].cats.forEach((cat) => {
+        categories[cat] = categories[cat] || {
+          name: response.categories[cat].name,
+          apps: []
+        }
+
+        categories[cat].apps[appName] = appName
+      })
+    }
+
+    for (const cat in categories) {
+      const apps = []
+
+      for (const appName in categories[cat].apps) {
+        const { confidenceTotal, version } = response.tabCache.detected[appName]
+
+        apps.push([
+          'a',
+          {
+            class: 'detected__app',
+            href: `https://www.wappalyzer.com/technologies/${slugify(
+              categories[cat].name
+            )}/${slugify(appName)}`
+          },
+          [
+            'img',
+            {
+              class: 'detected__app-icon',
+              src: `../images/icons/${response.apps[appName].icon ||
+                'default.svg'}`
+            }
+          ],
+          [
+            'span',
+            {
+              class: 'detected__app-name'
+            },
+            appName
+          ],
+          version
+            ? [
+                'span',
+                {
+                  class: 'detected__app-version'
+                },
+                version
+              ]
+            : null,
+          confidenceTotal < 100
+            ? [
+                'span',
+                {
+                  class: 'detected__app-confidence'
+                },
+                `${confidenceTotal}% sure`
+              ]
+            : null
+        ])
+      }
+
+      template.push([
+        'div',
+        {
+          class: 'detected__category'
+        },
+        [
+          'div',
+          {
+            class: 'detected__category-name'
+          },
+          [
+            'a',
+            {
+              class: 'detected__category-link',
+              href: `https://www.wappalyzer.com/categories/${slugify(
+                response.categories[cat].name
+              )}`
+            },
+            browser.i18n.getMessage(`categoryName${cat}`)
+          ],
+          [
+            'span',
+            {
+              class: `detected__category-pin-wrapper${
+                parseInt(pinnedCategory, 10) === parseInt(cat, 10)
+                  ? ' detected__category-pin-wrapper--active'
+                  : ''
+              }`,
+              'data-category-id': cat,
+              title: browser.i18n.getMessage('categoryPin')
+            },
+            [
+              'img',
+              {
+                class: 'detected__category-pin detected__category-pin--active',
+                src: '../images/pin-active.svg'
+              }
+            ],
+            [
+              'img',
+              {
+                class:
+                  'detected__category-pin detected__category-pin--inactive',
+                src: '../images/pin.svg'
+              }
+            ]
+          ]
+        ],
+        [
+          'div',
+          {
+            class: 'detected__apps'
+          },
+          apps
+        ]
+      ])
+    }
+
+    template = [
+      'div',
+      {
+        class: 'detected'
+      },
+      template
+    ]
+  } else {
+    template = [
+      'div',
+      {
+        class: 'empty'
+      },
+      [
+        'span',
+        {
+          class: 'empty__text'
+        },
+        browser.i18n.getMessage('noAppsDetected')
+      ]
+    ]
+  }
+
+  return template
+}
+
+async function getApps() {
+  try {
+    const tabs = await browser.tabs.query({
+      active: true,
+      currentWindow: true
+    })
+
+    const url = new URL(tabs[0].url)
+
+    document.querySelector(
+      '.footer__link'
+    ).href = `https://www.wappalyzer.com/alerts/manage?url=${encodeURIComponent(
+      `${url.protocol}//${url.hostname}`
+    )}`
+
+    port.postMessage({
+      id: 'get_apps',
+      tab: tabs[0]
+    })
+  } catch (error) {
+    console.error(error) // eslint-disable-line no-console
+  }
+}
+
+/**
+ * Async function to update body class based on option.
+ */
+function getThemeMode() {
+  try {
+    port.postMessage({
+      id: 'update_theme_mode'
+    })
+  } catch (error) {
+    console.error(error) // eslint-disable-line no-console
+  }
+}
+
+/**
+ * Update theme mode based on browser option.
+ * @param {object} res Response from port listener.
+ */
+function updateThemeMode(res) {
+  if (res.hasOwnProperty('themeMode') && res.themeMode !== false) {
+    document.body.classList.add('theme-mode-sync')
+  }
+}
+
+function displayApps(response) {
+  pinnedCategory = response.pinnedCategory // eslint-disable-line prefer-destructuring
+  termsAccepted = response.termsAccepted // eslint-disable-line prefer-destructuring
+
+  if (termsAccepted) {
+    replaceDomWhenReady(appsToDomTemplate(response))
+  } else {
+    i18n()
+
+    const wrapper = document.querySelector('.terms__wrapper')
+
+    document.querySelector('.terms__accept').addEventListener('click', () => {
+      port.postMessage({
+        id: 'set_option',
+        key: 'termsAccepted',
+        value: true
+      })
+
+      wrapper.classList.remove('terms__wrapper--active')
+
+      getApps()
+    })
+
+    wrapper.classList.add('terms__wrapper--active')
+  }
+}
+
+port.onMessage.addListener((message) => {
+  switch (message.id) {
+    case 'get_apps':
+      displayApps(message.response)
+
+      break
+    case 'update_theme_mode':
+      updateThemeMode(message.response)
+
+      break
+    default:
+    // Do nothing
+  }
+})
+
+getThemeMode()
+getApps()

src/drivers/webextension/js/popup.js

@@ -1,335 +1,110 @@
+'use strict'
 /* eslint-env browser */
-/* global browser, jsonToDOM */
+/* globals chrome, Utils */
 
-/** global: browser */
+const { agent, getOption } = Utils
-/** global: jsonToDOM */
 
-let pinnedCategory = null
+const Popup = {
-let termsAccepted = false
+  port: chrome.runtime.connect({ name: 'popup.js' }),
 
-const port = browser.runtime.connect({
+  async init() {
-  name: 'popup.js'
+    Popup.templates = Array.from(
-})
+      document.querySelectorAll('[data-template]')
-
+    ).reduce((templates, template) => {
-function slugify(string) {
+      templates[template.dataset.template] = template
-  return string
-    .toLowerCase()
-    .replace(/[^a-z0-9-]/g, '-')
-    .replace(/--+/g, '-')
-    .replace(/(?:^-|-$)/, '')
-}
-
-function i18n() {
-  const nodes = document.querySelectorAll('[data-i18n]')
-
-  Array.prototype.forEach.call(nodes, (node) => {
-    node.innerHTML = browser.i18n.getMessage(node.dataset.i18n)
-  })
-}
-
-function replaceDom(domTemplate) {
-  const container = document.getElementsByClassName('container')[0]
-
-  while (container.firstChild) {
-    container.removeChild(container.firstChild)
-  }
-
-  container.appendChild(jsonToDOM(domTemplate, document, {}))
-
-  i18n()
-
-  Array.from(
-    document.querySelectorAll('.detected__category-pin-wrapper')
-  ).forEach((pin) => {
-    pin.addEventListener('click', () => {
-      const categoryId = parseInt(pin.dataset.categoryId, 10)
-
-      if (categoryId === pinnedCategory) {
-        pin.className = 'detected__category-pin-wrapper'
-
-        pinnedCategory = null
-      } else {
-        const active = document.querySelector(
-          '.detected__category-pin-wrapper--active'
-        )
-
-        if (active) {
-          active.className = 'detected__category-pin-wrapper'
-        }
-
-        pin.className =
-          'detected__category-pin-wrapper detected__category-pin-wrapper--active'
-
-        pinnedCategory = categoryId
-      }
-
-      port.postMessage({
-        id: 'set_option',
-        key: 'pinnedCategory',
-        value: pinnedCategory
-      })
-    })
-  })
-
-  Array.from(document.querySelectorAll('a')).forEach((link) => {
-    link.addEventListener('click', () => {
-      browser.tabs.create({ url: link.href })
 
-      return false
+      return templates
-    })
+    }, {})
-  })
-}
-
-function replaceDomWhenReady(dom) {
-  if (/complete|interactive|loaded/.test(document.readyState)) {
-    replaceDom(dom)
-  } else {
-    document.addEventListener('DOMContentLoaded', () => {
-      replaceDom(dom)
-    })
-  }
-}
-
-function appsToDomTemplate(response) {
-  let template = []
 
-  if (response.tabCache && Object.keys(response.tabCache.detected).length > 0) {
+    Popup.log(agent)
-    const categories = {}
 
-    // Group apps by category
+    const termsAccepted =
-    for (const appName in response.tabCache.detected) {
+      agent === 'chrome' || (await getOption('termsAccepted', false))
-      response.apps[appName].cats.forEach((cat) => {
-        categories[cat] = categories[cat] || {
-          name: response.categories[cat].name,
-          apps: []
-        }
 
-        categories[cat].apps[appName] = appName
+    if (termsAccepted) {
-      })
+      document.querySelector('.terms').style.display = 'none'
-    }
-
-    for (const cat in categories) {
-      const apps = []
 
-      for (const appName in categories[cat].apps) {
+      Popup.driver('getDetections')
-        const { confidenceTotal, version } = response.tabCache.detected[appName]
+    } else {
+      document.querySelector('.detections').style.display = 'none'
 
-        apps.push([
+      Popup.i18n()
-          'a',
-          {
-            class: 'detected__app',
-            href: `https://www.wappalyzer.com/technologies/${slugify(
-              categories[cat].name
-            )}/${slugify(appName)}`
-          },
-          [
-            'img',
-            {
-              class: 'detected__app-icon',
-              src: `../images/icons/${response.apps[appName].icon ||
-                'default.svg'}`
-            }
-          ],
-          [
-            'span',
-            {
-              class: 'detected__app-name'
-            },
-            appName
-          ],
-          version
-            ? [
-                'span',
-                {
-                  class: 'detected__app-version'
-                },
-                version
-              ]
-            : null,
-          confidenceTotal < 100
-            ? [
-                'span',
-                {
-                  class: 'detected__app-confidence'
-                },
-                `${confidenceTotal}% sure`
-              ]
-            : null
-        ])
-      }
-
-      template.push([
-        'div',
-        {
-          class: 'detected__category'
-        },
-        [
-          'div',
-          {
-            class: 'detected__category-name'
-          },
-          [
-            'a',
-            {
-              class: 'detected__category-link',
-              href: `https://www.wappalyzer.com/categories/${slugify(
-                response.categories[cat].name
-              )}`
-            },
-            browser.i18n.getMessage(`categoryName${cat}`)
-          ],
-          [
-            'span',
-            {
-              class: `detected__category-pin-wrapper${
-                parseInt(pinnedCategory, 10) === parseInt(cat, 10)
-                  ? ' detected__category-pin-wrapper--active'
-                  : ''
-              }`,
-              'data-category-id': cat,
-              title: browser.i18n.getMessage('categoryPin')
-            },
-            [
-              'img',
-              {
-                class: 'detected__category-pin detected__category-pin--active',
-                src: '../images/pin-active.svg'
-              }
-            ],
-            [
-              'img',
-              {
-                class:
-                  'detected__category-pin detected__category-pin--inactive',
-                src: '../images/pin.svg'
-              }
-            ]
-          ]
-        ],
-        [
-          'div',
-          {
-            class: 'detected__apps'
-          },
-          apps
-        ]
-      ])
     }
+  },
 
-    template = [
+  driver(func, ...args) {
-      'div',
+    Popup.port.postMessage({ func, args })
-      {
+  },
-        class: 'detected'
-      },
-      template
-    ]
-  } else {
-    template = [
-      'div',
-      {
-        class: 'empty'
-      },
-      [
-        'span',
-        {
-          class: 'empty__text'
-        },
-        browser.i18n.getMessage('noAppsDetected')
-      ]
-    ]
-  }
 
-  return template
+  log(message) {
-}
+    Popup.driver('log', message, 'popup.js')
+  },
 
-async function getApps() {
+  i18n() {
-  try {
+    Array.from(document.querySelectorAll('[data-i18n]')).forEach(
-    const tabs = await browser.tabs.query({
+      (node) => (node.innerHTML = chrome.i18n.getMessage(node.dataset.i18n))
-      active: true,
+    )
-      currentWindow: true
+  },
-    })
 
-    const url = new URL(tabs[0].url)
+  categorise(technologies) {
+    return Object.values(
+      technologies.reduce((categories, technology) => {
+        technology.categories.forEach((category) => {
+          categories[category.id] = categories[category.id] || {
+            ...category,
+            technologies: []
+          }
 
-    document.querySelector(
+          categories[category.id].technologies.push(technology)
-      '.footer__link'
+        })
-    ).href = `https://www.wappalyzer.com/alerts/manage?url=${encodeURIComponent(
-      `${url.protocol}//${url.hostname}`
-    )}`
 
-    port.postMessage({
+        return categories
-      id: 'get_apps',
+      }, {})
-      tab: tabs[0]
+    )
-    })
+  },
-  } catch (error) {
-    console.error(error) // eslint-disable-line no-console
-  }
-}
 
-/**
+  onGetDetections(detections) {
- * Async function to update body class based on option.
+    Popup.categorise(detections).forEach(
- */
+      ({ name, slug: categorySlug, technologies }) => {
-function getThemeMode() {
+        const categoryNode = Popup.templates.category.cloneNode(true)
-  try {
-    port.postMessage({
-      id: 'update_theme_mode'
-    })
-  } catch (error) {
-    console.error(error) // eslint-disable-line no-console
-  }
-}
 
-/**
+        const link = categoryNode.querySelector('.category__link')
- * Update theme mode based on browser option.
- * @param {object} res Response from port listener.
- */
-function updateThemeMode(res) {
-  if (res.hasOwnProperty('themeMode') && res.themeMode !== false) {
-    document.body.classList.add('theme-mode-sync')
-  }
-}
 
-function displayApps(response) {
+        link.href = `https://www.wappalyzer.com/technologie/${categorySlug}`
-  pinnedCategory = response.pinnedCategory // eslint-disable-line prefer-destructuring
+        link.textContent = name
-  termsAccepted = response.termsAccepted // eslint-disable-line prefer-destructuring
 
-  if (termsAccepted) {
+        technologies.forEach(({ name, slug, icon, website }) => {
-    replaceDomWhenReady(appsToDomTemplate(response))
+          Popup.log(name)
-  } else {
+          const technologyNode = Popup.templates.technology.cloneNode(true)
-    i18n()
 
-    const wrapper = document.querySelector('.terms__wrapper')
+          const link = technologyNode.querySelector('.technology__link')
 
-    document.querySelector('.terms__accept').addEventListener('click', () => {
+          link.href = `https://www.wappalyzer.com/technologie/${categorySlug}/${slug}`
-      port.postMessage({
+          link.textContent = name
-        id: 'set_option',
-        key: 'termsAccepted',
-        value: true
-      })
 
-      wrapper.classList.remove('terms__wrapper--active')
+          categoryNode
+            .querySelector('.technologies')
+            .appendChild(technologyNode)
+        })
 
-      getApps()
+        document.querySelector('.detections').appendChild(categoryNode)
-    })
+      }
+    )
 
-    wrapper.classList.add('terms__wrapper--active')
+    Popup.i18n()
   }
 }
 
-port.onMessage.addListener((message) => {
+Popup.port.onMessage.addListener(({ func, args }) => {
-  switch (message.id) {
+  const onFunc = `on${func.charAt(0).toUpperCase() + func.slice(1)}`
-    case 'get_apps':
-      displayApps(message.response)
 
-      break
+  if (Popup[onFunc]) {
-    case 'update_theme_mode':
+    Popup[onFunc](args)
-      updateThemeMode(message.response)
-
-      break
-    default:
-    // Do nothing
   }
 })
 
-getThemeMode()
+if (/complete|interactive|loaded/.test(document.readyState)) {
-getApps()
+  Popup.init()
+} else {
+  document.addEventListener('DOMContentLoaded', Popup.init)
+}

src/drivers/webextension/js/utils.js

@@ -0,0 +1,47 @@
+'use strict'
+/* eslint-env browser */
+/* globals chrome */
+
+const Utils = {
+  agent: chrome.extension.getURL('/').startsWith('moz-') ? 'firefox' : 'chrome',
+
+  promisify(context, method, ...args) {
+    return new Promise((resolve, reject) => {
+      context[method](...args, (...args) => {
+        if (chrome.runtime.lastError) {
+          return reject(chrome.runtime.lastError)
+        }
+
+        resolve(...args)
+      })
+    })
+  },
+
+  open(url, active = true) {
+    chrome.tabs.create({ url, active })
+  },
+
+  async getOption(name, defaultValue = null) {
+    try {
+      const option = await Utils.promisify(chrome.storage.local, 'get', name)
+
+      if (option[name] !== undefined) {
+        return option[name]
+      }
+
+      return defaultValue
+    } catch (error) {
+      throw new Error(error.message || error.toString())
+    }
+  },
+
+  async setOption(name, value) {
+    try {
+      await Utils.promisify(chrome.storage.local, 'set', {
+        [name]: value
+      })
+    } catch (error) {
+      throw new Error(error.message || error.toString())
+    }
+  }
+}

src/drivers/webextension/js/wappalyzer.bak.js

@@ -0,0 +1,727 @@
+const validation = {
+  hostname: /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/,
+  hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/
+}
+
+/**
+ * Enclose string in array
+ */
+function asArray(value) {
+  return Array.isArray(value) ? value : [value]
+}
+
+/**
+ *
+ */
+function asyncForEach(iterable, iterator) {
+  return Promise.all(
+    (iterable || []).map(
+      (item) =>
+        new Promise((resolve) => setTimeout(() => resolve(iterator(item)), 1))
+    )
+  )
+}
+
+/**
+ * Mark application as detected, set confidence and version
+ */
+function addDetected(app, pattern, type, value, key) {
+  app.detected = true
+
+  // Set confidence level
+  app.confidence[`${type} ${key ? `${key} ` : ''}${pattern.regex}`] =
+    pattern.confidence === undefined ? 100 : parseInt(pattern.confidence, 10)
+
+  // Detect version number
+  if (pattern.version) {
+    const versions = []
+    const matches = pattern.regex.exec(value)
+
+    let { version } = pattern
+
+    if (matches) {
+      matches.forEach((match, i) => {
+        // Parse ternary operator
+        const ternary = new RegExp(`\\\\${i}\\?([^:]+):(.*)$`).exec(version)
+
+        if (ternary && ternary.length === 3) {
+          version = version.replace(ternary[0], match ? ternary[1] : ternary[2])
+        }
+
+        // Replace back references
+        version = version
+          .trim()
+          .replace(new RegExp(`\\\\${i}`, 'g'), match || '')
+      })
+
+      if (version && !versions.includes(version)) {
+        versions.push(version)
+      }
+
+      if (versions.length) {
+        // Use the longest detected version number
+        app.version = versions.reduce((a, b) => (a.length > b.length ? a : b))
+      }
+    }
+  }
+}
+
+function resolveExcludes(apps, detected) {
+  const excludes = []
+  const detectedApps = Object.assign({}, apps, detected)
+
+  // Exclude app in detected apps only
+  Object.keys(detectedApps).forEach((appName) => {
+    const app = detectedApps[appName]
+
+    if (app.props.excludes) {
+      asArray(app.props.excludes).forEach((excluded) => {
+        excludes.push(excluded)
+      })
+    }
+  })
+
+  // Remove excluded applications
+  Object.keys(apps).forEach((appName) => {
+    if (excludes.includes(appName)) {
+      delete apps[appName]
+    }
+  })
+}
+
+class Application {
+  constructor(name, props, detected) {
+    this.confidence = {}
+    this.confidenceTotal = 0
+    this.detected = Boolean(detected)
+    this.excludes = []
+    this.name = name
+    this.props = props
+    this.version = ''
+  }
+
+  /**
+   * Calculate confidence total
+   */
+  getConfidence() {
+    let total = 0
+
+    Object.keys(this.confidence).forEach((id) => {
+      total += this.confidence[id]
+    })
+
+    this.confidenceTotal = Math.min(total, 100)
+
+    return this.confidenceTotal
+  }
+}
+
+class Wappalyzer {
+  constructor() {
+    this.apps = {}
+    this.categories = {}
+    this.driver = {}
+    this.jsPatterns = {}
+    this.detected = {}
+    this.hostnameCache = {
+      expires: Date.now() + 1000 * 60 * 60 * 24,
+      hostnames: {}
+    }
+    this.adCache = []
+
+    this.config = {
+      websiteURL: 'https://www.wappalyzer.com/',
+      twitterURL: 'https://twitter.com/Wappalyzer',
+      githubURL: 'https://github.com/AliasIO/Wappalyzer'
+    }
+  }
+
+  /**
+   * Log messages to console
+   */
+  log(message, source, type) {
+    if (this.driver.log) {
+      this.driver.log(message, source || '', type || 'debug')
+    }
+  }
+
+  analyze(url, data, context) {
+    const apps = {}
+    const promises = []
+    const startTime = new Date()
+    const { scripts, cookies, headers, js } = data
+
+    let { html } = data
+
+    if (this.detected[url.canonical] === undefined) {
+      this.detected[url.canonical] = {}
+    }
+
+    const metaTags = []
+
+    // Additional information
+    let language = null
+
+    if (html) {
+      if (typeof html !== 'string') {
+        html = ''
+      }
+
+      let matches = data.html.match(
+        new RegExp('<html[^>]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"', 'i')
+      )
+
+      language = matches && matches.length ? matches[1] : data.language || null
+
+      // Meta tags
+      const regex = /<meta[^>]+>/gi
+
+      do {
+        matches = regex.exec(html)
+
+        if (!matches) {
+          break
+        }
+
+        metaTags.push(matches[0])
+      } while (matches)
+    }
+
+    Object.keys(this.apps).forEach((appName) => {
+      apps[appName] =
+        this.detected[url.canonical] && this.detected[url.canonical][appName]
+          ? this.detected[url.canonical][appName]
+          : new Application(appName, this.apps[appName])
+
+      const app = apps[appName]
+
+      promises.push(this.analyzeUrl(app, url))
+
+      if (html) {
+        promises.push(this.analyzeHtml(app, html))
+        promises.push(this.analyzeMeta(app, metaTags))
+      }
+
+      if (scripts) {
+        promises.push(this.analyzeScripts(app, scripts))
+      }
+
+      if (cookies) {
+        promises.push(this.analyzeCookies(app, cookies))
+      }
+
+      if (headers) {
+        promises.push(this.analyzeHeaders(app, headers))
+      }
+    })
+
+    if (js) {
+      Object.keys(js).forEach((appName) => {
+        if (typeof js[appName] !== 'function') {
+          promises.push(this.analyzeJs(apps[appName], js[appName]))
+        }
+      })
+    }
+
+    return new Promise(async (resolve) => {
+      await Promise.all(promises)
+
+      Object.keys(apps).forEach((appName) => {
+        const app = apps[appName]
+
+        if (!app.detected || !app.getConfidence()) {
+          delete apps[app.name]
+        }
+      })
+
+      resolveExcludes(apps, this.detected[url])
+      this.resolveImplies(apps, url.canonical)
+
+      this.cacheDetectedApps(apps, url.canonical)
+      this.trackDetectedApps(apps, url, language)
+
+      this.log(
+        `Processing ${Object.keys(data).join(', ')} took ${(
+          (new Date() - startTime) /
+          1000
+        ).toFixed(2)}s (${url.hostname})`,
+        'core'
+      )
+
+      if (Object.keys(apps).length) {
+        this.log(
+          `Identified ${Object.keys(apps).join(', ')} (${url.hostname})`,
+          'core'
+        )
+      }
+
+      this.driver.displayApps(
+        this.detected[url.canonical],
+        { language },
+        context
+      )
+
+      return resolve()
+    })
+  }
+
+  /**
+   * Cache detected ads
+   */
+  cacheDetectedAds(ad) {
+    this.adCache.push(ad)
+  }
+
+  /**
+   *
+   */
+  robotsTxtAllows(url) {
+    return new Promise(async (resolve, reject) => {
+      const parsed = this.parseUrl(url)
+
+      if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+        return reject()
+      }
+
+      const robotsTxt = await this.driver.getRobotsTxt(
+        parsed.host,
+        parsed.protocol === 'https:'
+      )
+
+      if (
+        robotsTxt.some(
+          (disallowedPath) => parsed.pathname.indexOf(disallowedPath) === 0
+        )
+      ) {
+        return reject()
+      }
+
+      return resolve()
+    })
+  }
+
+  /**
+   * Parse a URL
+   */
+  parseUrl(url) {
+    const a = this.driver.document.createElement('a')
+
+    a.href = url
+
+    a.canonical = `${a.protocol}//${a.host}${a.pathname}`
+
+    return a
+  }
+
+  /**
+   *
+   */
+  static parseRobotsTxt(robotsTxt) {
+    const disallow = []
+
+    let userAgent
+
+    robotsTxt.split('\n').forEach((line) => {
+      let matches = /^User-agent:\s*(.+)$/i.exec(line.trim())
+
+      if (matches) {
+        userAgent = matches[1].toLowerCase()
+      } else if (userAgent === '*' || userAgent === 'wappalyzer') {
+        matches = /^Disallow:\s*(.+)$/i.exec(line.trim())
+
+        if (matches) {
+          disallow.push(matches[1])
+        }
+      }
+    })
+
+    return disallow
+  }
+
+  /**
+   *
+   */
+  ping() {
+    if (
+      !this.hostnameCache.hostnames ||
+      Object.keys(this.hostnameCache.hostnames).length > 50 ||
+      this.hostnameCache.expires < Date.now()
+    ) {
+      this.driver.ping(this.hostnameCache)
+
+      this.hostnameCache = {
+        expires: Date.now() + 1000 * 60 * 60 * 24,
+        hostnames: {}
+      }
+    }
+
+    if (this.adCache.length > 50) {
+      this.driver.ping(undefined, this.adCache)
+
+      this.adCache = []
+    }
+  }
+
+  /**
+   * Parse apps.json patterns
+   */
+  parsePatterns(patterns) {
+    if (!patterns) {
+      return []
+    }
+
+    let parsed = {}
+
+    // Convert string to object containing array containing string
+    if (typeof patterns === 'string' || Array.isArray(patterns)) {
+      patterns = {
+        main: asArray(patterns)
+      }
+    }
+
+    Object.keys(patterns).forEach((key) => {
+      parsed[key] = []
+
+      asArray(patterns[key]).forEach((pattern) => {
+        const attrs = {}
+
+        pattern.split('\\;').forEach((attr, i) => {
+          if (i) {
+            // Key value pairs
+            attr = attr.split(':')
+
+            if (attr.length > 1) {
+              attrs[attr.shift()] = attr.join(':')
+            }
+          } else {
+            attrs.string = attr
+
+            try {
+              attrs.regex = new RegExp(attr.replace('/', '/'), 'i') // Escape slashes in regular expression
+            } catch (error) {
+              attrs.regex = new RegExp()
+
+              this.log(`${error.message}: ${attr}`, 'error', 'core')
+            }
+          }
+        })
+
+        parsed[key].push(attrs)
+      })
+    })
+
+    // Convert back to array if the original pattern list was an array (or string)
+    if ('main' in parsed) {
+      parsed = parsed.main
+    }
+
+    return parsed
+  }
+
+  /**
+   * Parse JavaScript patterns
+   */
+  parseJsPatterns() {
+    Object.keys(this.apps).forEach((appName) => {
+      if (this.apps[appName].js) {
+        this.jsPatterns[appName] = this.parsePatterns(this.apps[appName].js)
+      }
+    })
+  }
+
+  resolveImplies(apps, url) {
+    let checkImplies = true
+
+    const resolve = (appName) => {
+      const app = apps[appName]
+
+      if (app && app.props.implies) {
+        asArray(app.props.implies).forEach((implied) => {
+          ;[implied] = this.parsePatterns(implied)
+
+          if (!this.apps[implied.string]) {
+            this.log(
+              `Implied application ${implied.string} does not exist`,
+              'core',
+              'warn'
+            )
+
+            return
+          }
+
+          if (!(implied.string in apps)) {
+            apps[implied.string] =
+              this.detected[url] && this.detected[url][implied.string]
+                ? this.detected[url][implied.string]
+                : new Application(
+                    implied.string,
+                    this.apps[implied.string],
+                    true
+                  )
+
+            checkImplies = true
+          }
+
+          // Apply app confidence to implied app
+          Object.keys(app.confidence).forEach((id) => {
+            apps[implied.string].confidence[`${id} implied by ${appName}`] =
+              app.confidence[id] *
+              (implied.confidence === undefined ? 1 : implied.confidence / 100)
+          })
+        })
+      }
+    }
+
+    // Implied applications
+    // Run several passes as implied apps may imply other apps
+    while (checkImplies) {
+      checkImplies = false
+
+      Object.keys(apps).forEach(resolve)
+    }
+  }
+
+  /**
+   * Cache detected applications
+   */
+  cacheDetectedApps(apps, url) {
+    Object.keys(apps).forEach((appName) => {
+      const app = apps[appName]
+
+      // Per URL
+      this.detected[url][appName] = app
+
+      Object.keys(app.confidence).forEach((id) => {
+        this.detected[url][appName].confidence[id] = app.confidence[id]
+      })
+    })
+
+    if (this.driver.ping instanceof Function) {
+      this.ping()
+    }
+  }
+
+  /**
+   * Track detected applications
+   */
+  trackDetectedApps(apps, url, language) {
+    if (!(this.driver.ping instanceof Function)) {
+      return
+    }
+
+    const hostname = `${url.protocol}//${url.hostname}`
+
+    Object.keys(apps).forEach((appName) => {
+      const app = apps[appName]
+
+      if (this.detected[url.canonical][appName].getConfidence() >= 100) {
+        if (
+          validation.hostname.test(url.hostname) &&
+          !validation.hostnameBlacklist.test(url.hostname)
+        ) {
+          if (!(hostname in this.hostnameCache.hostnames)) {
+            this.hostnameCache.hostnames[hostname] = {
+              applications: {},
+              meta: {}
+            }
+          }
+
+          if (
+            !(appName in this.hostnameCache.hostnames[hostname].applications)
+          ) {
+            this.hostnameCache.hostnames[hostname].applications[appName] = {
+              hits: 0
+            }
+          }
+
+          this.hostnameCache.hostnames[hostname].applications[appName].hits += 1
+
+          if (apps[appName].version) {
+            this.hostnameCache.hostnames[hostname].applications[
+              appName
+            ].version = app.version
+          }
+        }
+      }
+    })
+
+    if (hostname in this.hostnameCache.hostnames) {
+      this.hostnameCache.hostnames[hostname].meta.language = language
+    }
+
+    this.ping()
+  }
+
+  /**
+   * Analyze URL
+   */
+  analyzeUrl(app, url) {
+    const patterns = this.parsePatterns(app.props.url)
+
+    if (!patterns.length) {
+      return Promise.resolve()
+    }
+
+    return asyncForEach(patterns, (pattern) => {
+      if (pattern.regex.test(url.canonical)) {
+        addDetected(app, pattern, 'url', url.canonical)
+      }
+    })
+  }
+
+  /**
+   * Analyze HTML
+   */
+  analyzeHtml(app, html) {
+    const patterns = this.parsePatterns(app.props.html)
+
+    if (!patterns.length) {
+      return Promise.resolve()
+    }
+
+    return asyncForEach(patterns, (pattern) => {
+      if (pattern.regex.test(html)) {
+        addDetected(app, pattern, 'html', html)
+      }
+    })
+  }
+
+  /**
+   * Analyze script tag
+   */
+  analyzeScripts(app, scripts) {
+    const patterns = this.parsePatterns(app.props.script)
+
+    if (!patterns.length) {
+      return Promise.resolve()
+    }
+
+    return asyncForEach(patterns, (pattern) => {
+      scripts.forEach((uri) => {
+        if (pattern.regex.test(uri)) {
+          addDetected(app, pattern, 'script', uri)
+        }
+      })
+    })
+  }
+
+  /**
+   * Analyze meta tag
+   */
+  analyzeMeta(app, metaTags) {
+    const patterns = this.parsePatterns(app.props.meta)
+    const promises = []
+
+    if (!app.props.meta) {
+      return Promise.resolve()
+    }
+
+    metaTags.forEach((match) => {
+      Object.keys(patterns).forEach((meta) => {
+        const r = new RegExp(`(?:name|property)=["']${meta}["']`, 'i')
+
+        if (r.test(match)) {
+          const content = match.match(/content=("|')([^"']+)("|')/i)
+
+          promises.push(
+            asyncForEach(patterns[meta], (pattern) => {
+              if (
+                content &&
+                content.length === 4 &&
+                pattern.regex.test(content[2])
+              ) {
+                addDetected(app, pattern, 'meta', content[2], meta)
+              }
+            })
+          )
+        }
+      })
+    })
+
+    return Promise.all(promises)
+  }
+
+  /**
+   * Analyze response headers
+   */
+  analyzeHeaders(app, headers) {
+    const patterns = this.parsePatterns(app.props.headers)
+    const promises = []
+
+    Object.keys(patterns).forEach((headerName) => {
+      if (typeof patterns[headerName] !== 'function') {
+        promises.push(
+          asyncForEach(patterns[headerName], (pattern) => {
+            headerName = headerName.toLowerCase()
+
+            if (headerName in headers) {
+              headers[headerName].forEach((headerValue) => {
+                if (pattern.regex.test(headerValue)) {
+                  addDetected(app, pattern, 'headers', headerValue, headerName)
+                }
+              })
+            }
+          })
+        )
+      }
+    })
+
+    return promises ? Promise.all(promises) : Promise.resolve()
+  }
+
+  /**
+   * Analyze cookies
+   */
+  analyzeCookies(app, cookies) {
+    const patterns = this.parsePatterns(app.props.cookies)
+    const promises = []
+
+    Object.keys(patterns).forEach((cookieName) => {
+      if (typeof patterns[cookieName] !== 'function') {
+        const cookieNameLower = cookieName.toLowerCase()
+
+        promises.push(
+          asyncForEach(patterns[cookieName], (pattern) => {
+            const cookie = cookies.find(
+              (_cookie) => _cookie.name.toLowerCase() === cookieNameLower
+            )
+
+            if (cookie && pattern.regex.test(cookie.value)) {
+              addDetected(app, pattern, 'cookies', cookie.value, cookieName)
+            }
+          })
+        )
+      }
+    })
+
+    return promises ? Promise.all(promises) : Promise.resolve()
+  }
+
+  /**
+   * Analyze JavaScript variables
+   */
+  analyzeJs(app, results) {
+    const promises = []
+
+    Object.keys(results).forEach((string) => {
+      if (typeof results[string] !== 'function') {
+        promises.push(
+          asyncForEach(Object.keys(results[string]), (index) => {
+            const pattern = this.jsPatterns[app.name][string][index]
+            const value = results[string][index]
+
+            if (pattern && pattern.regex.test(value)) {
+              addDetected(app, pattern, 'js', value, string)
+            }
+          })
+        )
+      }
+    })
+
+    return promises ? Promise.all(promises) : Promise.resolve()
+  }
+}
+
+if (typeof module === 'object') {
+  module.exports = Wappalyzer
+}

src/drivers/webextension/js/wappalyzer2.js

@@ -1,326 +0,0 @@
-'use strict'
-
-const Wappalyzer = {
-  technologies: [],
-  categories: [],
-
-  slugify(string) {
-    return string
-      .toLowerCase()
-      .replace(/[^a-z0-9-]/g, '-')
-      .replace(/--+/g, '-')
-      .replace(/(?:^-|-$)/, '')
-  },
-
-  unique(detections) {
-    return detections.filter(
-      ({ technology: { name }, pattern: { regex } }, index) => {
-        return (
-          detections.findIndex(
-            ({ technology: { name: _name }, pattern: { regex: _regex } }) =>
-              name === _name && (!regex || regex === _regex)
-          ) === index
-        )
-      }
-    )
-  },
-
-  getTechnology(name) {
-    return Wappalyzer.technologies.find(({ name: _name }) => name === _name)
-  },
-
-  getCategory(id) {
-    return Wappalyzer.categories.find(({ id: _id }) => id === _id)
-  },
-
-  resolve(detections) {
-    const resolved = detections.reduce((resolved, { technology }) => {
-      if (
-        resolved.findIndex(
-          ({ technology: { name } }) => name === technology.name
-        ) === -1
-      ) {
-        let version = ''
-        let confidence = 0
-
-        detections.forEach(({ technology: { name }, pattern, match }) => {
-          if (name === technology.name) {
-            const versionValue = Wappalyzer.resolveVersion(pattern, match)
-
-            confidence = Math.min(100, confidence + pattern.confidence)
-            version =
-              versionValue.length > version.length && versionValue.length <= 10
-                ? versionValue
-                : version
-          }
-        })
-
-        resolved.push({ technology, confidence, version })
-      }
-
-      return resolved
-    }, [])
-
-    Wappalyzer.resolveExcludes(resolved)
-    Wappalyzer.resolveImplies(resolved)
-
-    return resolved.map(
-      ({
-        technology: { name, slug, categories, icon, website },
-        confidence,
-        version
-      }) => ({
-        name,
-        slug,
-        categories: categories.map((id) => Wappalyzer.getCategory(id)),
-        confidence,
-        version,
-        icon,
-        website
-      })
-    )
-  },
-
-  resolveVersion({ version, regex }, match) {
-    let resolved = version
-
-    if (version) {
-      const matches = regex.exec(match)
-
-      if (matches) {
-        matches.forEach((match, index) => {
-          // Parse ternary operator
-          const ternary = new RegExp(`\\\\${index}\\?([^:]+):(.*)$`).exec(
-            version
-          )
-
-          if (ternary && ternary.length === 3) {
-            resolved = version.replace(
-              ternary[0],
-              match ? ternary[1] : ternary[2]
-            )
-          }
-
-          // Replace back references
-          resolved = resolved
-            .trim()
-            .replace(new RegExp(`\\\\${index}`, 'g'), match || '')
-        })
-      }
-    }
-
-    return resolved
-  },
-
-  resolveExcludes(resolved) {
-    resolved.forEach(({ technology }) => {
-      technology.excludes.forEach((name) => {
-        const excluded = Wappalyzer.getTechnology(name)
-
-        const index = resolved.findIndex(({ name }) => name === excluded.name)
-
-        if (index === -1) {
-          resolved.splice(index, 1)
-        }
-      })
-    })
-  },
-
-  resolveImplies(resolved) {
-    let done = false
-
-    while (!done) {
-      resolved.forEach(({ technology, confidence }) => {
-        done = true
-
-        technology.implies.forEach((name) => {
-          const implied = Wappalyzer.getTechnology(name)
-
-          if (
-            resolved.findIndex(
-              ({ technology: { name } }) => name === implied.name
-            ) === -1
-          ) {
-            resolved.push({ technology: implied, confidence, version: '' })
-
-            done = false
-          }
-        })
-      })
-    }
-  },
-
-  async analyze(url, { html, meta, headers, cookies, scripts, js }) {
-    const oo = Wappalyzer.analyzeOneToOne
-    const om = Wappalyzer.analyzeOneToMany
-    const mm = Wappalyzer.analyzeManyToMany
-
-    const flatten = (array) => Array.prototype.concat.apply([], array)
-
-    try {
-      const detections = flatten(
-        flatten(
-          await Promise.all(
-            Wappalyzer.technologies.map((technology) =>
-              Promise.all([
-                oo(technology, 'url', url),
-                oo(technology, 'html', html),
-                om(technology, 'meta', meta),
-                mm(technology, 'headers', headers),
-                om(technology, 'cookies', cookies),
-                om(technology, 'scripts', scripts)
-              ])
-            )
-          )
-        )
-      ).filter((technology) => technology)
-
-      return detections
-    } catch (error) {
-      throw new Error(error.message || error.toString())
-    }
-  },
-
-  setTechnologies(data) {
-    const transform = Wappalyzer.transformPatterns
-
-    Wappalyzer.technologies = Object.keys(data).reduce((technologies, name) => {
-      const {
-        cats,
-        url,
-        html,
-        meta,
-        headers,
-        cookies,
-        script,
-        implies,
-        excludes,
-        icon,
-        website
-      } = data[name]
-
-      technologies.push({
-        name,
-        categories: cats || [],
-        slug: Wappalyzer.slugify(name),
-        url: transform(url),
-        headers: transform(headers),
-        cookies: transform(cookies),
-        html: transform(html),
-        meta: transform(meta),
-        scripts: transform(script),
-        implies: typeof implies === 'string' ? [implies] : implies || [],
-        excludes: typeof excludes === 'string' ? [excludes] : excludes || [],
-        icon: icon || 'default.svg',
-        website: website || ''
-      })
-
-      return technologies
-    }, [])
-  },
-
-  setCategories(data) {
-    Wappalyzer.categories = Object.keys(data)
-      .reduce((categories, id) => {
-        const category = data[id]
-
-        categories.push({
-          id: parseInt(id, 10),
-          slug: Wappalyzer.slugify(category.name),
-          ...category
-        })
-
-        return categories
-      }, [])
-      .sort(({ priority: a }, { priority: b }) => (a > b ? -1 : 0))
-  },
-
-  transformPatterns(patterns) {
-    if (!patterns) {
-      return []
-    }
-
-    const toArray = (value) => (Array.isArray(value) ? value : [value])
-
-    if (typeof patterns === 'string' || Array.isArray(patterns)) {
-      patterns = { main: patterns }
-    }
-
-    const parsed = Object.keys(patterns).reduce((parsed, key) => {
-      parsed[key.toLowerCase()] = toArray(patterns[key]).map((pattern) => {
-        const { regex, confidence, version } = pattern
-          .split('\\;')
-          .reduce((attrs, attr, i) => {
-            if (i) {
-              // Key value pairs
-              attr = attr.split(':')
-
-              if (attr.length > 1) {
-                attrs[attr.shift()] = attr.join(':')
-              }
-            } else {
-              // Escape slashes in regular expression
-              attrs.regex = new RegExp(attr.replace(/\//g, '\\/'), 'i')
-            }
-
-            return attrs
-          }, {})
-
-        return {
-          regex,
-          confidence: parseInt(confidence || 100, 10),
-          version: version || ''
-        }
-      })
-
-      return parsed
-    }, {})
-
-    return 'main' in parsed ? parsed.main : parsed
-  },
-
-  analyzeOneToOne(technology, type, value) {
-    return technology[type].reduce((technologies, pattern) => {
-      if (pattern.regex.test(value)) {
-        technologies.push({ technology, pattern, match: value })
-      }
-
-      return technologies
-    }, [])
-  },
-
-  analyzeOneToMany(technology, type, items = []) {
-    return items.reduce((technologies, { key, value }) => {
-      const patterns = technology[type][key] || []
-
-      patterns.forEach((pattern) => {
-        if (pattern.regex.test(value)) {
-          technologies.push({ technology, pattern, match: value })
-        }
-      })
-
-      return technologies
-    }, [])
-  },
-
-  analyzeManyToMany(technology, type, items = {}) {
-    return Object.keys(technology[type]).reduce((technologies, key) => {
-      const patterns = technology[type][key] || []
-      const values = items[key] || []
-
-      patterns.forEach((pattern) => {
-        values.forEach((value) => {
-          if (pattern.regex.test(value)) {
-            technologies.push({ technology, pattern, match: value })
-          }
-        })
-      })
-
-      return technologies
-    }, [])
-  }
-}
-
-if (typeof module !== 'undefined') {
-  module.exports = Wappalyzer
-}