Implement requires in WebExtension driver

src/drivers/webextension/js/content.js

@@ -3,6 +3,12 @@
 /* globals chrome */
 
 const Content = {
+  href: location.href,
+  cache: {},
+  language: '',
+
+  requiresAnalyzed: [],
+
   /**
    * Initialise content script
    */
@@ -32,7 +38,7 @@ const Content = {
       html = chunks.join('\n')
 
       // Determine language based on the HTML lang attribute or content
-      const language =
+      Content.language =
         document.documentElement.getAttribute('lang') ||
         document.documentElement.getAttribute('xml:lang') ||
         (await new Promise((resolve) =>
@@ -86,10 +92,37 @@ const Content = {
         {}
       )
 
+      // Detect Google Ads
+      if (/^(www\.)?google(\.[a-z]{2,3}){1,2}$/.test(location.hostname)) {
+        const ads = document.querySelectorAll(
+          '#tads [data-text-ad] a[data-pcu]'
+        )
+
+        for (const ad of ads) {
+          Content.driver('detectTechnology', [ad.href, 'Google Ads'])
+        }
+      }
+
+      // Detect Microsoft Ads
+      if (/^(www\.)?bing\.com$/.test(location.hostname)) {
+        const ads = document.querySelectorAll('.b_ad .b_adurl cite')
+
+        for (const ad of ads) {
+          const url = ad.textContent.split(' ')[0].trim()
+
+          Content.driver('detectTechnology', [
+            url.startsWith('http') ? url : `http://${url}`,
+            'Microsoft Advertising',
+          ])
+        }
+      }
+
+      Content.cache = { html, css, scripts, meta }
+
       Content.driver('onContentLoad', [
-        location.href,
+        Content.href,
-        { html, css, scripts, meta },
+        Content.cache,
-        language,
+        Content.language,
       ])
 
       const technologies = await Content.driver('getTechnologies')
@@ -105,6 +138,32 @@ const Content = {
     }
   },
 
+  /**
+   * Enable scripts to call Driver functions through messaging
+   * @param {Object} message
+   * @param {Object} sender
+   * @param {Function} callback
+   */
+  onMessage({ source, func, args }, sender, callback) {
+    if (!func) {
+      return
+    }
+
+    Content.driver('log', { source, func, args })
+
+    if (!Content[func]) {
+      Content.error(new Error(`Method does not exist: Content.${func}`))
+
+      return
+    }
+
+    Promise.resolve(Content[func].call(Content[func], ...(args || [])))
+      .then(callback)
+      .catch(Content.error)
+
+    return !!callback
+  },
+
   driver(func, args) {
     return new Promise((resolve, reject) => {
       chrome.runtime.sendMessage(
@@ -136,6 +195,23 @@ const Content = {
     })
   },
 
+  analyzeRequires(requires) {
+    Object.keys(requires).forEach((name) => {
+      if (!Content.requiresAnalyzed.includes(name)) {
+        Content.requiresAnalyzed.push(name)
+
+        Content.driver('onContentLoad', [
+          Content.href,
+          Content.cache,
+          Content.language,
+          name,
+        ])
+
+        Content.onGetTechnologies(requires[name].technologies)
+      }
+    })
+  },
+
   /**
    * Callback for getTechnologies
    * @param {Array} technologies
@@ -259,33 +335,13 @@ const Content = {
         return technologies
       }, [])
 
-    // Detect Google Ads
-    if (/^(www\.)?google(\.[a-z]{2,3}){1,2}$/.test(location.hostname)) {
-      const ads = document.querySelectorAll('#tads [data-text-ad] a[data-pcu]')
-
-      for (const ad of ads) {
-        Content.driver('detectTechnology', [ad.href, 'Google Ads'])
-      }
-    }
-
-    // Detect Microsoft Ads
-    if (/^(www\.)?bing\.com$/.test(location.hostname)) {
-      const ads = document.querySelectorAll('.b_ad .b_adurl cite')
-
-      for (const ad of ads) {
-        const url = ad.textContent.split(' ')[0].trim()
-
-        Content.driver('detectTechnology', [
-          url.startsWith('http') ? url : `http://${url}`,
-          'Microsoft Advertising',
-        ])
-      }
-    }
-
     Content.driver('analyzeDom', [location.href, dom])
   },
 }
 
+// Enable messaging between scripts
+chrome.runtime.onMessage.addListener(Content.onMessage)
+
 if (/complete|interactive|loaded/.test(document.readyState)) {
   Content.init()
 } else {

src/drivers/webextension/js/driver.js

@@ -298,6 +298,38 @@ const Driver = {
     return !!callback
   },
 
+  async content(url, func, args) {
+    const [tab] = await promisify(chrome.tabs, 'query', {
+      url: globEscape(url),
+    })
+
+    if (!tab) {
+      return
+    }
+
+    return new Promise((resolve, reject) => {
+      chrome.tabs.sendMessage(
+        tab.id,
+        {
+          source: 'driver.js',
+          func,
+          args: args ? (Array.isArray(args) ? args : [args]) : [],
+        },
+        (response) => {
+          chrome.runtime.lastError
+            ? func === 'error'
+              ? resolve()
+              : Driver.error(
+                  new Error(
+                    `${chrome.runtime.lastError}: Driver.${func}(${args})`
+                  )
+                )
+            : resolve(response)
+        }
+      )
+    })
+  },
+
   /**
    * Analyse response headers
    * @param {Object} request
@@ -375,7 +407,7 @@ const Driver = {
    * @param {Object} items
    * @param {String} language
    */
-  async onContentLoad(url, items, language) {
+  async onContentLoad(url, items, language, requires) {
     try {
       const { hostname } = new URL(url)
 
@@ -393,7 +425,10 @@ const Driver = {
 
       await Driver.onDetect(
         url,
-        await analyze({ url, ...items }),
+        await analyze(
+          { url, ...items },
+          requires ? Wappalyzer.requires[requires].technologies : undefined
+        ),
         language,
         true
       )
@@ -429,7 +464,13 @@ const Driver = {
    * @param {String} language
    * @param {Boolean} incrementHits
    */
-  async onDetect(url, detections = [], language, incrementHits = false) {
+  async onDetect(
+    url,
+    detections = [],
+    language,
+    incrementHits = false,
+    analyzeRequires = true
+  ) {
     if (!url || !detections.length) {
       return
     }
@@ -526,6 +567,15 @@ const Driver = {
       return detection
     })
 
+    const requires = Wappalyzer.requires
+      .filter(({ name, technologies }) =>
+        resolved.some(({ name: _name }) => _name === name)
+      )
+      .map(({ technologies }) => technologies)
+      .flat()
+
+    Driver.content(url, 'analyzeRequires', [requires])
+
     await Driver.setIcon(url, resolved)
 
     if (url) {

src/technologies.json

@@ -18123,8 +18123,7 @@
       "description": "SiteSpect is the A/B testing and optimisation solution.",
       "icon": "SiteSpect.png",
       "js": {
-        "SS": "\\;confidence:50",
+        "ss_dom_var": ""
-        "ss_dom_var": "\\;confidence:50"
       },
       "pricing": [
         "poa"
@@ -24442,4 +24441,4 @@
       "website": "https://www.xt-commerce.com"
     }
   }
-}
+}

src/wappalyzer.js

@@ -11,6 +11,7 @@ function toArray(value) {
 const Wappalyzer = {
   technologies: [],
   categories: [],
+  requires: {},
 
   slugify: (string) =>
     string
@@ -194,20 +195,23 @@ const Wappalyzer = {
    * Initialize analyzation.
    * @param {*} param0
    */
-  async analyze({
+  async analyze(
-    url,
+    {
-    xhr,
+      url,
-    html,
+      xhr,
-    css,
+      html,
-    robots,
+      css,
-    magento,
+      robots,
-    meta,
+      magento,
-    headers,
+      meta,
-    dns,
+      headers,
-    certIssuer,
+      dns,
-    cookies,
+      certIssuer,
-    scripts,
+      cookies,
-  }) {
+      scripts,
+    },
+    technologies = Wappalyzer.technologies
+  ) {
     const oo = Wappalyzer.analyzeOneToOne
     const om = Wappalyzer.analyzeOneToMany
     const mm = Wappalyzer.analyzeManyToMany
@@ -217,7 +221,7 @@ const Wappalyzer = {
     try {
       const detections = flatten(
         await Promise.all(
-          Wappalyzer.technologies.map(async (technology) => {
+          technologies.map(async (technology) => {
             await next()
 
             return flatten([
@@ -270,6 +274,7 @@ const Wappalyzer = {
         js,
         implies,
         excludes,
+        requires,
         icon,
         website,
         cpe,
@@ -312,6 +317,9 @@ const Wappalyzer = {
         excludes: transform(excludes).map(({ value }) => ({
           name: value,
         })),
+        requires: transform(requires).map(({ value }) => ({
+          name: value,
+        })),
         icon: icon || 'default.svg',
         website: website || null,
         cpe: cpe || null,
@@ -319,6 +327,29 @@ const Wappalyzer = {
 
       return technologies
     }, [])
+
+    Wappalyzer.technologies
+      .filter(({ requires }) => requires.length)
+      .forEach((technology) =>
+        technology.requires.forEach(({ name }) => {
+          if (!Wappalyzer.getTechnology(name)) {
+            throw new Error(`Required technology does not exist: ${name}`)
+          }
+
+          Wappalyzer.requires[name] = Wappalyzer.requires[name] || []
+
+          Wappalyzer.requires[name].push(technology)
+        })
+      )
+
+    Wappalyzer.requires = Object.keys(Wappalyzer.requires).map((name) => ({
+      name,
+      technologies: Wappalyzer.requires[name],
+    }))
+
+    Wappalyzer.technologies = Wappalyzer.technologies.filter(
+      ({ requires }) => !requires.length
+    )
   },
 
   /**