Make 'requires' work with 'dom' and 'js' in WebExtension driver

src/drivers/webextension/css/styles.css

@@ -616,7 +616,7 @@ body.dynamic-icon .category__heading:hover .category__pin {
   color: var(--color-text);
   border-radius: 3px;
   font-size: .7rem;
-  padding: .3rem .3rem .1rem .3rem;
+  padding: .1rem .3rem;
   margin-left: .3rem;
   vertical-align: middle;
 }

src/drivers/webextension/js/content.js

@@ -2,12 +2,129 @@
 /* eslint-env browser */
 /* globals chrome */
 
+function getJs(technologies) {
+  return new Promise((resolve) => {
+    // Inject a script tag into the page to access methods of the window object
+    const script = document.createElement('script')
+
+    script.onload = () => {
+      const onMessage = ({ data }) => {
+        if (!data.wappalyzer || !data.wappalyzer.js) {
+          return
+        }
+
+        window.removeEventListener('message', onMessage)
+
+        resolve(data.wappalyzer.js)
+
+        script.remove()
+      }
+
+      window.addEventListener('message', onMessage)
+
+      window.postMessage({
+        wappalyzer: {
+          technologies: technologies
+            .filter(({ js }) => Object.keys(js).length)
+            .map(({ name, js }) => ({ name, chains: Object.keys(js) })),
+        },
+      })
+    }
+
+    script.setAttribute('src', chrome.extension.getURL('js/inject.js'))
+
+    document.body.appendChild(script)
+  })
+}
+
+function getDom(technologies) {
+  return technologies
+    .filter(({ dom }) => dom && dom.constructor === Object)
+    .map(({ name, dom }) => ({ name, dom }))
+    .reduce((technologies, { name, dom }) => {
+      const toScalar = (value) =>
+        typeof value === 'string' || typeof value === 'number' ? value : !!value
+
+      Object.keys(dom).forEach((selector) => {
+        let nodes = []
+
+        try {
+          nodes = document.querySelectorAll(selector)
+        } catch (error) {
+          Content.driver('error', error)
+        }
+
+        if (!nodes.length) {
+          return
+        }
+
+        dom[selector].forEach(({ exists, text, properties, attributes }) => {
+          nodes.forEach((node) => {
+            if (exists) {
+              technologies.push({
+                name,
+                selector,
+                exists: '',
+              })
+            }
+
+            if (text) {
+              const value = node.textContent.trim()
+
+              if (value) {
+                technologies.push({
+                  name,
+                  selector,
+                  text: value,
+                })
+              }
+            }
+
+            if (properties) {
+              Object.keys(properties).forEach((property) => {
+                if (Object.prototype.hasOwnProperty.call(node, property)) {
+                  const value = node[property]
+
+                  if (typeof value !== 'undefined') {
+                    technologies.push({
+                      name,
+                      selector,
+                      property,
+                      value: toScalar(value),
+                    })
+                  }
+                }
+              })
+            }
+
+            if (attributes) {
+              Object.keys(attributes).forEach((attribute) => {
+                if (node.hasAttribute(attribute)) {
+                  const value = node.getAttribute(attribute)
+
+                  technologies.push({
+                    name,
+                    selector,
+                    attribute,
+                    value: toScalar(value),
+                  })
+                }
+              })
+            }
+          })
+        })
+      })
+
+      return technologies
+    }, [])
+}
+
 const Content = {
   href: location.href,
   cache: {},
   language: '',
 
-  requiresAnalyzed: [],
+  analyzedRequires: [],
 
   /**
    * Initialise content script
@@ -119,7 +236,7 @@ const Content = {
 
       Content.cache = { html, css, scripts, meta }
 
-      Content.driver('onContentLoad', [
+      await Content.driver('onContentLoad', [
         Content.href,
         Content.cache,
         Content.language,
@@ -127,12 +244,12 @@ const Content = {
 
       const technologies = await Content.driver('getTechnologies')
 
-      Content.onGetTechnologies(technologies)
+      await Content.onGetTechnologies(technologies)
 
       // Delayed second pass to capture async JS
       await new Promise((resolve) => setTimeout(resolve, 5000))
 
-      Content.onGetTechnologies(technologies)
+      await Content.onGetTechnologies(technologies)
     } catch (error) {
       Content.driver('error', error)
     }
@@ -165,7 +282,7 @@ const Content = {
   },
 
   driver(func, args) {
-    return new Promise((resolve, reject) => {
+    return new Promise((resolve) => {
       chrome.runtime.sendMessage(
         {
           source: 'content.js',
@@ -186,7 +303,9 @@ const Content = {
               : Content.driver(
                   'error',
                   new Error(
-                    `${chrome.runtime.lastError}: Driver.${func}(${args})`
+                    `${
+                      chrome.runtime.lastError.message
+                    }: Driver.${func}(${JSON.stringify(args)})`
                   )
                 )
             : resolve(response)
@@ -195,147 +314,42 @@ const Content = {
     })
   },
 
-  analyzeRequires(requires) {
+  async analyzeRequires(requires) {
-    Object.keys(requires).forEach((name) => {
+    await Promise.all(
-      if (!Content.requiresAnalyzed.includes(name)) {
+      Object.keys(requires).map(async (name) => {
-        Content.requiresAnalyzed.push(name)
+        if (!Content.analyzedRequires.includes(name)) {
-
+          Content.analyzedRequires.push(name)
-        Content.driver('onContentLoad', [
+
-          Content.href,
+          const technologies = requires[name].technologies
-          Content.cache,
+
-          Content.language,
+          await Promise.all([
-          name,
+            Content.onGetTechnologies(technologies, name),
-        ])
+            Content.driver('onContentLoad', [
-
+              Content.href,
-        Content.onGetTechnologies(requires[name].technologies)
+              Content.cache,
-      }
+              Content.language,
-    })
+              name,
+            ]),
+          ])
+        }
+      })
+    )
   },
 
   /**
    * Callback for getTechnologies
    * @param {Array} technologies
    */
-  onGetTechnologies(technologies = []) {
+  async onGetTechnologies(technologies = [], requires) {
-    // Inject a script tag into the page to access methods of the window object
+    const url = location.href.split('#')[0]
-    const script = document.createElement('script')
-
-    script.onload = () => {
-      const onMessage = ({ data }) => {
-        if (!data.wappalyzer || !data.wappalyzer.js) {
-          return
-        }
-
-        window.removeEventListener('message', onMessage)
-
-        chrome.runtime.sendMessage({
-          source: 'content.js',
-          func: 'analyzeJs',
-          args: [location.href.split('#')[0], data.wappalyzer.js],
-        })
-
-        script.remove()
-      }
-
-      window.addEventListener('message', onMessage)
-
-      window.postMessage({
-        wappalyzer: {
-          technologies: technologies
-            .filter(({ js }) => Object.keys(js).length)
-            .map(({ name, js }) => ({ name, chains: Object.keys(js) })),
-        },
-      })
-    }
-
-    script.setAttribute('src', chrome.extension.getURL('js/inject.js'))
-
-    document.body.appendChild(script)
-
-    // DOM
-    const dom = technologies
-      .filter(({ dom }) => dom && dom.constructor === Object)
-      .map(({ name, dom }) => ({ name, dom }))
-      .reduce((technologies, { name, dom }) => {
-        const toScalar = (value) =>
-          typeof value === 'string' || typeof value === 'number'
-            ? value
-            : !!value
-
-        Object.keys(dom).forEach((selector) => {
-          let nodes = []
-
-          try {
-            nodes = document.querySelectorAll(selector)
-          } catch (error) {
-            Content.driver('error', error)
-          }
-
-          if (!nodes.length) {
-            return
-          }
-
-          dom[selector].forEach(({ exists, text, properties, attributes }) => {
-            nodes.forEach((node) => {
-              if (exists) {
-                technologies.push({
-                  name,
-                  selector,
-                  exists: '',
-                })
-              }
-
-              if (text) {
-                const value = node.textContent.trim()
-
-                if (value) {
-                  technologies.push({
-                    name,
-                    selector,
-                    text: value,
-                  })
-                }
-              }
-
-              if (properties) {
-                Object.keys(properties).forEach((property) => {
-                  if (Object.prototype.hasOwnProperty.call(node, property)) {
-                    const value = node[property]
-
-                    if (typeof value !== 'undefined') {
-                      technologies.push({
-                        name,
-                        selector,
-                        property,
-                        value: toScalar(value),
-                      })
-                    }
-                  }
-                })
-              }
-
-              if (attributes) {
-                Object.keys(attributes).forEach((attribute) => {
-                  if (node.hasAttribute(attribute)) {
-                    const value = node.getAttribute(attribute)
-
-                    technologies.push({
-                      name,
-                      selector,
-                      attribute,
-                      value: toScalar(value),
-                    })
-                  }
-                })
-              }
-            })
-          })
-        })
 
-        return technologies
+    const js = await getJs(technologies)
-      }, [])
+    const dom = getDom(technologies)
 
-    Content.driver('analyzeDom', [location.href, dom])
+    await Promise.all([
+      Content.driver('analyzeJs', [url, js, requires]),
+      Content.driver('analyzeDom', [url, dom, requires]),
+    ])
   },
 }
 

src/drivers/webextension/js/driver.js

@@ -42,9 +42,7 @@ const Driver = {
                 pattern: { regex, confidence },
                 version,
               }) => ({
-                technology: Wappalyzer.technologies.find(
+                technology: getTechnology(name, true),
-                  ({ name: _name }) => name === _name
-                ),
                 pattern: {
                   regex: new RegExp(regex, 'i'),
                   confidence,
@@ -121,7 +119,7 @@ const Driver = {
    */
   log(message, source = 'driver', type = 'log') {
     // eslint-disable-next-line no-console
-    console[type](`wappalyzer | ${source} |`, message)
+    console[type](message)
   },
 
   /**
@@ -177,7 +175,11 @@ const Driver = {
    * @param {String} url
    * @param {Array} js
    */
-  async analyzeJs(url, js) {
+  async analyzeJs(url, js, requires) {
+    const technologies = requires
+      ? Wappalyzer.requires[requires].technologies
+      : Wappalyzer.technologies
+
     return Driver.onDetect(
       url,
       Array.prototype.concat.apply(
@@ -187,7 +189,7 @@ const Driver = {
             await next()
 
             return analyzeManyToMany(
-              Wappalyzer.technologies.find(({ name: _name }) => name === _name),
+              technologies.find(({ name: _name }) => name === _name),
               'js',
               { [chain]: [value] }
             )
@@ -202,7 +204,11 @@ const Driver = {
    * @param {String} url
    * @param {Array} dom
    */
-  async analyzeDom(url, dom) {
+  async analyzeDom(url, dom, requires) {
+    const technologies = requires
+      ? Wappalyzer.requires[requires].technologies
+      : Wappalyzer.technologies
+
     return Driver.onDetect(
       url,
       Array.prototype.concat.apply(
@@ -215,7 +221,7 @@ const Driver = {
             ) => {
               await next()
 
-              const technology = Wappalyzer.technologies.find(
+              const technology = technologies.find(
                 ({ name: _name }) => name === _name
               )
 
@@ -283,7 +289,9 @@ const Driver = {
       return
     }
 
-    Driver.log({ source, func, args })
+    if (func !== 'log') {
+      Driver.log({ source, func, args })
+    }
 
     if (!Driver[func]) {
       Driver.error(new Error(`Method does not exist: Driver.${func}`))
@@ -307,6 +315,10 @@ const Driver = {
       return
     }
 
+    if (tab.status !== 'complete') {
+      throw new Error(`Tab ${tab.id} not ready for sendMessage: ${tab.status}`)
+    }
+
     return new Promise((resolve, reject) => {
       chrome.tabs.sendMessage(
         tab.id,
@@ -321,7 +333,9 @@ const Driver = {
               ? resolve()
               : Driver.error(
                   new Error(
-                    `${chrome.runtime.lastError}: Driver.${func}(${args})`
+                    `${
+                      chrome.runtime.lastError.message
+                    }: Driver.${func}(${JSON.stringify(args)})`
                   )
                 )
             : resolve(response)
@@ -567,14 +581,15 @@ const Driver = {
       return detection
     })
 
-    const requires = Wappalyzer.requires
+    const requires = Wappalyzer.requires.filter(({ name, technologies }) =>
-      .filter(({ name, technologies }) =>
+      resolved.some(({ name: _name }) => _name === name)
-        resolved.some(({ name: _name }) => _name === name)
+    )
-      )
-      .map(({ technologies }) => technologies)
-      .flat()
 
-    Driver.content(url, 'analyzeRequires', [requires])
+    try {
+      await Driver.content(url, 'analyzeRequires', [requires])
+    } catch (error) {
+      // Continue
+    }
 
     await Driver.setIcon(url, resolved)
 

src/wappalyzer.js

@@ -11,7 +11,7 @@ function toArray(value) {
 const Wappalyzer = {
   technologies: [],
   categories: [],
-  requires: {},
+  requires: [],
 
   slugify: (string) =>
     string
@@ -21,7 +21,10 @@ const Wappalyzer = {
       .replace(/(?:^-|-$)/g, ''),
 
   getTechnology: (name) =>
-    Wappalyzer.technologies.find(({ name: _name }) => name === _name),
+    [
+      ...Wappalyzer.technologies,
+      ...Wappalyzer.requires.map(({ technologies }) => technologies).flat(),
+    ].find(({ name: _name }) => name === _name),
 
   getCategory: (id) => Wappalyzer.categories.find(({ id: _id }) => id === _id),