Implement categoryRequires in WebExtension

src/drivers/npm/driver.js

@@ -1,4 +1,3 @@
-/* eslint-disable unicorn/prefer-text-content */
 const { URL } = require('url')
 const fs = require('fs')
 const dns = require('dns').promises
@@ -733,6 +732,7 @@ class Site {
           (
             await this.promiseTimeout(
               page.evaluateHandle(() =>
+                // eslint-disable-next-line unicorn/prefer-text-content
                 document.body.innerText.replace(/\s+/g, ' ')
               ),
               { jsonValue: () => '' },

src/drivers/webextension/js/content.js

@@ -176,6 +176,10 @@ const Content = {
         {}
       )
 
+      // Text
+      // eslint-disable-next-line unicorn/prefer-text-content
+      const text = document.body.innerText.replace(/\s+/g, ' ')
+
       // CSS rules
       let css = []
 
@@ -269,7 +273,7 @@ const Content = {
         }
       }
 
-      Content.cache = { html, css, scriptSrc, scripts, meta, cookies }
+      Content.cache = { html, text, css, scriptSrc, scripts, meta, cookies }
 
       await Content.driver('onContentLoad', [
         url,
@@ -351,17 +355,20 @@ const Content = {
 
   async analyzeRequires(url, requires) {
     await Promise.all(
-      requires.map(async ({ name, technologies }) => {
+      requires.map(async ({ name, categoryId, technologies }) => {
-        if (!Content.analyzedRequires.includes(name)) {
+        const id = categoryId ? `category:${categoryId}` : `technology:${name}`
-          Content.analyzedRequires.push(name)
+
+        if (!Content.analyzedRequires.includes(id)) {
+          Content.analyzedRequires.push(id)
 
           await Promise.all([
-            Content.onGetTechnologies(technologies, name),
+            Content.onGetTechnologies(technologies, name, categoryId),
             Content.driver('onContentLoad', [
               url,
               Content.cache,
               Content.language,
               name,
+              categoryId,
             ]),
           ])
         }
@@ -373,15 +380,15 @@ const Content = {
    * Callback for getTechnologies
    * @param {Array} technologies
    */
-  async onGetTechnologies(technologies = [], requires) {
+  async onGetTechnologies(technologies = [], requires, categoryRequires) {
     const url = location.href
 
     const js = await getJs(technologies)
     const dom = await getDom(technologies)
 
     await Promise.all([
-      Content.driver('analyzeJs', [url, js, requires]),
+      Content.driver('analyzeJs', [url, js, requires, categoryRequires]),
-      Content.driver('analyzeDom', [url, dom, requires]),
+      Content.driver('analyzeDom', [url, dom, requires, categoryRequires]),
     ])
   },
 }

src/drivers/webextension/js/driver.js

@@ -21,6 +21,16 @@ const xhrDebounce = []
 
 const scriptsPending = []
 
+function getRequiredTechnologies(name, categoryId) {
+  return name
+    ? Wappalyzer.requires.find(({ name: _name }) => _name === name).technologies
+    : categoryId
+    ? Wappalyzer.categoryRequires.find(
+        ({ categoryId: _categoryId }) => _categoryId === categoryId
+      ).technologies
+    : undefined
+}
+
 const Driver = {
   lastPing: Date.now(),
 
@@ -198,10 +208,8 @@ const Driver = {
    * @param {String} url
    * @param {Array} js
    */
-  async analyzeJs(url, js, requires) {
+  async analyzeJs(url, js, requires, categoryRequires) {
-    const technologies = requires
+    const technologies = getRequiredTechnologies(requires, categoryRequires)
-      ? Wappalyzer.requires.find(({ name }) => name === requires).technologies
-      : Wappalyzer.technologies
 
     return Driver.onDetect(
       url,
@@ -227,10 +235,8 @@ const Driver = {
    * @param {String} url
    * @param {Array} dom
    */
-  async analyzeDom(url, dom, requires) {
+  async analyzeDom(url, dom, requires, categoryRequires) {
-    const technologies = requires
+    const technologies = getRequiredTechnologies(requires, categoryRequires)
-      ? Wappalyzer.requires[requires].technologies
-      : Wappalyzer.technologies
 
     return Driver.onDetect(
       url,
@@ -468,7 +474,7 @@ const Driver = {
    * @param {Object} items
    * @param {String} language
    */
-  async onContentLoad(url, items, language, requires) {
+  async onContentLoad(url, items, language, requires, categoryRequires) {
     try {
       items.cookies = items.cookies || {}
 
@@ -481,12 +487,11 @@ const Driver = {
         ({ name, value }) => (items.cookies[name.toLowerCase()] = [value])
       )
 
+      const technologies = getRequiredTechnologies(requires, categoryRequires)
+
       await Driver.onDetect(
         url,
-        await analyze(
+        await analyze({ url, ...items }, technologies),
-          { url, ...items },
-          requires ? Wappalyzer.requires[requires].technologies : undefined
-        ),
         language,
         true
       )
@@ -533,10 +538,6 @@ const Driver = {
       return
     }
 
-    Driver.log([
-      ...new Set(detections.map(({ technology }) => technology.name)),
-    ])
-
     url = url.split('#')[0]
 
     const { hostname } = new URL(url)
@@ -632,9 +633,16 @@ const Driver = {
       return detection
     })
 
-    const requires = Wappalyzer.requires.filter(({ name, technologies }) =>
+    const requires = [
-      resolved.some(({ name: _name }) => _name === name)
+      ...Wappalyzer.requires.filter(({ name }) =>
-    )
+        resolved.some(({ name: _name }) => _name === name)
+      ),
+      ...Wappalyzer.categoryRequires.filter(({ categoryId }) =>
+        resolved.some(({ categories }) =>
+          categories.some(({ id }) => id === categoryId)
+        )
+      ),
+    ]
 
     try {
       await Driver.content(url, 'analyzeRequires', [url, requires])