Performance improvements

src/drivers/npm/driver.js

@@ -10,10 +10,6 @@ const Wappalyzer = require('./wappalyzer')
 const { setTechnologies, setCategories, analyze, analyzeManyToMany, resolve } =
   Wappalyzer
 
-function next() {
-  return new Promise((resolve) => setImmediate(resolve))
-}
-
 const { CHROMIUM_BIN, CHROMIUM_DATA_DIR, CHROMIUM_WEBSOCKET } = process.env
 
 const chromiumArgs = [
@@ -98,21 +94,16 @@ function getJs(page, technologies = Wappalyzer.technologies) {
   }, technologies)
 }
 
-async function analyzeJs(js, technologies = Wappalyzer.technologies) {
+function analyzeJs(js, technologies = Wappalyzer.technologies) {
-  return Array.prototype.concat.apply(
+  return js
-    [],
+    .map(({ name, chain, value }) => {
-    await Promise.all(
-      js.map(async ({ name, chain, value }) => {
-        await next()
-
       return analyzeManyToMany(
         technologies.find(({ name: _name }) => name === _name),
         'js',
         { [chain]: [value] }
       )
     })
-    )
+    .flat()
-  )
 }
 
 function getDom(page, technologies = Wappalyzer.technologies) {
@@ -200,25 +191,10 @@ function getDom(page, technologies = Wappalyzer.technologies) {
   }, technologies)
 }
 
-async function analyzeDom(dom, technologies = Wappalyzer.technologies) {
+function analyzeDom(dom, technologies = Wappalyzer.technologies) {
-  return Array.prototype.concat.apply(
+  return dom
-    [],
+    .map(({ name, selector, exists, text, property, attribute, value }) => {
-    await Promise.all(
+      const technology = technologies.find(({ name: _name }) => name === _name)
-      dom.map(
-        async ({
-          name,
-          selector,
-          exists,
-          text,
-          property,
-          attribute,
-          value,
-        }) => {
-          await next()
-
-          const technology = technologies.find(
-            ({ name: _name }) => name === _name
-          )
 
       if (typeof exists !== 'undefined') {
         return analyzeManyToMany(technology, 'dom.exists', {
@@ -239,20 +215,12 @@ async function analyzeDom(dom, technologies = Wappalyzer.technologies) {
       }
 
       if (typeof attribute !== 'undefined') {
-            return analyzeManyToMany(
+        return analyzeManyToMany(technology, `dom.attributes.${attribute}`, {
-              technology,
-              `dom.attributes.${attribute}`,
-              {
           [selector]: [value],
+        })
       }
-            )
+    })
-          }
+    .flat()
-
-          return []
-        }
-      )
-    )
-  )
 }
 
 function get(url, options = {}) {
@@ -860,8 +828,7 @@ class Site {
 
       await this.onDetect(
         url,
-        (
+        [
-          await Promise.all([
           analyzeDom(dom),
           analyzeJs(js),
           analyze({
@@ -874,8 +841,7 @@ class Site {
             scriptSrc,
             meta,
           }),
-          ])
+        ].flat()
-        ).flat()
       )
 
       const reducedLinks = Array.prototype.reduce.call(
@@ -1189,11 +1155,10 @@ class Site {
 
             await this.onDetect(
               url,
-              (
+              [
-                await Promise.all([
                 analyzeDom(dom, technologies),
                 analyzeJs(js, technologies),
-                  analyze(
+                await analyze(
                   {
                     url,
                     cookies,
@@ -1206,8 +1171,7 @@ class Site {
                   },
                   technologies
                 ),
-                ])
+              ].flat()
-              ).flat()
             )
           }
         })

src/drivers/webextension/js/content.js

@@ -288,7 +288,9 @@ const Content = {
       // Delayed second pass to capture async JS
       await new Promise((resolve) => setTimeout(resolve, 5000))
 
-      await Content.onGetTechnologies(technologies)
+      const js = await getJs(technologies)
+
+      await Content.driver('analyzeJs', [url, js])
     } catch (error) {
       Content.driver('error', error)
     }

src/drivers/webextension/js/driver.js

@@ -1,6 +1,6 @@
 'use strict'
 /* eslint-env browser */
-/* globals chrome, Wappalyzer, Utils, next */
+/* globals chrome, Wappalyzer, Utils */
 
 const {
   setTechnologies,
@@ -217,27 +217,22 @@ const Driver = {
    * @param {String} url
    * @param {Array} js
    */
-  async analyzeJs(url, js, requires, categoryRequires) {
+  analyzeJs(url, js, requires, categoryRequires) {
     const technologies =
       getRequiredTechnologies(requires, categoryRequires) ||
       Wappalyzer.technologies
 
     return Driver.onDetect(
       url,
-      Array.prototype.concat.apply(
+      js
-        [],
+        .map(({ name, chain, value }) => {
-        await Promise.all(
-          js.map(async ({ name, chain, value }) => {
-            await next()
-
           return analyzeManyToMany(
             technologies.find(({ name: _name }) => name === _name),
             'js',
             { [chain]: [value] }
           )
         })
-        )
+        .flat()
-      )
     )
   },
 
@@ -246,23 +241,19 @@ const Driver = {
    * @param {String} url
    * @param {Array} dom
    */
-  async analyzeDom(url, dom, requires, categoryRequires) {
+  analyzeDom(url, dom, requires, categoryRequires) {
     const technologies =
       getRequiredTechnologies(requires, categoryRequires) ||
       Wappalyzer.technologies
 
     return Driver.onDetect(
       url,
-      Array.prototype.concat.apply(
+      dom
-        [],
+        .map(
-        await Promise.all(
+          (
-          dom.map(
-            async (
             { name, selector, exists, text, property, attribute, value },
             index
           ) => {
-              await next()
-
             const technology = technologies.find(
               ({ name: _name }) => name === _name
             )
@@ -298,12 +289,9 @@ const Driver = {
                 }
               )
             }
-
-              return []
           }
         )
-        )
+        .flat()
-      )
     )
   },
 
@@ -673,11 +661,7 @@ const Driver = {
       ),
     ]
 
-    try {
+    Driver.content(url, 'analyzeRequires', [url, requires])
-      await Driver.content(url, 'analyzeRequires', [url, requires])
-    } catch (error) {
-      // Continue
-    }
 
     await Driver.setIcon(url, resolved)
 

src/technologies/l.json

@@ -1105,7 +1105,7 @@
     ],
     "description": "LottieFiles is an open-source animation file format that's tiny, high quality, interactive, and can be manipulated at runtime.",
     "icon": "LottieFiles.svg",
-    "dom": "clipPath[id*='__lottie_element_']",
+    "dom": "lottie-player, div[data-animation-type='lottie'], div[data-testid='lottie-player'], clipPath[id*='__lottie_element_']",
     "scriptSrc": [
       "/dist/lottie-player\\.js",
       "/dist/tgs-player\\.js"