Non-blocking analyse

src/README.md

@@ -25,16 +25,16 @@ const { technologies, categories } = JSON.parse(
 Wappalyzer.setTechnologies(technologies)
 Wappalyzer.setCategories(categories)
 
-const detections = Wappalyzer.analyze({
+Wappalyzer.analyze({
   url: 'https://example.github.io/',
   meta: { generator: ['WordPress'] },
   headers: { server: ['Nginx'] },
   scripts: ['jquery-3.0.0.js'],
   cookies: { awselb: [''] },
   html: '<div ng-app="">'
-})
-
-const results = Wappalyzer.resolve(detections)
+}).then((detections) => {
+  const results = Wappalyzer.resolve(detections)
 
-console.log(results)
+  console.log(results)
+})
 ```

src/drivers/npm/driver.js

@@ -9,6 +9,10 @@ const Wappalyzer = require('./wappalyzer')
 const { setTechnologies, setCategories, analyze, analyzeManyToMany, resolve } =
   Wappalyzer
 
+function next() {
+  return new Promise((resolve) => setImmediate(resolve))
+}
+
 const { AWS_LAMBDA_FUNCTION_NAME, CHROMIUM_BIN, CHROMIUM_DATA_DIR } =
   process.env
 
@@ -49,23 +53,39 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms))
 }
 
-function analyzeJs(js) {
+async function analyzeJs(js) {
   return Array.prototype.concat.apply(
     [],
-    js.map(({ name, chain, value }) =>
-      analyzeManyToMany(
+    await Promise.all(
+      js.map(async ({ name, chain, value }) => {
+        await next()
+
+        return analyzeManyToMany(
           Wappalyzer.technologies.find(({ name: _name }) => name === _name),
           'js',
           { [chain]: [value] }
         )
+      })
     )
   )
 }
 
-function analyzeDom(dom) {
+async function analyzeDom(dom) {
   return Array.prototype.concat.apply(
     [],
-    dom.map(({ name, selector, exists, text, property, attribute, value }) => {
+    await Promise.all(
+      dom.map(
+        async ({
+          name,
+          selector,
+          exists,
+          text,
+          property,
+          attribute,
+          value,
+        }) => {
+          await next()
+
           const technology = Wappalyzer.technologies.find(
             ({ name: _name }) => name === _name
           )
@@ -89,13 +109,19 @@ function analyzeDom(dom) {
           }
 
           if (typeof attribute !== 'undefined') {
-        return analyzeManyToMany(technology, `dom.attributes.${attribute}`, {
+            return analyzeManyToMany(
+              technology,
+              `dom.attributes.${attribute}`,
+              {
                 [selector]: [value],
-        })
+              }
+            )
           }
 
           return []
-    })
+        }
+      )
+    )
   )
 }
 
@@ -343,10 +369,10 @@ class Site {
           if (!xhrDebounce.includes(hostname)) {
             xhrDebounce.push(hostname)
 
-            setTimeout(() => {
+            setTimeout(async () => {
               xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1)
 
-              this.onDetect(analyze({ xhr: hostname }))
+              this.onDetect(await analyze({ xhr: hostname }))
             }, 1000)
           }
         }
@@ -402,7 +428,7 @@ class Site {
               ? response.securityDetails().issuer()
               : ''
 
-            this.onDetect(analyze({ headers, certIssuer }))
+            this.onDetect(await analyze({ headers, certIssuer }))
 
             await this.emit('response', { page, response, headers, certIssuer })
           }
@@ -720,7 +746,7 @@ class Site {
           return dns
         }, {})
 
-        this.onDetect(analyze({ dns: this.dns }))
+        this.onDetect(await analyze({ dns: this.dns }))
       }
 
       // Validate response
@@ -736,10 +762,10 @@ class Site {
         throw new Error('No response from server')
       }
 
-      this.onDetect(analyzeDom(dom))
-      this.onDetect(analyzeJs(js))
+      this.onDetect(await analyzeDom(dom))
+      this.onDetect(await analyzeJs(js))
       this.onDetect(
-        analyze({
+        await analyze({
           url,
           cookies,
           html,
@@ -874,7 +900,7 @@ class Site {
 
         this.log(`get ${path}: ok`)
 
-        this.onDetect(analyze({ [file]: body }))
+        this.onDetect(await analyze({ [file]: body }))
       } catch (error) {
         this.error(`get ${path}: ${error.message || error}`)
       }

src/drivers/webextension/js/driver.js

@@ -1,6 +1,6 @@
 'use strict'
 /* eslint-env browser */
-/* globals chrome, Wappalyzer, Utils */
+/* globals chrome, Wappalyzer, Utils, next */
 
 const {
   setTechnologies,
@@ -161,17 +161,21 @@ const Driver = {
    * @param {String} url
    * @param {Array} js
    */
-  analyzeJs(url, js) {
+  async analyzeJs(url, js) {
     return Driver.onDetect(
       url,
       Array.prototype.concat.apply(
         [],
-        js.map(({ name, chain, value }) =>
-          analyzeManyToMany(
+        await Promise.all(
+          js.map(async ({ name, chain, value }) => {
+            await next()
+
+            return analyzeManyToMany(
               Wappalyzer.technologies.find(({ name: _name }) => name === _name),
               'js',
               { [chain]: [value] }
             )
+          })
         )
       )
     )
@@ -182,16 +186,19 @@ const Driver = {
    * @param {String} url
    * @param {Array} dom
    */
-  analyzeDom(url, dom) {
+  async analyzeDom(url, dom) {
     return Driver.onDetect(
       url,
       Array.prototype.concat.apply(
         [],
+        await Promise.all(
           dom.map(
-          (
+            async (
               { name, selector, exists, text, property, attribute, value },
               index
             ) => {
+              await next()
+
               const technology = Wappalyzer.technologies.find(
                 ({ name: _name }) => name === _name
               )
@@ -233,6 +240,7 @@ const Driver = {
           )
         )
       )
+    )
   },
 
   /**
@@ -304,7 +312,9 @@ const Driver = {
             )
           })
 
-          Driver.onDetect(request.url, analyze({ headers })).catch(Driver.error)
+          Driver.onDetect(request.url, await analyze({ headers })).catch(
+            Driver.error
+          )
         }
       } catch (error) {
         Driver.error(error)
@@ -332,12 +342,13 @@ const Driver = {
     if (!xhrDebounce.includes(hostname)) {
       xhrDebounce.push(hostname)
 
-      setTimeout(() => {
+      setTimeout(async () => {
         xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1)
 
-        Driver.onDetect(request.originUrl, analyze({ xhr: hostname })).catch(
-          Driver.error
-        )
+        Driver.onDetect(
+          request.originUrl,
+          await analyze({ xhr: hostname })
+        ).catch(Driver.error)
       }, 1000)
     }
   },
@@ -364,7 +375,12 @@ const Driver = {
         {}
       )
 
-      await Driver.onDetect(url, analyze({ url, ...items }), language, true)
+      await Driver.onDetect(
+        url,
+        await analyze({ url, ...items }),
+        language,
+        true
+      )
     } catch (error) {
       Driver.error(error)
     }

src/wappalyzer.js

@@ -1,5 +1,13 @@
 'use strict'
 
+function next() {
+  return new Promise((resolve) =>
+    (typeof setImmediate !== 'undefined'
+      ? setImmediate
+      : requestAnimationFrame)(resolve)
+  )
+}
+
 function toArray(value) {
   return Array.isArray(value) ? value : [value]
 }
@@ -187,7 +195,7 @@ const Wappalyzer = {
    * Initialize analyzation.
    * @param {*} param0
    */
-  analyze({
+  async analyze({
     url,
     xhr,
     html,
@@ -208,8 +216,11 @@ const Wappalyzer = {
 
     try {
       const detections = flatten(
-        Wappalyzer.technologies.map((technology) =>
-          flatten([
+        await Promise.all(
+          Wappalyzer.technologies.map(async (technology) => {
+            await next()
+
+            return flatten([
               oo(technology, 'url', url),
               oo(technology, 'xhr', xhr),
               oo(technology, 'html', html),
@@ -222,6 +233,7 @@ const Wappalyzer = {
               mm(technology, 'headers', headers),
               mm(technology, 'dns', dns),
             ])
+          })
         )
       ).filter((technology) => technology)