Performance improvements

src/drivers/webextension/js/driver.js

@@ -121,10 +121,10 @@ const Driver = {
         'https://www.wappalyzer.com/installed/?utm_source=installed&utm_medium=extension&utm_campaign=wappalyzer'
       )
     } else if (version !== previous && upgradeMessage) {
-      // open(
+      open(
-      //   `https://www.wappalyzer.com/upgraded/?utm_source=upgraded&utm_medium=extension&utm_campaign=wappalyzer`,
+        `https://www.wappalyzer.com/upgraded/?utm_source=upgraded&utm_medium=extension&utm_campaign=wappalyzer`,
-      //   false
+        false
-      // )
+      )
     }
 
     await setOption('version', version)
@@ -209,6 +209,8 @@ const Driver = {
    * Wrapper for analyze
    */
   analyze(...args) {
+    Driver.log('analyze: Driver.analyze')
+
     return analyze(...args)
   },
 
@@ -404,6 +406,8 @@ const Driver = {
             )
           })
 
+          Driver.log('analyze: onWebRequestComplete | headers')
+
           Driver.onDetect(request.url, analyze({ headers })).catch(Driver.error)
         }
       } catch (error) {
@@ -430,6 +434,8 @@ const Driver = {
 
       const scripts = await response.text()
 
+      console.log('analyze: onScriptRequestComplete | scripts')
+
       Driver.onDetect(request.documentUrl, analyze({ scripts })).catch(
         Driver.error
       )
@@ -446,16 +452,10 @@ const Driver = {
     }
 
     let hostname
-
-    try {
-      ;({ hostname } = new URL(request.url))
-    } catch (error) {
-      return
-    }
-
     let originHostname
 
     try {
+      ;({ hostname } = new URL(request.url))
       ;({ hostname: originHostname } = new URL(request.originUrl))
     } catch (error) {
       return
@@ -476,6 +476,8 @@ const Driver = {
             xhrAnalyzed = {}
           }
 
+          console.log('analyze: onXhrRequestComplete | xhr', hostname)
+
           Driver.onDetect(
             request.originUrl || request.initiator,
             analyze({ xhr: hostname })
@@ -506,6 +508,8 @@ const Driver = {
 
       const technologies = getRequiredTechnologies(requires, categoryRequires)
 
+      Driver.log('analyze: onContentLoad | url, ...item')
+
       await Driver.onDetect(
         url,
         analyze({ url, ...items }, technologies),

src/technologies/u.json

@@ -67,7 +67,7 @@
     "icon": "UPS.svg",
     "requiresCategory": 6,
     "text": [
-      "\\bUPS\\b"
+      "\\b(?<!-)UPS\\b"
     ],
     "website": "https://www.ups.com"
   },
@@ -599,4 +599,4 @@
     },
     "website": "https://user.com"
   }
-}
+}

src/wappalyzer.js

@@ -4,6 +4,68 @@ function toArray(value) {
   return Array.isArray(value) ? value : [value]
 }
 
+const benchmarkEnabled = !!process.env.WAPPALYZER_BENCHMARK
+
+let benchmarks = []
+
+function benchmark(duration, pattern, value = '', technology) {
+  if (!benchmarkEnabled) {
+    return
+  }
+
+  benchmarks.push({
+    duration,
+    pattern: String(pattern.regex),
+    value: String(value).slice(0, 100),
+    valueLength: value.length,
+    technology: technology.name,
+  })
+}
+
+function benchmarkSummary() {
+  if (!benchmarkEnabled) {
+    return
+  }
+
+  const totalPatterns = Object.values(benchmarks).length
+  const totalDuration = Object.values(benchmarks).reduce(
+    (sum, { duration }) => sum + duration,
+    0
+  )
+
+  // eslint-disable-next-line no-console
+  console.log({
+    totalPatterns,
+    totalDuration,
+    averageDuration: Math.round(totalDuration / totalPatterns),
+    slowestTechnologies: Object.values(
+      benchmarks.reduce((benchmarks, { duration, technology }) => {
+        if (benchmarks[technology]) {
+          benchmarks[technology].duration += duration
+        } else {
+          benchmarks[technology] = { technology, duration }
+        }
+
+        return benchmarks
+      }, {})
+    )
+      .sort(({ duration: a }, { duration: b }) => (a > b ? -1 : 1))
+      .filter(({ duration }) => duration)
+      .slice(0, 5)
+      .reduce(
+        (technologies, { technology, duration }) => ({
+          ...technologies,
+          [technology]: duration,
+        }),
+        {}
+      ),
+    slowestPatterns: Object.values(benchmarks)
+      .sort(({ duration: a }, { duration: b }) => (a > b ? -1 : 1))
+      .filter(({ duration }) => duration)
+      .slice(0, 5),
+  })
+}
+
 const Wappalyzer = {
   technologies: [],
   categories: [],
@@ -200,52 +262,44 @@ const Wappalyzer = {
    * Initialize analyzation.
    * @param {*} param0
    */
-  analyze(
+  analyze(items, technologies = Wappalyzer.technologies) {
-    {
+    benchmarks = []
-      url,
+
-      xhr,
-      html,
-      text,
-      scripts,
-      css,
-      robots,
-      magento,
-      meta,
-      headers,
-      dns,
-      certIssuer,
-      cookies,
-      scriptSrc,
-    },
-    technologies = Wappalyzer.technologies
-  ) {
     const oo = Wappalyzer.analyzeOneToOne
     const om = Wappalyzer.analyzeOneToMany
     const mm = Wappalyzer.analyzeManyToMany
 
-    const flatten = (array) => Array.prototype.concat.apply([], array)
+    const relations = {
+      url: oo,
+      xhr: oo,
+      html: oo,
+      text: oo,
+      scripts: oo,
+      css: oo,
+      robots: oo,
+      magento: oo,
+      certIssuer: oo,
+      scriptSrc: om,
+      cookies: mm,
+      meta: mm,
+      headers: mm,
+      dns: mm,
+    }
 
     try {
-      const detections = flatten(
+      const detections = technologies
-        technologies.map((technology) => {
+        .map((technology) =>
-          return flatten([
+          Object.keys(relations)
-            oo(technology, 'url', url),
+            .map(
-            oo(technology, 'xhr', xhr),
+              (type) =>
-            oo(technology, 'html', html),
+                items[type] && relations[type](technology, type, items[type])
-            oo(technology, 'text', text),
+            )
-            oo(technology, 'scripts', scripts),
+            .flat()
-            oo(technology, 'css', css),
+        )
-            oo(technology, 'robots', robots),
+        .flat()
-            oo(technology, 'magento', magento),
+        .filter((technology) => technology)
-            oo(technology, 'certIssuer', certIssuer),
+
-            om(technology, 'scriptSrc', scriptSrc),
+      benchmarkSummary()
-            mm(technology, 'cookies', cookies),
-            mm(technology, 'meta', meta),
-            mm(technology, 'headers', headers),
-            mm(technology, 'dns', dns),
-          ])
-        })
-      ).filter((technology) => technology)
 
       return detections
     } catch (error) {
@@ -461,9 +515,15 @@ const Wappalyzer = {
           } else {
             attrs.value = typeof pattern === 'number' ? pattern : attr
 
-            // Escape slashes in regular expression
             attrs.regex = new RegExp(
-              isRegex ? attr.replace(/\//g, '\\/') : '',
+              isRegex
+                ? attr
+                    // Escape slashes
+                    .replace(/\//g, '\\/')
+                    // Optimise quantifiers for long strings
+                    .replace(/\+/g, '{1,250}')
+                    .replace(/\*/g, '{0,250}')
+                : '',
               'i'
             )
           }
@@ -488,6 +548,8 @@ const Wappalyzer = {
    */
   analyzeOneToOne(technology, type, value) {
     return technology[type].reduce((technologies, pattern) => {
+      const startTime = Date.now()
+
       if (pattern.regex.test(value)) {
         technologies.push({
           technology,
@@ -496,6 +558,8 @@ const Wappalyzer = {
         })
       }
 
+      benchmark(Date.now() - startTime, pattern, value, technology)
+
       return technologies
     }, [])
   },
@@ -511,6 +575,8 @@ const Wappalyzer = {
       const patterns = technology[type] || []
 
       patterns.forEach((pattern) => {
+        const startTime = Date.now()
+
         if (pattern.regex.test(value)) {
           technologies.push({
             technology,
@@ -518,6 +584,8 @@ const Wappalyzer = {
             version: Wappalyzer.resolveVersion(pattern, value),
           })
         }
+
+        benchmark(Date.now() - startTime, pattern, value, technology)
       })
 
       return technologies
@@ -544,6 +612,8 @@ const Wappalyzer = {
         )
 
         values.forEach((value) => {
+          const startTime = Date.now()
+
           if (pattern.regex.test(value)) {
             technologies.push({
               technology,
@@ -551,6 +621,8 @@ const Wappalyzer = {
               version: Wappalyzer.resolveVersion(pattern, value),
             })
           }
+
+          benchmark(Date.now() - startTime, pattern, value, technology)
         })
       })