Track rootPath detections

2 years ago · 670614082e
parent c7417f1b89
commit 670614082e
9 changed files with 128 additions and 84 deletions
--- a/src/drivers/npm/driver.js
+++ b/src/drivers/npm/driver.js
@ -1126,6 +1126,7 @@ class Site {
          website,
          cpe,
          categories,
+          rootPath,
        }) => ({
          slug,
          name,
@ -1140,6 +1141,7 @@ class Site {
            slug,
            name,
          })),
+          rootPath,
        })
      ),
      patterns,
@ -1272,6 +1274,15 @@ class Site {
          ) === index
      )

+    // Track if technology was identified on website's root path
+    detections.forEach(({ technology: { name } }) => {
+      const detection = this.detections.find(
+        ({ technology: { name: _name } }) => name === _name
+      )
+
+      detection.rootPath = detection.rootPath || url.pathname === '/'
+    })
+
    if (this.cache[url.href]) {
      const resolved = resolve(this.detections)

--- a/src/drivers/npm/package.json
+++ b/src/drivers/npm/package.json
@ -13,7 +13,7 @@
    "software"
  ],
  "homepage": "https://www.wappalyzer.com/",
-  "version": "6.10.46",
+  "version": "6.10.47",
  "author": "Wappalyzer",
  "license": "MIT",
  "repository": {
--- a/src/drivers/webextension/js/driver.js
+++ b/src/drivers/webextension/js/driver.js
@ -101,10 +101,12 @@ const Driver = {
        )
      }
    } else if (version !== previous && upgradeMessage) {
+      /*
      open(
        `https://www.wappalyzer.com/upgraded/?utm_source=upgraded&utm_medium=extension&utm_campaign=wappalyzer`,
        false
      )
+      */
    }

    initDone()
@ -152,6 +154,13 @@ const Driver = {
        }
      }

+      Object.keys(technologies).forEach((name) => {
+        delete technologies[name].description
+        delete technologies[name].cpe
+        delete technologies[name].pricing
+        delete technologies[name].website
+      })
+
      setTechnologies(technologies)
      setCategories(categories)
    } catch (error) {
@ -560,7 +569,7 @@ const Driver = {

    url = url.split('#')[0]

-    const { hostname } = new URL(url)
+    const { hostname, pathname } = new URL(url)

    // Cache detections
    const cache = (Driver.cache.hostnames[hostname] = {
@ -605,6 +614,39 @@ const Driver = {
        return detection
      })

+    // Track if technology was identified on website's root path
+    detections.forEach(({ technology: { name } }) => {
+      const detection = cache.detections.find(
+        ({ technology: { name: _name } }) => name === _name
+      )
+
+      detection.rootPath = detection.rootPath || pathname === '/'
+    })
+
+    const resolved = resolve(cache.detections).map((detection) => detection)
+
+    // Look for technologies that require other technologies to be present on the page
+    const requires = [
+      ...Wappalyzer.requires.filter(({ name }) =>
+        resolved.some(({ name: _name }) => _name === name)
+      ),
+      ...Wappalyzer.categoryRequires.filter(({ categoryId }) =>
+        resolved.some(({ categories }) =>
+          categories.some(({ id }) => id === categoryId)
+        )
+      ),
+    ]
+
+    try {
+      await Driver.content(url, 'analyzeRequires', [url, requires])
+    } catch (error) {
+      // Continue
+    }
+
+    await Driver.setIcon(url, resolved)
+
+    await Driver.ping()
+
    cache.hits += incrementHits ? 1 : 0
    cache.language = cache.language || language

@ -628,6 +670,7 @@ const Driver = {
        return hostnames
      }, {})

+    // Save cache
    await setOption(
      'hostnames',
      Object.keys(Driver.cache.hostnames).reduce(
@ -642,6 +685,7 @@ const Driver = {
                  technology: { name: technology },
                  pattern: { regex, confidence },
                  version,
+                  rootPath,
                  lastUrl,
                }) => ({
                  technology,
@ -650,6 +694,7 @@ const Driver = {
                    confidence,
                  },
                  version,
+                  rootPath,
                  lastUrl,
                })
              ),
@ -659,30 +704,7 @@ const Driver = {
      )
    )

-    const resolved = resolve(cache.detections).map((detection) => detection)
-
-    const requires = [
-      ...Wappalyzer.requires.filter(({ name }) =>
-        resolved.some(({ name: _name }) => _name === name)
-      ),
-      ...Wappalyzer.categoryRequires.filter(({ categoryId }) =>
-        resolved.some(({ categories }) =>
-          categories.some(({ id }) => id === categoryId)
-        )
-      ),
-    ]
-
-    try {
-      await Driver.content(url, 'analyzeRequires', [url, requires])
-    } catch (error) {
-      // Continue
-    }
-
-    await Driver.setIcon(url, resolved)
-
    Driver.log({ hostname, technologies: resolved })
-
-    await Driver.ping()
  },

  /**
@ -930,11 +952,12 @@ const Driver = {
          if (!hostnameIgnoreList.test(hostname) && hits) {
            urls[url] = urls[url] || {
              technologies: resolve(detections).reduce(
-                (technologies, { name, confidence, version }) => {
+                (technologies, { name, confidence, version, rootPath }) => {
                  if (confidence === 100) {
                    technologies[name] = {
                      version,
                      hits,
+                      rootPath,
                    }
                  }

--- a/src/drivers/webextension/manifest-v2.json
+++ b/src/drivers/webextension/manifest-v2.json
@ -4,7 +4,7 @@
  "author": "Wappalyzer",
  "homepage_url": "https://www.wappalyzer.com/",
  "description": "Identify web technologies",
-  "version": "6.10.46",
+  "version": "6.10.47",
  "default_locale": "en",
  "manifest_version": 2,
  "icons": {
--- a/src/drivers/webextension/manifest-v3.json
+++ b/src/drivers/webextension/manifest-v3.json
@ -4,7 +4,7 @@
  "author": "Wappalyzer",
  "homepage_url": "https://www.wappalyzer.com/",
  "description": "Identify web technologies",
-  "version": "6.10.46",
+  "version": "6.10.47",
  "default_locale": "en",
  "manifest_version": 3,
  "icons": {
--- a/src/package.json
+++ b/src/package.json
@ -13,7 +13,7 @@
    "software"
  ],
  "homepage": "https://www.wappalyzer.com/",
-  "version": "6.10.46",
+  "version": "6.10.47",
  "author": "Wappalyzer",
  "license": "MIT",
  "repository": {
--- a/src/technologies/c.json
+++ b/src/technologies/c.json
@ -147,6 +147,35 @@
    },
    "website": "http://dragonflycms.org"
  },
+  "CRM+": {
+    "cats": [
+      53
+    ],
+    "description": "CRM+ is a German CRM software product building on Vtiger with GDPR-compliant extensions and improvements.",
+    "dom": {
+      "div.footer > div.floatRight ": {
+        "text": "Powered by Brainformatik GmbH"
+      }
+    },
+    "icon": "CRM+.png",
+    "implies": [
+      "MariaDB",
+      "amCharts",
+      "Sentry",
+      "Vtiger"
+    ],
+    "pricing": [
+      "mid",
+      "recurring",
+      "poa",
+      "freemium"
+    ],
+    "requires": [
+      "Apache",
+      "PHP"
+    ],
+    "website": "https://www.brainformatik.com"
+  },
  "CS Cart": {
    "cats": [
      6
@ -3366,35 +3395,6 @@
    ],
    "website": "http://criteo.com"
  },
-  "CRM+": {
-    "cats": [
-      53
-    ],
-    "description": "CRM+ is a German CRM software product building on Vtiger with GDPR-compliant extensions and improvements.",
-    "dom": {
-      "div.footer > div.floatRight ": {
-          "text": "Powered by Brainformatik GmbH"
-      }
-    },
-    "icon": "CRM+.png",
-    "implies": [
-      "MariaDB",
-      "amCharts",
-      "Sentry",
-      "Vtiger"
-    ],
-    "requires": [
-      "Apache",
-      "PHP"
-    ],
-    "pricing": [
-      "mid",
-      "recurring",
-      "poa",
-      "freemium"
-    ],
-    "website": "https://www.brainformatik.com"
-  },
  "Crobox": {
    "cats": [
      5
--- a/src/technologies/v.json
+++ b/src/technologies/v.json
@ -140,17 +140,6 @@
    "scriptSrc": "cdn\\.pushcrew\\.\\w+",
    "website": "https://vwo.com/engage"
  },
-  "vxe-table": {
-    "cats": [
-      59
-    ],
-    "description": "vxe-table is a Vue.js based PC form component, support add, delete, change, virtual scroll, lazy load, shortcut menu, data validation, tree structure, print export, form rendering, data paging, virtual list, modal window, custom template, renderer, flexible configuration items, extension interface.",
-    "icon": "vxe-table.png",
-    "requires": "Vue.js",
-    "dom": "div[class*='vxe-table']",
-    "oss": true,
-    "website": "https://vxetable.cn"
-  },
  "Vaadin": {
    "cats": [
      18
@ -770,8 +759,8 @@
      }
    },
    "icon": "Visx.svg",
-    "requires": "React",
    "oss": true,
+    "requires": "React",
    "website": "https://airbnb.io/visx/"
  },
  "Vitals": {
@ -1108,5 +1097,16 @@
      "generator": "vibecommerce"
    },
    "website": "http://vibecommerce.com.br"
+  },
+  "vxe-table": {
+    "cats": [
+      59
+    ],
+    "description": "vxe-table is a Vue.js based PC form component, support add, delete, change, virtual scroll, lazy load, shortcut menu, data validation, tree structure, print export, form rendering, data paging, virtual list, modal window, custom template, renderer, flexible configuration items, extension interface.",
+    "dom": "div[class*='vxe-table']",
+    "icon": "vxe-table.png",
+    "oss": true,
+    "requires": "Vue.js",
+    "website": "https://vxetable.cn"
  }
-}
+}
--- a/src/wappalyzer.js
+++ b/src/wappalyzer.js
@ -104,24 +104,32 @@ const Wappalyzer = {
      ) {
        let version = ''
        let confidence = 0
+        let rootPath

        detections
-          .filter(({ technology }) => technology)
+          .filter(
+            ({ technology: _technology }) =>
+              _technology && _technology.name === technology.name
+          )
          .forEach(
-            ({ technology: { name }, pattern, version: _version = '' }) => {
-              if (name === technology.name) {
-                confidence = Math.min(100, confidence + pattern.confidence)
-                version =
-                  _version.length > version.length &&
-                  _version.length <= 15 &&
-                  (parseInt(_version, 10) || 0) < 10000 // Ignore long numeric strings like timestamps
-                    ? _version
-                    : version
-              }
+            ({
+              technology: { name },
+              pattern,
+              version: _version = '',
+              rootPath: _rootPath,
+            }) => {
+              confidence = Math.min(100, confidence + pattern.confidence)
+              version =
+                _version.length > version.length &&
+                _version.length <= 15 &&
+                (parseInt(_version, 10) || 0) < 10000 // Ignore long numeric strings like timestamps
+                  ? _version
+                  : version
+              rootPath = rootPath || _rootPath || undefined
            }
          )

-        resolved.push({ technology, confidence, version, lastUrl })
+        resolved.push({ technology, confidence, version, rootPath, lastUrl })
      }

      return resolved
@ -152,6 +160,7 @@ const Wappalyzer = {
          },
          confidence,
          version,
+          rootPath,
          lastUrl,
        }) => ({
          name,
@ -164,6 +173,7 @@ const Wappalyzer = {
          website,
          pricing,
          cpe,
+          rootPath,
          lastUrl,
        })
      )
@ -360,7 +370,7 @@ const Wappalyzer = {

      technologies.push({
        name,
-        description,
+        description: description || null,
        categories: cats || [],
        slug: Wappalyzer.slugify(name),
        url: transform(url),