From bfc9278b5cb95b1700040ddc5a6ef260c94b4211 Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Tue, 26 Oct 2021 15:06:13 +1100 Subject: [PATCH] Add DHL detection. Add Shipping Carriers category. Add "text" detection method. Add "requiresCategory" field. --- README.md | 9 ++++ schema.json | 26 +++++++++++ src/categories.json | 9 +++- src/drivers/npm/driver.js | 45 ++++++++++++++++--- .../webextension/_locales/ca/messages.json | 3 +- .../webextension/_locales/de/messages.json | 3 +- .../webextension/_locales/el/messages.json | 3 +- .../webextension/_locales/en/messages.json | 3 +- .../webextension/_locales/es/messages.json | 3 +- .../webextension/_locales/fa/messages.json | 3 +- .../webextension/_locales/fr/messages.json | 3 +- .../webextension/_locales/gl_ES/messages.json | 3 +- .../webextension/_locales/gr/messages.json | 3 +- .../webextension/_locales/id/messages.json | 3 +- .../webextension/_locales/it/messages.json | 3 +- .../webextension/_locales/ja/messages.json | 3 +- .../webextension/_locales/ko/messages.json | 3 +- .../webextension/_locales/nl/messages.json | 3 +- .../webextension/_locales/pl/messages.json | 3 +- .../webextension/_locales/pt/messages.json | 3 +- .../webextension/_locales/pt_BR/messages.json | 3 +- .../webextension/_locales/ro/messages.json | 3 +- .../webextension/_locales/ru/messages.json | 3 +- .../webextension/_locales/sk/messages.json | 3 +- .../webextension/_locales/tr/messages.json | 3 +- .../webextension/_locales/uk/messages.json | 3 +- .../webextension/_locales/uz/messages.json | 3 +- .../webextension/_locales/zh_CN/messages.json | 3 +- .../webextension/_locales/zh_TW/messages.json | 3 +- src/drivers/webextension/images/icons/DHL.svg | 5 +++ src/technologies/d.json | 13 +++++- src/wappalyzer.js | 42 +++++++++++++++-- yarn.lock | 6 +-- 33 files changed, 189 insertions(+), 41 deletions(-) create mode 100644 src/drivers/webextension/images/icons/DHL.svg diff --git a/README.md b/README.md index 9d8c4f7ac..5f8ddaf08 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ Patterns (regular expressions) are kept in [`src/technologies/`](https://github. "X-Powered-By": "Example" }, "html": "]example\\.css", + "text": "\bexample\b", "css": "\\.example-class", "robots": "Disallow: /unique-path/", "implies": "PHP\\;confidence:50", @@ -343,6 +344,14 @@ Plus any of: "<a [^>]*href=\"index.html" + + text + String | Array + + Matches plain text. Should only be used in very specific cases where other methods can't be used. + + \bexample\b + css String | Array diff --git a/schema.json b/schema.json index 1ee29fb7a..4dccf1808 100644 --- a/schema.json +++ b/schema.json @@ -113,6 +113,19 @@ } ] }, + "text": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/non-empty-non-blank-string" + } + }, + { + "$ref": "#/definitions/non-empty-non-blank-string" + } + ] + }, "css": { "oneOf": [ { @@ -204,6 +217,19 @@ } ] }, + "requiresCategory": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "number" + } + }, + { + "type": "number" + } + ] + }, "meta": { "type": "object", "additionalProperties": false, diff --git a/src/categories.json b/src/categories.json index d6e8c8bf7..fd877e01d 100644 --- a/src/categories.json +++ b/src/categories.json @@ -677,5 +677,12 @@ ], "name": "Cart abandonment", "priority": 9 + }, + "99": { + "groups": [ + 1 + ], + "name": "Shipping carriers", + "priority": 9 } -} \ No newline at end of file +} diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 3a345e60d..a8441831b 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -1,3 +1,4 @@ +/* eslint-disable unicorn/prefer-text-content */ const { URL } = require('url') const fs = require('fs') const dns = require('dns').promises @@ -692,6 +693,7 @@ class Site { } let links = [] + let text = '' let css = '' let scriptSrc = [] let scripts = [] @@ -726,6 +728,21 @@ class Site { 'Timeout (links)' ) + // Text + text = await this.promiseTimeout( + ( + await this.promiseTimeout( + page.evaluateHandle(() => + document.body.innerText.replace(/\s+/g, ' ') + ), + { jsonValue: () => '' }, + 'Timeout (text)' + ) + ).jsonValue(), + '', + 'Timeout (text)' + ) + // CSS css = await this.promiseTimeout( ( @@ -829,6 +846,7 @@ class Site { this.cache[url.href] = { page, html, + text, cookies, scripts, scriptSrc, @@ -845,6 +863,7 @@ class Site { url, cookies, html, + text, css, scripts, scriptSrc, @@ -1120,19 +1139,30 @@ class Site { if (this.cache[url.href]) { const resolved = resolve(this.detections) - const requires = Wappalyzer.requires.filter(({ name, technologies }) => - resolved.some(({ name: _name }) => _name === name) - ) + const requires = [ + ...Wappalyzer.requires.filter(({ name }) => + resolved.some(({ name: _name }) => _name === name) + ), + ...Wappalyzer.categoryRequires.filter(({ categoryId }) => + resolved.some(({ categories }) => + categories.some(({ id }) => id === categoryId) + ) + ), + ] await Promise.all( - requires.map(async ({ name, technologies }) => { + requires.map(async ({ name, categoryId, technologies }) => { + const id = categoryId + ? `category:${categoryId}` + : `technology:${name}` + this.analyzedRequires[url.href] = this.analyzedRequires[url.href] || [] - if (!this.analyzedRequires[url.href].includes(name)) { - this.analyzedRequires[url.href].push(name) + if (!this.analyzedRequires[url.href].includes(id)) { + this.analyzedRequires[url.href].push(id) - const { page, cookies, html, css, scripts, scriptSrc, meta } = + const { page, cookies, html, text, css, scripts, scriptSrc, meta } = this.cache[url.href] const js = await this.promiseTimeout( @@ -1157,6 +1187,7 @@ class Site { url, cookies, html, + text, css, scripts, scriptSrc, diff --git a/src/drivers/webextension/_locales/ca/messages.json b/src/drivers/webextension/_locales/ca/messages.json index 6a527728e..c541a783c 100644 --- a/src/drivers/webextension/_locales/ca/messages.json +++ b/src/drivers/webextension/_locales/ca/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/de/messages.json b/src/drivers/webextension/_locales/de/messages.json index 0d43ffce7..5f2cb397f 100644 --- a/src/drivers/webextension/_locales/de/messages.json +++ b/src/drivers/webextension/_locales/de/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/el/messages.json b/src/drivers/webextension/_locales/el/messages.json index 9088e833e..1018ef77a 100644 --- a/src/drivers/webextension/_locales/el/messages.json +++ b/src/drivers/webextension/_locales/el/messages.json @@ -188,5 +188,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/en/messages.json b/src/drivers/webextension/_locales/en/messages.json index 2ab7c71f1..13435c873 100644 --- a/src/drivers/webextension/_locales/en/messages.json +++ b/src/drivers/webextension/_locales/en/messages.json @@ -190,5 +190,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/es/messages.json b/src/drivers/webextension/_locales/es/messages.json index 090af5f4f..a080da570 100644 --- a/src/drivers/webextension/_locales/es/messages.json +++ b/src/drivers/webextension/_locales/es/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/fa/messages.json b/src/drivers/webextension/_locales/fa/messages.json index eb5c482f5..47fb224e1 100644 --- a/src/drivers/webextension/_locales/fa/messages.json +++ b/src/drivers/webextension/_locales/fa/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/fr/messages.json b/src/drivers/webextension/_locales/fr/messages.json index ba013e99f..59e91a111 100644 --- a/src/drivers/webextension/_locales/fr/messages.json +++ b/src/drivers/webextension/_locales/fr/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/gl_ES/messages.json b/src/drivers/webextension/_locales/gl_ES/messages.json index 56db3d701..8c561e4db 100644 --- a/src/drivers/webextension/_locales/gl_ES/messages.json +++ b/src/drivers/webextension/_locales/gl_ES/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/gr/messages.json b/src/drivers/webextension/_locales/gr/messages.json index 3289d605f..dcbf12f89 100644 --- a/src/drivers/webextension/_locales/gr/messages.json +++ b/src/drivers/webextension/_locales/gr/messages.json @@ -188,5 +188,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/id/messages.json b/src/drivers/webextension/_locales/id/messages.json index 654bcd42e..1b5074750 100644 --- a/src/drivers/webextension/_locales/id/messages.json +++ b/src/drivers/webextension/_locales/id/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/it/messages.json b/src/drivers/webextension/_locales/it/messages.json index 9666b2646..46e7d32c9 100644 --- a/src/drivers/webextension/_locales/it/messages.json +++ b/src/drivers/webextension/_locales/it/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/ja/messages.json b/src/drivers/webextension/_locales/ja/messages.json index f01defde3..3f2c9b2bc 100644 --- a/src/drivers/webextension/_locales/ja/messages.json +++ b/src/drivers/webextension/_locales/ja/messages.json @@ -190,5 +190,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/ko/messages.json b/src/drivers/webextension/_locales/ko/messages.json index 983ea1906..bbacf366d 100644 --- a/src/drivers/webextension/_locales/ko/messages.json +++ b/src/drivers/webextension/_locales/ko/messages.json @@ -190,5 +190,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/nl/messages.json b/src/drivers/webextension/_locales/nl/messages.json index 09bcd02a1..4b451d9a2 100644 --- a/src/drivers/webextension/_locales/nl/messages.json +++ b/src/drivers/webextension/_locales/nl/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/pl/messages.json b/src/drivers/webextension/_locales/pl/messages.json index 8305ca91d..663be5092 100644 --- a/src/drivers/webextension/_locales/pl/messages.json +++ b/src/drivers/webextension/_locales/pl/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/pt/messages.json b/src/drivers/webextension/_locales/pt/messages.json index d1043bd3c..2a2620ab3 100644 --- a/src/drivers/webextension/_locales/pt/messages.json +++ b/src/drivers/webextension/_locales/pt/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/pt_BR/messages.json b/src/drivers/webextension/_locales/pt_BR/messages.json index 840fd058c..ad36017e7 100644 --- a/src/drivers/webextension/_locales/pt_BR/messages.json +++ b/src/drivers/webextension/_locales/pt_BR/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/ro/messages.json b/src/drivers/webextension/_locales/ro/messages.json index 7d0b5f351..d61f2d843 100644 --- a/src/drivers/webextension/_locales/ro/messages.json +++ b/src/drivers/webextension/_locales/ro/messages.json @@ -188,5 +188,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/ru/messages.json b/src/drivers/webextension/_locales/ru/messages.json index 6195791bc..253661c8d 100644 --- a/src/drivers/webextension/_locales/ru/messages.json +++ b/src/drivers/webextension/_locales/ru/messages.json @@ -190,5 +190,6 @@ "categoryName95": { "message": "Управление цифровыми активами" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/sk/messages.json b/src/drivers/webextension/_locales/sk/messages.json index e1b4cfe56..e83b5f9f8 100644 --- a/src/drivers/webextension/_locales/sk/messages.json +++ b/src/drivers/webextension/_locales/sk/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/tr/messages.json b/src/drivers/webextension/_locales/tr/messages.json index d44e7458a..389e20683 100644 --- a/src/drivers/webextension/_locales/tr/messages.json +++ b/src/drivers/webextension/_locales/tr/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/uk/messages.json b/src/drivers/webextension/_locales/uk/messages.json index fef9913a5..7c91cfe5c 100644 --- a/src/drivers/webextension/_locales/uk/messages.json +++ b/src/drivers/webextension/_locales/uk/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/uz/messages.json b/src/drivers/webextension/_locales/uz/messages.json index d35f65272..67bb87640 100644 --- a/src/drivers/webextension/_locales/uz/messages.json +++ b/src/drivers/webextension/_locales/uz/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/zh_CN/messages.json b/src/drivers/webextension/_locales/zh_CN/messages.json index 6bbc8af7c..2ec846c36 100644 --- a/src/drivers/webextension/_locales/zh_CN/messages.json +++ b/src/drivers/webextension/_locales/zh_CN/messages.json @@ -188,5 +188,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/_locales/zh_TW/messages.json b/src/drivers/webextension/_locales/zh_TW/messages.json index 4882a8e52..5d007922e 100644 --- a/src/drivers/webextension/_locales/zh_TW/messages.json +++ b/src/drivers/webextension/_locales/zh_TW/messages.json @@ -192,5 +192,6 @@ "categoryName95": { "message": "Digital asset management" }, "categoryName96": { "message": "Content curation" }, "categoryName97": { "message": "Customer data platform" }, - "categoryName98": { "message": "Cart abandonment" } + "categoryName98": { "message": "Cart abandonment" }, + "categoryName99": { "message": "Shipping carriers" } } diff --git a/src/drivers/webextension/images/icons/DHL.svg b/src/drivers/webextension/images/icons/DHL.svg new file mode 100644 index 000000000..4bc4afdba --- /dev/null +++ b/src/drivers/webextension/images/icons/DHL.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/technologies/d.json b/src/technologies/d.json index db7358325..b32179031 100644 --- a/src/technologies/d.json +++ b/src/technologies/d.json @@ -30,6 +30,15 @@ }, "website": "https://derak.cloud" }, + "DHL": { + "cats": [ + 99 + ], + "text": "\\bDHL\\b", + "icon": "DHL.svg", + "requiresCategory": 6, + "website": "https://www.dhl.com" + }, "DHTMLX": { "cats": [ 59 @@ -417,7 +426,7 @@ "Digital Showroom": { "cats": [ 6 - ], + ], "description": "Digital Showroom is an ecommerce platform.", "icon": "Digital Showroom.png", "dom": "div.dd-showrom__layout", @@ -1075,4 +1084,4 @@ "scriptSrc": "deepMiner\\.js", "website": "https://github.com/deepwn/deepMiner" } -} \ No newline at end of file +} diff --git a/src/wappalyzer.js b/src/wappalyzer.js index 252fae5ca..fbc77625d 100644 --- a/src/wappalyzer.js +++ b/src/wappalyzer.js @@ -12,6 +12,7 @@ const Wappalyzer = { technologies: [], categories: [], requires: [], + categoryRequires: [], slugify: (string) => string @@ -24,6 +25,9 @@ const Wappalyzer = { [ ...Wappalyzer.technologies, ...Wappalyzer.requires.map(({ technologies }) => technologies).flat(), + ...Wappalyzer.categoryRequires + .map(({ technologies }) => technologies) + .flat(), ].find(({ name: _name }) => name === _name), getCategory: (id) => Wappalyzer.categories.find(({ id: _id }) => id === _id), @@ -205,6 +209,7 @@ const Wappalyzer = { url, xhr, html, + text, scripts, css, robots, @@ -234,6 +239,7 @@ const Wappalyzer = { oo(technology, 'url', url), oo(technology, 'xhr', xhr), oo(technology, 'html', html), + oo(technology, 'text', text), oo(technology, 'scripts', scripts), oo(technology, 'css', css), oo(technology, 'robots', robots), @@ -269,6 +275,7 @@ const Wappalyzer = { xhr, dom, html, + text, scripts, css, robots, @@ -283,6 +290,7 @@ const Wappalyzer = { implies, excludes, requires, + requiresCategory, icon, website, cpe, @@ -311,6 +319,7 @@ const Wappalyzer = { false ), html: transform(html), + text: transform(text), scripts: transform(scripts), css: transform(css), certIssuer: transform(certIssuer), @@ -329,6 +338,9 @@ const Wappalyzer = { requires: transform(requires).map(({ value }) => ({ name: value, })), + requiresCategory: transform(requiresCategory).map(({ value }) => ({ + id: value, + })), icon: icon || 'default.svg', website: website || null, cpe: cpe || null, @@ -356,8 +368,27 @@ const Wappalyzer = { technologies: Wappalyzer.requires[name], })) + Wappalyzer.technologies + .filter(({ requiresCategory }) => requiresCategory.length) + .forEach((technology) => + technology.requiresCategory.forEach(({ id }) => { + Wappalyzer.categoryRequires[id] = + Wappalyzer.categoryRequires[id] || [] + + Wappalyzer.categoryRequires[id].push(technology) + }) + ) + + Wappalyzer.categoryRequires = Object.keys(Wappalyzer.categoryRequires).map( + (id) => ({ + categoryId: parseInt(id, 10), + technologies: Wappalyzer.categoryRequires[id], + }) + ) + Wappalyzer.technologies = Wappalyzer.technologies.filter( - ({ requires }) => !requires.length + ({ requires, requiresCategory }) => + !requires.length && !requiresCategory.length ) }, @@ -391,7 +422,11 @@ const Wappalyzer = { return [] } - if (typeof patterns === 'string' || Array.isArray(patterns)) { + if ( + typeof patterns === 'string' || + typeof patterns === 'number' || + Array.isArray(patterns) + ) { patterns = { main: patterns } } @@ -421,6 +456,7 @@ const Wappalyzer = { ) } else { const { value, regex, confidence, version } = pattern + .toString() .split('\\;') .reduce((attrs, attr, i) => { if (i) { @@ -431,7 +467,7 @@ const Wappalyzer = { attrs[attr.shift()] = attr.join(':') } } else { - attrs.value = attr + attrs.value = typeof pattern === 'number' ? pattern : attr // Escape slashes in regular expression attrs.regex = new RegExp( diff --git a/yarn.lock b/yarn.lock index adb411e0f..bfe7a6371 100644 --- a/yarn.lock +++ b/yarn.lock @@ -475,9 +475,9 @@ callsites@^3.0.0: integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== caniuse-lite@^1.0.30001219: - version "1.0.30001228" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001228.tgz#bfdc5942cd3326fa51ee0b42fbef4da9d492a7fa" - integrity sha512-QQmLOGJ3DEgokHbMSA8cj2a+geXqmnpyOFT0lhQV6P3/YOJvGDEwoedcwxEQ30gJIwIIunHIicunJ2rzK5gB2A== + version "1.0.30001271" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001271.tgz" + integrity sha512-BBruZFWmt3HFdVPS8kceTBIguKxu4f99n5JNp06OlPD/luoAMIaIK5ieV5YjnBLH3Nysai9sxj9rpJj4ZisXOA== chalk@^2.0.0, chalk@^2.4.1: version "2.4.2"