Add DHL detection. Add Shipping Carriers category. Add "text" detection method. Add "requiresCategory" field.

main
Elbert Alias 3 years ago
parent 07459dbef4
commit bfc9278b5c

@ -91,6 +91,7 @@ Patterns (regular expressions) are kept in [`src/technologies/`](https://github.
"X-Powered-By": "Example"
},
"html": "<link[^>]example\\.css",
"text": "\bexample\b",
"css": "\\.example-class",
"robots": "Disallow: /unique-path/",
"implies": "PHP\\;confidence:50",
@ -343,6 +344,14 @@ Plus any of:
</td>
<td><code>"&lt;a [^&gt;]*href=\"index.html"</code></td>
</tr>
<tr>
<td><code>text</code></td>
<td>String | Array</td>
<td>
Matches plain text. Should only be used in very specific cases where other methods can't be used.
</td>
<td><code>\bexample\b</code></td>
</tr>
<tr>
<td><code>css</code></td>
<td>String | Array</td>

@ -113,6 +113,19 @@
}
]
},
"text": {
"oneOf": [
{
"type": "array",
"items": {
"$ref": "#/definitions/non-empty-non-blank-string"
}
},
{
"$ref": "#/definitions/non-empty-non-blank-string"
}
]
},
"css": {
"oneOf": [
{
@ -204,6 +217,19 @@
}
]
},
"requiresCategory": {
"oneOf": [
{
"type": "array",
"items": {
"type": "number"
}
},
{
"type": "number"
}
]
},
"meta": {
"type": "object",
"additionalProperties": false,

@ -677,5 +677,12 @@
],
"name": "Cart abandonment",
"priority": 9
},
"99": {
"groups": [
1
],
"name": "Shipping carriers",
"priority": 9
}
}
}

@ -1,3 +1,4 @@
/* eslint-disable unicorn/prefer-text-content */
const { URL } = require('url')
const fs = require('fs')
const dns = require('dns').promises
@ -692,6 +693,7 @@ class Site {
}
let links = []
let text = ''
let css = ''
let scriptSrc = []
let scripts = []
@ -726,6 +728,21 @@ class Site {
'Timeout (links)'
)
// Text
text = await this.promiseTimeout(
(
await this.promiseTimeout(
page.evaluateHandle(() =>
document.body.innerText.replace(/\s+/g, ' ')
),
{ jsonValue: () => '' },
'Timeout (text)'
)
).jsonValue(),
'',
'Timeout (text)'
)
// CSS
css = await this.promiseTimeout(
(
@ -829,6 +846,7 @@ class Site {
this.cache[url.href] = {
page,
html,
text,
cookies,
scripts,
scriptSrc,
@ -845,6 +863,7 @@ class Site {
url,
cookies,
html,
text,
css,
scripts,
scriptSrc,
@ -1120,19 +1139,30 @@ class Site {
if (this.cache[url.href]) {
const resolved = resolve(this.detections)
const requires = Wappalyzer.requires.filter(({ name, technologies }) =>
resolved.some(({ name: _name }) => _name === name)
)
const requires = [
...Wappalyzer.requires.filter(({ name }) =>
resolved.some(({ name: _name }) => _name === name)
),
...Wappalyzer.categoryRequires.filter(({ categoryId }) =>
resolved.some(({ categories }) =>
categories.some(({ id }) => id === categoryId)
)
),
]
await Promise.all(
requires.map(async ({ name, technologies }) => {
requires.map(async ({ name, categoryId, technologies }) => {
const id = categoryId
? `category:${categoryId}`
: `technology:${name}`
this.analyzedRequires[url.href] =
this.analyzedRequires[url.href] || []
if (!this.analyzedRequires[url.href].includes(name)) {
this.analyzedRequires[url.href].push(name)
if (!this.analyzedRequires[url.href].includes(id)) {
this.analyzedRequires[url.href].push(id)
const { page, cookies, html, css, scripts, scriptSrc, meta } =
const { page, cookies, html, text, css, scripts, scriptSrc, meta } =
this.cache[url.href]
const js = await this.promiseTimeout(
@ -1157,6 +1187,7 @@ class Site {
url,
cookies,
html,
text,
css,
scripts,
scriptSrc,

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -188,5 +188,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -190,5 +190,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -188,5 +188,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -190,5 +190,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -190,5 +190,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -188,5 +188,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -190,5 +190,6 @@
"categoryName95": { "message": "Управление цифровыми активами" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -188,5 +188,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -192,5 +192,6 @@
"categoryName95": { "message": "Digital asset management" },
"categoryName96": { "message": "Content curation" },
"categoryName97": { "message": "Customer data platform" },
"categoryName98": { "message": "Cart abandonment" }
"categoryName98": { "message": "Cart abandonment" },
"categoryName99": { "message": "Shipping carriers" }
}

@ -0,0 +1,5 @@
<svg width="195" height="195" viewBox="0 0 195 195" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect width="195" height="195" fill="#FFCC00"/>
<path d="M195 47H0V148H195V47Z" fill="#FFCC00"/>
<path d="M41.3993 106.919H0V115.018H35.4487L41.3993 106.919ZM0 130H24.4138L30.3376 121.957H0V130ZM52.4685 91.8922H0V99.98H46.506L52.4685 91.8922ZM122.322 89.169C120.315 91.886 116.952 96.619 114.907 99.379C113.869 100.779 111.992 103.335 118.213 103.335H149.751C149.751 103.335 155.026 96.1552 159.45 90.1541C165.467 81.9877 159.97 65 138.462 65H53.7609L39.0756 84.9525H120.307C124.352 84.9525 124.299 86.4894 122.322 89.169V89.169ZM166.902 91.8927L160.979 99.936H195V91.8927H166.902ZM144.582 110.274H108.24L94.44 110.273C88.22 110.273 90.0972 107.717 91.1353 106.314C93.1805 103.552 96.623 98.8267 98.6307 96.1066C100.607 93.4271 101.833 91.8917 97.7862 91.8917H61.1823L33.1328 130H105.063C126.393 130 139.231 116.978 144.582 110.274ZM149.917 114.973H195V106.875H155.868L149.917 114.973ZM138.875 130H195V121.912H144.818L138.875 130Z" fill="#D40511"/>
</svg>

After

Width:  |  Height:  |  Size: 1.0 KiB

@ -30,6 +30,15 @@
},
"website": "https://derak.cloud"
},
"DHL": {
"cats": [
99
],
"text": "\\bDHL\\b",
"icon": "DHL.svg",
"requiresCategory": 6,
"website": "https://www.dhl.com"
},
"DHTMLX": {
"cats": [
59
@ -417,7 +426,7 @@
"Digital Showroom": {
"cats": [
6
],
],
"description": "Digital Showroom is an ecommerce platform.",
"icon": "Digital Showroom.png",
"dom": "div.dd-showrom__layout",
@ -1075,4 +1084,4 @@
"scriptSrc": "deepMiner\\.js",
"website": "https://github.com/deepwn/deepMiner"
}
}
}

@ -12,6 +12,7 @@ const Wappalyzer = {
technologies: [],
categories: [],
requires: [],
categoryRequires: [],
slugify: (string) =>
string
@ -24,6 +25,9 @@ const Wappalyzer = {
[
...Wappalyzer.technologies,
...Wappalyzer.requires.map(({ technologies }) => technologies).flat(),
...Wappalyzer.categoryRequires
.map(({ technologies }) => technologies)
.flat(),
].find(({ name: _name }) => name === _name),
getCategory: (id) => Wappalyzer.categories.find(({ id: _id }) => id === _id),
@ -205,6 +209,7 @@ const Wappalyzer = {
url,
xhr,
html,
text,
scripts,
css,
robots,
@ -234,6 +239,7 @@ const Wappalyzer = {
oo(technology, 'url', url),
oo(technology, 'xhr', xhr),
oo(technology, 'html', html),
oo(technology, 'text', text),
oo(technology, 'scripts', scripts),
oo(technology, 'css', css),
oo(technology, 'robots', robots),
@ -269,6 +275,7 @@ const Wappalyzer = {
xhr,
dom,
html,
text,
scripts,
css,
robots,
@ -283,6 +290,7 @@ const Wappalyzer = {
implies,
excludes,
requires,
requiresCategory,
icon,
website,
cpe,
@ -311,6 +319,7 @@ const Wappalyzer = {
false
),
html: transform(html),
text: transform(text),
scripts: transform(scripts),
css: transform(css),
certIssuer: transform(certIssuer),
@ -329,6 +338,9 @@ const Wappalyzer = {
requires: transform(requires).map(({ value }) => ({
name: value,
})),
requiresCategory: transform(requiresCategory).map(({ value }) => ({
id: value,
})),
icon: icon || 'default.svg',
website: website || null,
cpe: cpe || null,
@ -356,8 +368,27 @@ const Wappalyzer = {
technologies: Wappalyzer.requires[name],
}))
Wappalyzer.technologies
.filter(({ requiresCategory }) => requiresCategory.length)
.forEach((technology) =>
technology.requiresCategory.forEach(({ id }) => {
Wappalyzer.categoryRequires[id] =
Wappalyzer.categoryRequires[id] || []
Wappalyzer.categoryRequires[id].push(technology)
})
)
Wappalyzer.categoryRequires = Object.keys(Wappalyzer.categoryRequires).map(
(id) => ({
categoryId: parseInt(id, 10),
technologies: Wappalyzer.categoryRequires[id],
})
)
Wappalyzer.technologies = Wappalyzer.technologies.filter(
({ requires }) => !requires.length
({ requires, requiresCategory }) =>
!requires.length && !requiresCategory.length
)
},
@ -391,7 +422,11 @@ const Wappalyzer = {
return []
}
if (typeof patterns === 'string' || Array.isArray(patterns)) {
if (
typeof patterns === 'string' ||
typeof patterns === 'number' ||
Array.isArray(patterns)
) {
patterns = { main: patterns }
}
@ -421,6 +456,7 @@ const Wappalyzer = {
)
} else {
const { value, regex, confidence, version } = pattern
.toString()
.split('\\;')
.reduce((attrs, attr, i) => {
if (i) {
@ -431,7 +467,7 @@ const Wappalyzer = {
attrs[attr.shift()] = attr.join(':')
}
} else {
attrs.value = attr
attrs.value = typeof pattern === 'number' ? pattern : attr
// Escape slashes in regular expression
attrs.regex = new RegExp(

@ -475,9 +475,9 @@ callsites@^3.0.0:
integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==
caniuse-lite@^1.0.30001219:
version "1.0.30001228"
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001228.tgz#bfdc5942cd3326fa51ee0b42fbef4da9d492a7fa"
integrity sha512-QQmLOGJ3DEgokHbMSA8cj2a+geXqmnpyOFT0lhQV6P3/YOJvGDEwoedcwxEQ30gJIwIIunHIicunJ2rzK5gB2A==
version "1.0.30001271"
resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001271.tgz"
integrity sha512-BBruZFWmt3HFdVPS8kceTBIguKxu4f99n5JNp06OlPD/luoAMIaIK5ieV5YjnBLH3Nysai9sxj9rpJj4ZisXOA==
chalk@^2.0.0, chalk@^2.4.1:
version "2.4.2"