You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
This repo is archived. You can view files and clone it, but cannot push or open issues/pull-requests.

641 lines
16 KiB

5 years ago
'use strict'
function toArray(value) {
return Array.isArray(value) ? value : [value]
}
3 years ago
const benchmarkEnabled =
typeof process !== 'undefined' ? !!process.env.WAPPALYZER_BENCHMARK : false
let benchmarks = []
function benchmark(duration, pattern, value = '', technology) {
if (!benchmarkEnabled) {
return
}
benchmarks.push({
duration,
pattern: String(pattern.regex),
value: String(value).slice(0, 100),
valueLength: value.length,
technology: technology.name,
})
}
function benchmarkSummary() {
if (!benchmarkEnabled) {
return
}
const totalPatterns = Object.values(benchmarks).length
const totalDuration = Object.values(benchmarks).reduce(
(sum, { duration }) => sum + duration,
0
)
// eslint-disable-next-line no-console
console.log({
totalPatterns,
totalDuration,
averageDuration: Math.round(totalDuration / totalPatterns),
slowestTechnologies: Object.values(
benchmarks.reduce((benchmarks, { duration, technology }) => {
if (benchmarks[technology]) {
benchmarks[technology].duration += duration
} else {
benchmarks[technology] = { technology, duration }
}
return benchmarks
}, {})
)
.sort(({ duration: a }, { duration: b }) => (a > b ? -1 : 1))
.filter(({ duration }) => duration)
.slice(0, 5)
.reduce(
(technologies, { technology, duration }) => ({
...technologies,
[technology]: duration,
}),
{}
),
slowestPatterns: Object.values(benchmarks)
.sort(({ duration: a }, { duration: b }) => (a > b ? -1 : 1))
.filter(({ duration }) => duration)
.slice(0, 5),
})
}
5 years ago
const Wappalyzer = {
technologies: [],
categories: [],
requires: [],
categoryRequires: [],
5 years ago
slugify: (string) =>
string
5 years ago
.toLowerCase()
.replace(/[^a-z0-9-]/g, '-')
.replace(/--+/g, '-')
.replace(/(?:^-|-$)/g, ''),
5 years ago
getTechnology: (name) =>
[
...Wappalyzer.technologies,
...Wappalyzer.requires.map(({ technologies }) => technologies).flat(),
...Wappalyzer.categoryRequires
.map(({ technologies }) => technologies)
.flat(),
].find(({ name: _name }) => name === _name),
5 years ago
getCategory: (id) => Wappalyzer.categories.find(({ id: _id }) => id === _id),
5 years ago
/**
* Resolve promises for implied technology.
* @param {Array} detections
*/
5 years ago
resolve(detections = []) {
const resolved = detections.reduce((resolved, { technology, lastUrl }) => {
5 years ago
if (
resolved.findIndex(
({ technology: { name } }) => name === technology.name
) === -1
) {
let version = ''
let confidence = 0
detections
.filter(({ technology }) => technology)
.forEach(
({ technology: { name }, pattern, version: _version = '' }) => {
if (name === technology.name) {
confidence = Math.min(100, confidence + pattern.confidence)
version =
_version.length > version.length &&
_version.length <= 15 &&
(parseInt(_version, 10) || 0) < 10000 // Ignore long numeric strings like timestamps
? _version
: version
}
4 years ago
}
)
6 years ago
resolved.push({ technology, confidence, version, lastUrl })
6 years ago
}
5 years ago
return resolved
}, [])
Wappalyzer.resolveExcludes(resolved)
Wappalyzer.resolveImplies(resolved)
const priority = ({ technology: { categories } }) =>
categories.reduce(
(max, id) => Math.max(max, Wappalyzer.getCategory(id).priority),
0
)
return resolved
.sort((a, b) => (priority(a) > priority(b) ? 1 : -1))
.map(
({
technology: { name, slug, categories, icon, website, pricing, cpe },
confidence,
4 years ago
version,
lastUrl,
}) => ({
name,
slug,
categories: categories.map((id) => Wappalyzer.getCategory(id)),
confidence,
version,
icon,
website,
pricing,
4 years ago
cpe,
lastUrl,
})
)
5 years ago
},
/**
* Resolve promises for version of technology.
* @param {Promise} resolved
* @param match
*/
5 years ago
resolveVersion({ version, regex }, match) {
let resolved = version
7 years ago
5 years ago
if (version) {
const matches = regex.exec(match)
7 years ago
5 years ago
if (matches) {
matches.forEach((match, index) => {
// Parse ternary operator
const ternary = new RegExp(`\\\\${index}\\?([^:]+):(.*)$`).exec(
version
)
5 years ago
if (ternary && ternary.length === 3) {
resolved = version.replace(
ternary[0],
match ? ternary[1] : ternary[2]
)
}
5 years ago
// Replace back references
resolved = resolved
.trim()
.replace(new RegExp(`\\\\${index}`, 'g'), match || '')
})
}
}
5 years ago
return resolved
},
7 years ago
/**
* Resolve promises for excluded technology.
* @param {Promise} resolved
*/
5 years ago
resolveExcludes(resolved) {
resolved.forEach(({ technology }) => {
technology.excludes.forEach(({ name }) => {
5 years ago
const excluded = Wappalyzer.getTechnology(name)
7 years ago
5 years ago
if (!excluded) {
throw new Error(`Excluded technology does not exist: ${name}`)
}
7 years ago
let index
do {
index = resolved.findIndex(
({ technology: { name } }) => name === excluded.name
)
if (index !== -1) {
resolved.splice(index, 1)
}
} while (index !== -1)
})
})
5 years ago
},
7 years ago
/**
* Resolve promises for implied technology.
* @param {Promise} resolved
*/
5 years ago
resolveImplies(resolved) {
let done = false
do {
done = true
7 years ago
resolved.forEach(({ technology, confidence, lastUrl }) => {
technology.implies.forEach(({ name, confidence: _confidence }) => {
5 years ago
const implied = Wappalyzer.getTechnology(name)
7 years ago
5 years ago
if (!implied) {
throw new Error(`Implied technology does not exist: ${name}`)
}
7 years ago
5 years ago
if (
resolved.findIndex(
({ technology: { name } }) => name === implied.name
) === -1
) {
resolved.push({
technology: implied,
confidence: Math.min(confidence, _confidence),
4 years ago
version: '',
lastUrl,
})
7 years ago
5 years ago
done = false
}
})
})
} while (resolved.length && !done)
5 years ago
},
/**
* Initialize analyzation.
* @param {*} param0
*/
analyze(items, technologies = Wappalyzer.technologies) {
benchmarks = []
5 years ago
const oo = Wappalyzer.analyzeOneToOne
const om = Wappalyzer.analyzeOneToMany
const mm = Wappalyzer.analyzeManyToMany
const relations = {
url: oo,
xhr: oo,
html: oo,
text: oo,
scripts: oo,
css: oo,
robots: oo,
magento: oo,
certIssuer: oo,
scriptSrc: om,
cookies: mm,
meta: mm,
headers: mm,
dns: mm,
}
5 years ago
try {
const detections = technologies
.map((technology) =>
Object.keys(relations)
.map(
(type) =>
items[type] && relations[type](technology, type, items[type])
)
.flat()
)
.flat()
.filter((technology) => technology)
benchmarkSummary()
5 years ago
return detections
} catch (error) {
throw new Error(error.message || error.toString())
7 years ago
}
5 years ago
},
/**
* Extract technologies from data collected.
* @param {object} data
*/
5 years ago
setTechnologies(data) {
const transform = Wappalyzer.transformPatterns
Wappalyzer.technologies = Object.keys(data).reduce((technologies, name) => {
const {
cats,
url,
xhr,
dom,
5 years ago
html,
text,
scripts,
css,
robots,
magento,
5 years ago
meta,
headers,
dns,
certIssuer,
5 years ago
cookies,
scriptSrc,
5 years ago
js,
implies,
excludes,
requires,
requiresCategory,
5 years ago
icon,
website,
pricing,
4 years ago
cpe,
5 years ago
} = data[name]
technologies.push({
name,
categories: cats || [],
slug: Wappalyzer.slugify(name),
url: transform(url),
xhr: transform(xhr),
4 years ago
headers: transform(headers),
dns: transform(dns),
5 years ago
cookies: transform(cookies),
dom: transform(
typeof dom === 'string' || Array.isArray(dom)
? toArray(dom).reduce(
(dom, selector) => ({
...dom,
[selector]: { exists: '' },
}),
{}
)
: dom,
true,
false
),
5 years ago
html: transform(html),
text: transform(text),
scripts: transform(scripts),
css: transform(css),
certIssuer: transform(certIssuer),
robots: transform(robots),
magento: transform(magento),
5 years ago
meta: transform(meta),
scriptSrc: transform(scriptSrc),
js: transform(js, true),
implies: transform(implies).map(({ value, confidence }) => ({
name: value,
4 years ago
confidence,
})),
excludes: transform(excludes).map(({ value }) => ({
4 years ago
name: value,
})),
requires: transform(requires).map(({ value }) => ({
name: value,
})),
requiresCategory: transform(requiresCategory).map(({ value }) => ({
id: value,
})),
5 years ago
icon: icon || 'default.svg',
website: website || null,
pricing: pricing || [],
4 years ago
cpe: cpe || null,
5 years ago
})
7 years ago
5 years ago
return technologies
}, [])
Wappalyzer.technologies
.filter(({ requires }) => requires.length)
.forEach((technology) =>
technology.requires.forEach(({ name }) => {
if (!Wappalyzer.getTechnology(name)) {
throw new Error(`Required technology does not exist: ${name}`)
}
Wappalyzer.requires[name] = Wappalyzer.requires[name] || []
Wappalyzer.requires[name].push(technology)
})
)
Wappalyzer.requires = Object.keys(Wappalyzer.requires).map((name) => ({
name,
technologies: Wappalyzer.requires[name],
}))
Wappalyzer.technologies
.filter(({ requiresCategory }) => requiresCategory.length)
.forEach((technology) =>
technology.requiresCategory.forEach(({ id }) => {
Wappalyzer.categoryRequires[id] =
Wappalyzer.categoryRequires[id] || []
Wappalyzer.categoryRequires[id].push(technology)
})
)
Wappalyzer.categoryRequires = Object.keys(Wappalyzer.categoryRequires).map(
(id) => ({
categoryId: parseInt(id, 10),
technologies: Wappalyzer.categoryRequires[id],
})
)
Wappalyzer.technologies = Wappalyzer.technologies.filter(
({ requires, requiresCategory }) =>
!requires.length && !requiresCategory.length
)
5 years ago
},
/**
* Assign categories for data.
* @param {Object} data
*/
5 years ago
setCategories(data) {
Wappalyzer.categories = Object.keys(data)
.reduce((categories, id) => {
const category = data[id]
5 years ago
categories.push({
id: parseInt(id, 10),
slug: Wappalyzer.slugify(category.name),
4 years ago
...category,
5 years ago
})
5 years ago
return categories
}, [])
.sort(({ priority: a }, { priority: b }) => (a > b ? -1 : 0))
},
7 years ago
/**
* Transform patterns for internal use.
* @param {string|array} patterns
* @param {boolean} caseSensitive
*/
transformPatterns(patterns, caseSensitive = false, isRegex = true) {
5 years ago
if (!patterns) {
return []
7 years ago
}
7 years ago
if (
typeof patterns === 'string' ||
typeof patterns === 'number' ||
Array.isArray(patterns)
) {
5 years ago
patterns = { main: patterns }
}
7 years ago
5 years ago
const parsed = Object.keys(patterns).reduce((parsed, key) => {
parsed[caseSensitive ? key : key.toLowerCase()] = toArray(
patterns[key]
).map((pattern) => Wappalyzer.parsePattern(pattern, isRegex))
5 years ago
return parsed
}, {})
7 years ago
5 years ago
return 'main' in parsed ? parsed.main : parsed
},
/**
* Extract information from regex pattern.
* @param {string|object} pattern
*/
parsePattern(pattern, isRegex = true) {
if (typeof pattern === 'object') {
return Object.keys(pattern).reduce(
(parsed, key) => ({
...parsed,
[key]: Wappalyzer.parsePattern(pattern[key]),
}),
{}
)
} else {
const { value, regex, confidence, version } = pattern
.toString()
.split('\\;')
.reduce((attrs, attr, i) => {
if (i) {
// Key value pairs
attr = attr.split(':')
if (attr.length > 1) {
attrs[attr.shift()] = attr.join(':')
}
} else {
attrs.value = typeof pattern === 'number' ? pattern : attr
attrs.regex = new RegExp(
isRegex
? attr
// Escape slashes
.replace(/\//g, '\\/')
// Optimise quantifiers for long strings
.replace(/\+/g, '{1,250}')
.replace(/\*/g, '{0,250}')
: '',
'i'
)
}
return attrs
}, {})
return {
value,
regex,
confidence: parseInt(confidence || 100, 10),
version: version || '',
}
}
},
/**
* @todo describe
* @param {Object} technology
* @param {String} type
* @param {String} value
*/
5 years ago
analyzeOneToOne(technology, type, value) {
return technology[type].reduce((technologies, pattern) => {
const startTime = Date.now()
5 years ago
if (pattern.regex.test(value)) {
4 years ago
technologies.push({
technology,
pattern: { ...pattern, type },
4 years ago
version: Wappalyzer.resolveVersion(pattern, value),
4 years ago
})
}
7 years ago
benchmark(Date.now() - startTime, pattern, value, technology)
5 years ago
return technologies
}, [])
},
/**
* @todo update
* @param {Object} technology
* @param {String} type
* @param {Array} items
*/
5 years ago
analyzeOneToMany(technology, type, items = []) {
4 years ago
return items.reduce((technologies, value) => {
const patterns = technology[type] || []
5 years ago
patterns.forEach((pattern) => {
const startTime = Date.now()
5 years ago
if (pattern.regex.test(value)) {
4 years ago
technologies.push({
technology,
pattern: { ...pattern, type },
4 years ago
version: Wappalyzer.resolveVersion(pattern, value),
4 years ago
})
}
benchmark(Date.now() - startTime, pattern, value, technology)
})
5 years ago
return technologies
}, [])
},
/**
*
* @param {Object} technology
* @param {string} types
* @param {Array} items
*/
analyzeManyToMany(technology, types, items = {}) {
const [type, ...subtypes] = types.split('.')
5 years ago
return Object.keys(technology[type]).reduce((technologies, key) => {
const patterns = technology[type][key] || []
const values = items[key] || []
6 years ago
patterns.forEach((_pattern) => {
const pattern = (subtypes || []).reduce(
(pattern, subtype) => pattern[subtype] || {},
_pattern
)
5 years ago
values.forEach((value) => {
const startTime = Date.now()
5 years ago
if (pattern.regex.test(value)) {
4 years ago
technologies.push({
technology,
pattern: { ...pattern, type },
4 years ago
version: Wappalyzer.resolveVersion(pattern, value),
4 years ago
})
5 years ago
}
benchmark(Date.now() - startTime, pattern, value, technology)
5 years ago
})
})
5 years ago
return technologies
}, [])
4 years ago
},
}
5 years ago
if (typeof module !== 'undefined') {
module.exports = Wappalyzer
}