'use strict' function toArray(value) { return Array.isArray(value) ? value : [value] } const benchmarkEnabled = typeof process !== 'undefined' ? !!process.env.WAPPALYZER_BENCHMARK : false let benchmarks = [] function benchmark(duration, pattern, value = '', technology) { if (!benchmarkEnabled) { return } benchmarks.push({ duration, pattern: String(pattern.regex), value: String(value).slice(0, 100), valueLength: value.length, technology: technology.name, }) } function benchmarkSummary() { if (!benchmarkEnabled) { return } const totalPatterns = Object.values(benchmarks).length const totalDuration = Object.values(benchmarks).reduce( (sum, { duration }) => sum + duration, 0 ) // eslint-disable-next-line no-console console.log({ totalPatterns, totalDuration, averageDuration: Math.round(totalDuration / totalPatterns), slowestTechnologies: Object.values( benchmarks.reduce((benchmarks, { duration, technology }) => { if (benchmarks[technology]) { benchmarks[technology].duration += duration } else { benchmarks[technology] = { technology, duration } } return benchmarks }, {}) ) .sort(({ duration: a }, { duration: b }) => (a > b ? -1 : 1)) .filter(({ duration }) => duration) .slice(0, 5) .reduce( (technologies, { technology, duration }) => ({ ...technologies, [technology]: duration, }), {} ), slowestPatterns: Object.values(benchmarks) .sort(({ duration: a }, { duration: b }) => (a > b ? -1 : 1)) .filter(({ duration }) => duration) .slice(0, 5), }) } const Wappalyzer = { technologies: [], categories: [], requires: [], categoryRequires: [], slugify: (string) => string .toLowerCase() .replace(/[^a-z0-9-]/g, '-') .replace(/--+/g, '-') .replace(/(?:^-|-$)/g, ''), getTechnology: (name) => [ ...Wappalyzer.technologies, ...Wappalyzer.requires.map(({ technologies }) => technologies).flat(), ...Wappalyzer.categoryRequires .map(({ technologies }) => technologies) .flat(), ].find(({ name: _name }) => name === _name), getCategory: (id) => Wappalyzer.categories.find(({ id: _id }) => id === _id), /** * Resolve promises for implied technology. * @param {Array} detections */ resolve(detections = []) { const resolved = detections.reduce((resolved, { technology, lastUrl }) => { if ( resolved.findIndex( ({ technology: { name } }) => name === technology?.name ) === -1 ) { let version = '' let confidence = 0 let rootPath detections .filter( ({ technology: _technology }) => _technology && _technology.name === technology.name ) .forEach( ({ technology: { name }, pattern, version: _version = '', rootPath: _rootPath, }) => { confidence = Math.min(100, confidence + pattern.confidence) version = _version.length > version.length && _version.length <= 15 && (parseInt(_version, 10) || 0) < 10000 // Ignore long numeric strings like timestamps ? _version : version rootPath = rootPath || _rootPath || undefined } ) resolved.push({ technology, confidence, version, rootPath, lastUrl }) } return resolved }, []) Wappalyzer.resolveExcludes(resolved) Wappalyzer.resolveImplies(resolved) const priority = ({ technology: { categories } }) => categories.reduce( (max, id) => Math.max(max, Wappalyzer.getCategory(id).priority), 0 ) return resolved .sort((a, b) => (priority(a) > priority(b) ? 1 : -1)) .map( ({ technology: { name, description, slug, categories, icon, website, pricing, cpe, }, confidence, version, rootPath, lastUrl, }) => ({ name, description, slug, categories: categories.map((id) => Wappalyzer.getCategory(id)), confidence, version, icon, website, pricing, cpe, rootPath, lastUrl, }) ) }, /** * Resolve promises for version of technology. * @param {Promise} resolved * @param match */ resolveVersion({ version, regex }, match) { let resolved = version if (version) { const matches = regex.exec(match) if (matches) { matches.forEach((match, index) => { if (String(match).length > 10) { return } // Parse ternary operator const ternary = new RegExp(`\\\\${index}\\?([^:]+):(.*)$`).exec( version ) if (ternary && ternary.length === 3) { resolved = version.replace( ternary[0], match ? ternary[1] : ternary[2] ) } // Replace back references resolved = resolved .trim() .replace(new RegExp(`\\\\${index}`, 'g'), match || '') }) // Remove unmatched back references resolved = resolved.replace(/\\\d/, '') } } return resolved }, /** * Resolve promises for excluded technology. * @param {Promise} resolved */ resolveExcludes(resolved) { resolved.forEach(({ technology }) => { technology.excludes.forEach(({ name }) => { const excluded = Wappalyzer.getTechnology(name) if (!excluded) { throw new Error(`Excluded technology does not exist: ${name}`) } let index do { index = resolved.findIndex( ({ technology: { name } }) => name === excluded.name ) if (index !== -1) { resolved.splice(index, 1) } } while (index !== -1) }) }) }, /** * Resolve promises for implied technology. * @param {Promise} resolved */ resolveImplies(resolved) { let done = false do { done = true resolved.forEach(({ technology, confidence, lastUrl }) => { technology.implies.forEach( ({ name, confidence: _confidence, version }) => { const implied = Wappalyzer.getTechnology(name) if (!implied) { throw new Error(`Implied technology does not exist: ${name}`) } if ( resolved.findIndex( ({ technology: { name } }) => name === implied.name ) === -1 ) { resolved.push({ technology: implied, confidence: Math.min(confidence, _confidence), version: version || '', lastUrl, }) done = false } } ) }) } while (resolved.length && !done) }, /** * Initialize analyzation. * @param {*} param0 */ analyze(items, technologies = Wappalyzer.technologies) { benchmarks = [] const oo = Wappalyzer.analyzeOneToOne const om = Wappalyzer.analyzeOneToMany const mm = Wappalyzer.analyzeManyToMany const relations = { certIssuer: oo, cookies: mm, css: oo, dns: mm, headers: mm, html: oo, meta: mm, probe: mm, robots: oo, scriptSrc: om, scripts: oo, text: oo, url: oo, xhr: oo, } try { const detections = technologies .map((technology) => Object.keys(relations) .map( (type) => items[type] && relations[type](technology, type, items[type]) ) .flat() ) .flat() .filter((technology) => technology) benchmarkSummary() return detections } catch (error) { throw new Error(error.message || error.toString()) } }, /** * Extract technologies from data collected. * @param {object} data */ setTechnologies(data) { const transform = Wappalyzer.transformPatterns Wappalyzer.technologies = Object.keys(data).reduce((technologies, name) => { const { cats, certIssuer, cookies, cpe, css, description, dns, dom, excludes, headers, html, icon, implies, js, meta, pricing, probe, requires, requiresCategory, robots, scriptSrc, scripts, text, url, website, xhr, } = data[name] technologies.push({ categories: cats || [], certIssuer: transform(certIssuer), cookies: transform(cookies), cpe: cpe || null, css: transform(css), description: description || null, dns: transform(dns), dom: transform( typeof dom === 'string' || Array.isArray(dom) ? toArray(dom).reduce( (dom, selector) => ({ ...dom, [selector]: { exists: '' } }), {} ) : dom, true, false ), excludes: transform(excludes).map(({ value }) => ({ name: value })), headers: transform(headers), html: transform(html), icon: icon || 'default.svg', implies: transform(implies).map(({ value, confidence, version }) => ({ name: value, confidence, version, })), js: transform(js, true), meta: transform(meta), name, pricing: pricing || [], probe: transform(probe, true), requires: transform(requires).map(({ value }) => ({ name: value })), requiresCategory: transform(requiresCategory).map(({ value }) => ({ id: value, })), robots: transform(robots), scriptSrc: transform(scriptSrc), scripts: transform(scripts), slug: Wappalyzer.slugify(name), text: transform(text), url: transform(url), website: website || null, xhr: transform(xhr), }) return technologies }, []) Wappalyzer.technologies .filter(({ requires }) => requires.length) .forEach((technology) => technology.requires.forEach(({ name }) => { if (!Wappalyzer.getTechnology(name)) { throw new Error(`Required technology does not exist: ${name}`) } Wappalyzer.requires[name] = Wappalyzer.requires[name] || [] Wappalyzer.requires[name].push(technology) }) ) Wappalyzer.requires = Object.keys(Wappalyzer.requires).map((name) => ({ name, technologies: Wappalyzer.requires[name], })) Wappalyzer.technologies .filter(({ requiresCategory }) => requiresCategory.length) .forEach((technology) => technology.requiresCategory.forEach(({ id }) => { Wappalyzer.categoryRequires[id] = Wappalyzer.categoryRequires[id] || [] Wappalyzer.categoryRequires[id].push(technology) }) ) Wappalyzer.categoryRequires = Object.keys(Wappalyzer.categoryRequires).map( (id) => ({ categoryId: parseInt(id, 10), technologies: Wappalyzer.categoryRequires[id], }) ) Wappalyzer.technologies = Wappalyzer.technologies.filter( ({ requires, requiresCategory }) => !requires.length && !requiresCategory.length ) }, /** * Assign categories for data. * @param {Object} data */ setCategories(data) { Wappalyzer.categories = Object.keys(data) .reduce((categories, id) => { const category = data[id] categories.push({ id: parseInt(id, 10), slug: Wappalyzer.slugify(category.name), ...category, }) return categories }, []) .sort(({ priority: a }, { priority: b }) => (a > b ? -1 : 0)) }, /** * Transform patterns for internal use. * @param {string|array} patterns * @param {boolean} caseSensitive */ transformPatterns(patterns, caseSensitive = false, isRegex = true) { if (!patterns) { return [] } if ( typeof patterns === 'string' || typeof patterns === 'number' || Array.isArray(patterns) ) { patterns = { main: patterns } } const parsed = Object.keys(patterns).reduce((parsed, key) => { parsed[caseSensitive ? key : key.toLowerCase()] = toArray( patterns[key] ).map((pattern) => Wappalyzer.parsePattern(pattern, isRegex)) return parsed }, {}) return 'main' in parsed ? parsed.main : parsed }, /** * Extract information from regex pattern. * @param {string|object} pattern */ parsePattern(pattern, isRegex = true) { if (typeof pattern === 'object') { return Object.keys(pattern).reduce( (parsed, key) => ({ ...parsed, [key]: Wappalyzer.parsePattern(pattern[key]), }), {} ) } else { const { value, regex, confidence, version } = pattern .toString() .split('\\;') .reduce((attrs, attr, i) => { if (i) { // Key value pairs attr = attr.split(':') if (attr.length > 1) { attrs[attr.shift()] = attr.join(':') } } else { attrs.value = typeof pattern === 'number' ? pattern : attr attrs.regex = new RegExp( isRegex ? attr // Escape slashes .replace(/\//g, '\\/') // Optimise quantifiers for long strings .replace(/\\\+/g, '__escapedPlus__') .replace(/\+/g, '{1,250}') .replace(/\*/g, '{0,250}') .replace(/__escapedPlus__/g, '\\+') : '', 'i' ) } return attrs }, {}) return { value, regex, confidence: parseInt(confidence || 100, 10), version: version || '', } } }, /** * @todo describe * @param {Object} technology * @param {String} type * @param {String} value */ analyzeOneToOne(technology, type, value) { return technology[type].reduce((technologies, pattern) => { const startTime = Date.now() const matches = pattern.regex.exec(value) if (matches) { technologies.push({ technology, pattern: { ...pattern, type, value, match: matches[0], }, version: Wappalyzer.resolveVersion(pattern, value), }) } benchmark(Date.now() - startTime, pattern, value, technology) return technologies }, []) }, /** * @todo update * @param {Object} technology * @param {String} type * @param {Array} items */ analyzeOneToMany(technology, type, items = []) { return items.reduce((technologies, value) => { const patterns = technology[type] || [] patterns.forEach((pattern) => { const startTime = Date.now() const matches = pattern.regex.exec(value) if (matches) { technologies.push({ technology, pattern: { ...pattern, type, value, match: matches[0], }, version: Wappalyzer.resolveVersion(pattern, value), }) } benchmark(Date.now() - startTime, pattern, value, technology) }) return technologies }, []) }, /** * * @param {Object} technology * @param {string} types * @param {Array} items */ analyzeManyToMany(technology, types, items = {}) { const [type, ...subtypes] = types.split('.') return Object.keys(technology[type]).reduce((technologies, key) => { const patterns = technology[type][key] || [] const values = items[key] || [] patterns.forEach((_pattern) => { const pattern = (subtypes || []).reduce( (pattern, subtype) => pattern[subtype] || {}, _pattern ) values.forEach((value) => { const startTime = Date.now() const matches = pattern.regex.exec(value) if (matches) { technologies.push({ technology, pattern: { ...pattern, type, value, match: matches[0], }, version: Wappalyzer.resolveVersion(pattern, value), }) } benchmark(Date.now() - startTime, pattern, value, technology) }) }) return technologies }, []) }, } if (typeof module !== 'undefined') { module.exports = Wappalyzer }