You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
539 lines
12 KiB
539 lines
12 KiB
/**
|
|
* Wappalyzer v4
|
|
*
|
|
* Created by Elbert Alias <elbert@alias.io>
|
|
*
|
|
* License: GPLv3 http://www.gnu.org/licenses/gpl-3.0.txt
|
|
*/
|
|
|
|
var wappalyzer = (function() {
|
|
//'use strict';
|
|
|
|
/**
|
|
* Application class
|
|
*/
|
|
var Application = function(app, detected) {
|
|
this.app = app;
|
|
this.confidence = { };
|
|
this.confidenceTotal = 0;
|
|
this.detected = Boolean(detected);
|
|
this.excludes = [ ];
|
|
this.version = '';
|
|
this.versions = [ ];
|
|
};
|
|
|
|
Application.prototype = {
|
|
/**
|
|
* Calculate confidence total
|
|
*/
|
|
getConfidence: function() {
|
|
var
|
|
id,
|
|
total = 0;
|
|
|
|
for ( id in this.confidence ) {
|
|
total += this.confidence[id];
|
|
}
|
|
|
|
return this.confidenceTotal = Math.min(total, 100);
|
|
},
|
|
|
|
/**
|
|
* Resolve version number (find the longest version number that contains all shorter detected version numbers)
|
|
*/
|
|
getVersion: function() {
|
|
var i, resolved;
|
|
|
|
if ( !this.versions.length ) {
|
|
return;
|
|
}
|
|
|
|
this.versions.sort(function(a, b) {
|
|
return a.length - b.length;
|
|
});
|
|
|
|
resolved = this.versions[0];
|
|
|
|
for ( i = 1; i < this.versions.length; i++ ) {
|
|
if ( this.versions[i].indexOf(resolved) === -1 ) {
|
|
break;
|
|
}
|
|
|
|
resolved = this.versions[i];
|
|
}
|
|
|
|
return this.version = resolved;
|
|
},
|
|
|
|
setDetected: function(pattern, type, value, key) {
|
|
this.detected = true;
|
|
|
|
// Set confidence level
|
|
this.confidence[type + ' ' + ( key ? key + ' ' : '' ) + pattern.regex] = pattern.confidence || 100;
|
|
|
|
// Detect version number
|
|
if ( pattern.version ) {
|
|
var
|
|
version = pattern.version,
|
|
matches = pattern.regex.exec(value);
|
|
|
|
if ( matches ) {
|
|
matches.forEach(function(match, i) {
|
|
// Parse ternary operator
|
|
var ternary = new RegExp('\\\\' + i + '\\?([^:]+):(.*)$').exec(version);
|
|
|
|
if ( ternary && ternary.length === 3 ) {
|
|
version = version.replace(ternary[0], match ? ternary[1] : ternary[2]);
|
|
}
|
|
|
|
// Replace back references
|
|
version = version.replace(new RegExp('\\\\' + i, 'g'), match || '');
|
|
});
|
|
|
|
if ( version && this.versions.indexOf(version) < 0 ) {
|
|
this.versions.push(version);
|
|
}
|
|
|
|
this.getVersion();
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
var asArray = function(value) {
|
|
return typeof value === 'string' ? [ value ] : value;
|
|
};
|
|
|
|
/**
|
|
* Call driver functions
|
|
*/
|
|
var driver = function(func, args) {
|
|
if ( typeof w.driver[func] !== 'function' ) {
|
|
w.log('not implemented: w.driver.' + func, 'core', 'warn');
|
|
|
|
return;
|
|
}
|
|
|
|
return w.driver[func](args);
|
|
};
|
|
|
|
/**
|
|
* Parse apps.json patterns
|
|
*/
|
|
var parsePatterns = function(patterns) {
|
|
var
|
|
key,
|
|
parsed = {};
|
|
|
|
// Convert string to object containing array containing string
|
|
if ( typeof patterns === 'string' || patterns instanceof Array ) {
|
|
patterns = {
|
|
main: asArray(patterns)
|
|
};
|
|
}
|
|
|
|
for ( key in patterns ) {
|
|
parsed[key] = [];
|
|
|
|
asArray(patterns[key]).forEach(function(pattern) {
|
|
var attrs = {};
|
|
|
|
pattern.split('\\;').forEach(function(attr, i) {
|
|
if ( i ) {
|
|
// Key value pairs
|
|
attr = attr.split(':');
|
|
|
|
if ( attr.length > 1 ) {
|
|
attrs[attr.shift()] = attr.join(':');
|
|
}
|
|
} else {
|
|
attrs.string = attr;
|
|
|
|
try {
|
|
attrs.regex = new RegExp(attr.replace('/', '\/'), 'i'); // Escape slashes in regular expression
|
|
} catch (e) {
|
|
attrs.regex = new RegExp();
|
|
|
|
w.log(e + ': ' + attr, 'error', 'core');
|
|
}
|
|
}
|
|
});
|
|
|
|
parsed[key].push(attrs);
|
|
});
|
|
}
|
|
|
|
// Convert back to array if the original pattern list was an array (or string)
|
|
if ( parsed.hasOwnProperty('main') ) {
|
|
parsed = parsed.main;
|
|
}
|
|
|
|
return parsed;
|
|
};
|
|
|
|
/**
|
|
* Main script
|
|
*/
|
|
var w = {
|
|
apps: {},
|
|
cats: null,
|
|
ping: {
|
|
hostnames: { }
|
|
},
|
|
adCache: [],
|
|
detected: {},
|
|
|
|
config: {
|
|
websiteURL: 'https://wappalyzer.com/',
|
|
twitterURL: 'https://twitter.com/Wappalyzer',
|
|
githubURL: 'https://github.com/AliasIO/Wappalyzer',
|
|
},
|
|
|
|
validation: {
|
|
hostname: /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/,
|
|
hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/
|
|
},
|
|
|
|
/**
|
|
* Log messages to console
|
|
*/
|
|
log: function(message, source, type) {
|
|
driver('log', {
|
|
source: source || '',
|
|
message: JSON.stringify(message),
|
|
type: type || 'debug'
|
|
});
|
|
},
|
|
|
|
/**
|
|
* Initialize
|
|
*/
|
|
init: function() {
|
|
w.log('Function call: w.init()', 'core');
|
|
|
|
// Initialize driver
|
|
if ( w.driver !== undefined ) {
|
|
driver('init');
|
|
} else {
|
|
w.log('No driver, exiting', 'core');
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Analyze the request
|
|
*/
|
|
analyze: function(hostname, url, data) {
|
|
var
|
|
app,
|
|
apps = {};
|
|
|
|
w.log('Function call: w.analyze()', 'core');
|
|
|
|
if ( w.apps === undefined || w.categories === undefined ) {
|
|
w.log('apps.json not loaded, check for syntax errors', 'core');
|
|
|
|
return;
|
|
}
|
|
|
|
// Remove hash from URL
|
|
data.url = url = url.split('#')[0];
|
|
|
|
if ( typeof data.html !== 'string' ) {
|
|
data.html = '';
|
|
}
|
|
|
|
if ( w.detected[url] === undefined ) {
|
|
w.detected[url] = {};
|
|
}
|
|
|
|
for ( app in w.apps ) {
|
|
apps[app] = w.detected[url] && w.detected[url][app] ? w.detected[url][app] : new Application(app);
|
|
|
|
if ( url ) {
|
|
w.analyzeUrl(apps[app], url);
|
|
}
|
|
|
|
if ( data.html ) {
|
|
w.analyzeHtml(apps[app], data.html);
|
|
w.analyzeScript(apps[app], data.html);
|
|
w.analyzeMeta(apps[app], data.html);
|
|
}
|
|
|
|
if ( data.headers ) {
|
|
w.analyzeHeaders(apps[app], data.headers);
|
|
}
|
|
|
|
if ( data.env ) {
|
|
w.analyzeEnv(apps[app], data.env);
|
|
}
|
|
}
|
|
|
|
for ( app in apps ) {
|
|
if ( !apps[app].detected ) {
|
|
delete apps[app];
|
|
}
|
|
}
|
|
|
|
w.resolveExcludes(apps);
|
|
w.resolveImplies(apps, url);
|
|
|
|
w.cacheDetectedApps(apps, url);
|
|
w.trackDetectedApps(apps, url, hostname, data.html);
|
|
|
|
if ( Object.keys(apps).length ) {
|
|
w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url, 'core');
|
|
}
|
|
|
|
driver('displayApps');
|
|
},
|
|
|
|
resolveExcludes: function(apps) {
|
|
var
|
|
app,
|
|
excludes = [];
|
|
|
|
// Exclude app in detected apps only
|
|
for ( app in apps ) {
|
|
if ( w.apps[app].excludes ) {
|
|
asArray(w.apps[app].excludes).forEach(function(excluded) {
|
|
excludes.push(excluded);
|
|
});
|
|
}
|
|
}
|
|
|
|
// Remove excluded applications
|
|
for ( app in apps ) {
|
|
if ( excludes.indexOf(app) !== -1 ) {
|
|
delete apps[app];
|
|
}
|
|
}
|
|
},
|
|
|
|
resolveImplies: function(apps, url) {
|
|
var
|
|
confidence,
|
|
id,
|
|
checkImplies = true;
|
|
|
|
// Implied applications
|
|
// Run several passes as implied apps may imply other apps
|
|
while ( checkImplies ) {
|
|
checkImplies = false;
|
|
|
|
for ( app in apps ) {
|
|
confidence = apps[app].confidence;
|
|
|
|
if ( w.apps[app] && w.apps[app].implies ) {
|
|
asArray(w.apps[app].implies).forEach(function(implied) {
|
|
implied = parsePatterns(implied)[0];
|
|
|
|
if ( !w.apps[implied.string] ) {
|
|
w.log('Implied application ' + implied.string + ' does not exist', 'core', 'warn');
|
|
|
|
return;
|
|
}
|
|
|
|
if ( !apps.hasOwnProperty(implied.string) ) {
|
|
apps[implied.string] = w.detected[url] && w.detected[url][implied.string] ? w.detected[url][implied.string] : new Application(implied.string, true);
|
|
|
|
checkImplies = true;
|
|
}
|
|
|
|
// Apply app confidence to implied app
|
|
for ( id in confidence ) {
|
|
apps[implied.string].confidence[id + ' implied by ' + app] = confidence[id] * ( implied.confidence ? implied.confidence / 100 : 1 );
|
|
}
|
|
});
|
|
}
|
|
}
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Cache detected applications
|
|
*/
|
|
cacheDetectedApps: function(apps, url) {
|
|
var app, id, confidence;
|
|
|
|
for ( app in apps ) {
|
|
confidence = apps[app].confidence;
|
|
|
|
// Per URL
|
|
w.detected[url][app] = apps[app];
|
|
|
|
for ( id in confidence ) {
|
|
w.detected[url][app].confidence[id] = confidence[id];
|
|
}
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Track detected applications
|
|
*/
|
|
trackDetectedApps: function(apps, url, hostname, html) {
|
|
var app, match;
|
|
|
|
for ( app in apps ) {
|
|
if ( w.detected[url][app].getConfidence() >= 100 && w.validation.hostname.test(hostname) && !w.validation.hostnameBlacklist.test(url) ) {
|
|
if ( !w.ping.hostnames.hasOwnProperty(hostname) ) {
|
|
w.ping.hostnames[hostname] = {
|
|
applications: {},
|
|
meta: {}
|
|
};
|
|
}
|
|
|
|
if ( !w.ping.hostnames[hostname].applications.hasOwnProperty(app) ) {
|
|
w.ping.hostnames[hostname].applications[app] = {
|
|
hits: 0
|
|
};
|
|
}
|
|
|
|
w.ping.hostnames[hostname].applications[app].hits ++;
|
|
|
|
if ( apps[app].version ) {
|
|
w.ping.hostnames[hostname].applications[app].version = apps[app].version;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Additional information
|
|
if ( w.ping.hostnames.hasOwnProperty(hostname) ) {
|
|
match = html.match(/<html[^>]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"/i);
|
|
|
|
if ( match && match.length ) {
|
|
w.ping.hostnames[hostname].meta['language'] = match[1];
|
|
}
|
|
}
|
|
|
|
if ( Object.keys(w.ping.hostnames).length >= 50 || w.adCache.length >= 50 ) {
|
|
driver('ping');
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Analyze URL
|
|
*/
|
|
analyzeUrl: function(app, url) {
|
|
var patterns = parsePatterns(w.apps[app.app].url);
|
|
|
|
if ( patterns.length ) {
|
|
patterns.forEach(function(pattern) {
|
|
if ( pattern.regex.test(url) ) {
|
|
app.setDetected(pattern, 'url', url);
|
|
}
|
|
});
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Analyze HTML
|
|
*/
|
|
analyzeHtml: function(app, html) {
|
|
var patterns = parsePatterns(w.apps[app.app].html);
|
|
|
|
if ( patterns.length ) {
|
|
patterns.forEach(function(pattern) {
|
|
if ( pattern.regex.test(html) ) {
|
|
app.setDetected(pattern, 'html', html);
|
|
}
|
|
});
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Analyze script tag
|
|
*/
|
|
analyzeScript: function(app, html) {
|
|
var
|
|
regex = new RegExp('<script[^>]+src=("|\')([^"\']+)', 'ig'),
|
|
patterns = parsePatterns(w.apps[app.app].script);
|
|
|
|
if ( patterns.length ) {
|
|
patterns.forEach(function(pattern) {
|
|
var match;
|
|
|
|
while ( match = regex.exec(html) ) {
|
|
if ( pattern.regex.test(match[2]) ) {
|
|
app.setDetected(pattern, 'script', match[2]);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Analyze meta tag
|
|
*/
|
|
analyzeMeta: function(app, html) {
|
|
var
|
|
content, match, meta,
|
|
regex = /<meta[^>]+>/ig,
|
|
patterns = parsePatterns(w.apps[app.app].meta);
|
|
|
|
if ( patterns.length ) {
|
|
while ( match = regex.exec(html) ) {
|
|
for ( meta in patterns ) {
|
|
if ( new RegExp('(name|property)=["\']' + meta + '["\']', 'i').test(match) ) {
|
|
content = match.toString().match(/content=("|')([^"']+)("|')/i);
|
|
|
|
patterns[meta].forEach(function(pattern) {
|
|
if ( content && content.length === 4 && pattern.regex.test(content[2]) ) {
|
|
app.setDetected(pattern, 'meta', content[2], meta);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
|
|
/**
|
|
* analyze response headers
|
|
*/
|
|
analyzeHeaders: function(app, headers) {
|
|
var
|
|
header,
|
|
patterns = parsePatterns(w.apps[app.app].headers);
|
|
|
|
if ( headers ) {
|
|
for ( header in patterns ) {
|
|
patterns[header].forEach(function(pattern) {
|
|
header = header.toLowerCase();
|
|
|
|
if ( headers.hasOwnProperty(header) && pattern.regex.test(headers[header]) ) {
|
|
app.setDetected(pattern, 'headers', headers[header], header);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Analyze environment variables
|
|
*/
|
|
analyzeEnv: function(app, envs) {
|
|
var patterns = parsePatterns(w.apps[app.app].env);
|
|
|
|
if ( patterns.length ) {
|
|
patterns.forEach(function(pattern) {
|
|
var env;
|
|
|
|
for ( env in envs ) {
|
|
if ( pattern.regex.test(envs[env]) ) {
|
|
app.setDetected(pattern, 'env', envs[env]);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
};
|
|
|
|
return w;
|
|
})();
|
|
|
|
// CommonJS package
|
|
// See http://wiki.commonjs.org/wiki/CommonJS
|
|
if ( typeof exports === 'object' ) {
|
|
exports.wappalyzer = wappalyzer;
|
|
}
|