|
|
|
|
@ -13,517 +13,523 @@ const validation = {
|
|
|
|
|
hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
var wappalyzer = {
|
|
|
|
|
apps: {},
|
|
|
|
|
categories: {},
|
|
|
|
|
driver: {}
|
|
|
|
|
};
|
|
|
|
|
class Wappalyzer {
|
|
|
|
|
constructor() {
|
|
|
|
|
this.apps = {};
|
|
|
|
|
this.categories = {};
|
|
|
|
|
this.driver = {};
|
|
|
|
|
|
|
|
|
|
this.detected = {};
|
|
|
|
|
this.hostnameCache = {};
|
|
|
|
|
this.adCache = [];
|
|
|
|
|
|
|
|
|
|
this.config = {
|
|
|
|
|
websiteURL: 'https://wappalyzer.com/',
|
|
|
|
|
twitterURL: 'https://twitter.com/Wappalyzer',
|
|
|
|
|
githubURL: 'https://github.com/AliasIO/Wappalyzer',
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var detected = {};
|
|
|
|
|
var hostnameCache = {};
|
|
|
|
|
var adCache = [];
|
|
|
|
|
/**
|
|
|
|
|
* Log messages to console
|
|
|
|
|
*/
|
|
|
|
|
log(message, source, type) {
|
|
|
|
|
this.driver.log(message, source || '', type || 'debug');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
wappalyzer.config = {
|
|
|
|
|
websiteURL: 'https://wappalyzer.com/',
|
|
|
|
|
twitterURL: 'https://twitter.com/Wappalyzer',
|
|
|
|
|
githubURL: 'https://github.com/AliasIO/Wappalyzer',
|
|
|
|
|
};
|
|
|
|
|
analyze(hostname, url, data, context) {
|
|
|
|
|
var apps = {};
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Log messages to console
|
|
|
|
|
*/
|
|
|
|
|
wappalyzer.log = (message, source, type) => {
|
|
|
|
|
wappalyzer.driver.log(message, source || '', type || 'debug');
|
|
|
|
|
};
|
|
|
|
|
// Remove hash from URL
|
|
|
|
|
data.url = url = url.split('#')[0];
|
|
|
|
|
|
|
|
|
|
wappalyzer.analyze = (hostname, url, data, context) => {
|
|
|
|
|
var apps = {};
|
|
|
|
|
|
|
|
|
|
// Remove hash from URL
|
|
|
|
|
data.url = url = url.split('#')[0];
|
|
|
|
|
if ( typeof data.html !== 'string' ) {
|
|
|
|
|
data.html = '';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( typeof data.html !== 'string' ) {
|
|
|
|
|
data.html = '';
|
|
|
|
|
}
|
|
|
|
|
if ( this.detected[url] === undefined ) {
|
|
|
|
|
this.detected[url] = {};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( detected[url] === undefined ) {
|
|
|
|
|
detected[url] = {};
|
|
|
|
|
}
|
|
|
|
|
Object.keys(this.apps).forEach(appName => {
|
|
|
|
|
apps[appName] = this.detected[url] && this.detected[url][appName] ? this.detected[url][appName] : new Application(appName, this.apps[appName]);
|
|
|
|
|
|
|
|
|
|
Object.keys(wappalyzer.apps).forEach(appName => {
|
|
|
|
|
apps[appName] = detected[url] && detected[url][appName] ? detected[url][appName] : new Application(appName, wappalyzer.apps[appName]);
|
|
|
|
|
var app = apps[appName];
|
|
|
|
|
|
|
|
|
|
var app = apps[appName];
|
|
|
|
|
if ( url ) {
|
|
|
|
|
this.analyzeUrl(app, url);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( url ) {
|
|
|
|
|
analyzeUrl(app, url);
|
|
|
|
|
}
|
|
|
|
|
if ( data.html ) {
|
|
|
|
|
this.analyzeHtml(app, data.html);
|
|
|
|
|
this.analyzeScript(app, data.html);
|
|
|
|
|
this.analyzeMeta(app, data.html);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( data.html ) {
|
|
|
|
|
analyzeHtml(app, data.html);
|
|
|
|
|
analyzeScript(app, data.html);
|
|
|
|
|
analyzeMeta(app, data.html);
|
|
|
|
|
}
|
|
|
|
|
if ( data.headers ) {
|
|
|
|
|
this.analyzeHeaders(app, data.headers);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( data.headers ) {
|
|
|
|
|
analyzeHeaders(app, data.headers);
|
|
|
|
|
}
|
|
|
|
|
if ( data.env ) {
|
|
|
|
|
this.analyzeEnv(app, data.env);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( data.env ) {
|
|
|
|
|
analyzeEnv(app, data.env);
|
|
|
|
|
}
|
|
|
|
|
if ( data.robotsTxt ) {
|
|
|
|
|
this.analyzeRobotsTxt(app, data.robotsTxt);
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
if ( data.robotsTxt ) {
|
|
|
|
|
analyzeRobotsTxt(app, data.robotsTxt);
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
Object.keys(apps).forEach(appName => {
|
|
|
|
|
var app = apps[appName];
|
|
|
|
|
|
|
|
|
|
Object.keys(apps).forEach(appName => {
|
|
|
|
|
var app = apps[appName];
|
|
|
|
|
if ( !app.detected || !app.getConfidence() ) {
|
|
|
|
|
delete apps[app.name];
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if ( !app.detected || !app.getConfidence() ) {
|
|
|
|
|
delete apps[app.name];
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
this.resolveExcludes(apps);
|
|
|
|
|
this.resolveImplies(apps, url);
|
|
|
|
|
|
|
|
|
|
resolveExcludes(apps);
|
|
|
|
|
resolveImplies(apps, url);
|
|
|
|
|
this.cacheDetectedApps(apps, url);
|
|
|
|
|
this.trackDetectedApps(apps, url, hostname, data.html);
|
|
|
|
|
|
|
|
|
|
cacheDetectedApps(apps, url);
|
|
|
|
|
trackDetectedApps(apps, url, hostname, data.html);
|
|
|
|
|
if ( Object.keys(apps).length ) {
|
|
|
|
|
this.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url, 'core');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( Object.keys(apps).length ) {
|
|
|
|
|
wappalyzer.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url, 'core');
|
|
|
|
|
this.driver.displayApps(this.detected[url], context);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
wappalyzer.driver.displayApps(detected[url], context);
|
|
|
|
|
}
|
|
|
|
|
/**
|
|
|
|
|
* Cache detected ads
|
|
|
|
|
*/
|
|
|
|
|
cacheDetectedAds(ad) {
|
|
|
|
|
this.adCache.push(ad);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Cache detected ads
|
|
|
|
|
*/
|
|
|
|
|
wappalyzer.cacheDetectedAds = ad => {
|
|
|
|
|
adCache.push(ad);
|
|
|
|
|
}
|
|
|
|
|
/**
|
|
|
|
|
*
|
|
|
|
|
*/
|
|
|
|
|
robotsTxtAllows(url) {
|
|
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
|
var parsed = this.parseUrl(url);
|
|
|
|
|
|
|
|
|
|
this.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
|
|
|
|
|
.then(robotsTxt => {
|
|
|
|
|
robotsTxt.forEach(disallow => {
|
|
|
|
|
if ( parsed.pathname.indexOf(disallow) === 0 ) {
|
|
|
|
|
reject();
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
*
|
|
|
|
|
*/
|
|
|
|
|
wappalyzer.robotsTxtAllows = url => {
|
|
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
|
var parsed = wappalyzer.parseUrl(url);
|
|
|
|
|
|
|
|
|
|
wappalyzer.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
|
|
|
|
|
.then(robotsTxt => {
|
|
|
|
|
robotsTxt.forEach(disallow => {
|
|
|
|
|
if ( parsed.pathname.indexOf(disallow) === 0 ) {
|
|
|
|
|
reject();
|
|
|
|
|
}
|
|
|
|
|
resolve();
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
resolve();
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Parse a URL
|
|
|
|
|
*/
|
|
|
|
|
wappalyzer.parseUrl = url => {
|
|
|
|
|
var a = wappalyzer.driver.document.createElement('a');
|
|
|
|
|
/**
|
|
|
|
|
* Parse a URL
|
|
|
|
|
*/
|
|
|
|
|
parseUrl(url) {
|
|
|
|
|
var a = this.driver.document.createElement('a');
|
|
|
|
|
|
|
|
|
|
a.href = url;
|
|
|
|
|
a.href = url;
|
|
|
|
|
|
|
|
|
|
a.canonical = a.protocol + '//' + a.host + a.pathname;
|
|
|
|
|
a.canonical = a.protocol + '//' + a.host + a.pathname;
|
|
|
|
|
|
|
|
|
|
return a;
|
|
|
|
|
}
|
|
|
|
|
return a;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
*
|
|
|
|
|
*/
|
|
|
|
|
wappalyzer.parseRobotsTxt = robotsTxt => {
|
|
|
|
|
var userAgent;
|
|
|
|
|
var disallow = [];
|
|
|
|
|
/**
|
|
|
|
|
*
|
|
|
|
|
*/
|
|
|
|
|
parseRobotsTxt(robotsTxt) {
|
|
|
|
|
var userAgent;
|
|
|
|
|
var disallow = [];
|
|
|
|
|
|
|
|
|
|
robotsTxt.split('\n').forEach(line => {
|
|
|
|
|
var matches = /^User-agent:\s*(.+)$/i.exec(line);
|
|
|
|
|
robotsTxt.split('\n').forEach(line => {
|
|
|
|
|
var matches = /^User-agent:\s*(.+)$/i.exec(line);
|
|
|
|
|
|
|
|
|
|
if ( matches ) {
|
|
|
|
|
userAgent = matches[1].toLowerCase();
|
|
|
|
|
} else {
|
|
|
|
|
if ( userAgent === '*' || userAgent === 'wappalyzer' ) {
|
|
|
|
|
matches = /^Disallow:\s*(.+)$/i.exec(line);
|
|
|
|
|
if ( matches ) {
|
|
|
|
|
userAgent = matches[1].toLowerCase();
|
|
|
|
|
} else {
|
|
|
|
|
if ( userAgent === '*' || userAgent === 'wappalyzer' ) {
|
|
|
|
|
matches = /^Disallow:\s*(.+)$/i.exec(line);
|
|
|
|
|
|
|
|
|
|
if ( matches ) {
|
|
|
|
|
disallow.push(matches[1]);
|
|
|
|
|
if ( matches ) {
|
|
|
|
|
disallow.push(matches[1]);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
return disallow;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
*
|
|
|
|
|
*/
|
|
|
|
|
wappalyzer.ping = () => {
|
|
|
|
|
if ( Object.keys(hostnameCache).length >= 50 || adCache.length >= 50 ) {
|
|
|
|
|
wappalyzer.driver.ping(hostnameCache, adCache);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
hostnameCache = {};
|
|
|
|
|
adCache = [];
|
|
|
|
|
return disallow;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Enclose string in array
|
|
|
|
|
*/
|
|
|
|
|
function asArray(value) {
|
|
|
|
|
return typeof value === 'string' ? [ value ] : value;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Parse apps.json patterns
|
|
|
|
|
*/
|
|
|
|
|
function parsePatterns(patterns) {
|
|
|
|
|
var parsed = {};
|
|
|
|
|
/**
|
|
|
|
|
*
|
|
|
|
|
*/
|
|
|
|
|
ping() {
|
|
|
|
|
if ( Object.keys(this.hostnameCache).length >= 50 || this.adCache.length >= 50 ) {
|
|
|
|
|
this.driver.ping(this.hostnameCache, this.adCache);
|
|
|
|
|
|
|
|
|
|
// Convert string to object containing array containing string
|
|
|
|
|
if ( typeof patterns === 'string' || patterns instanceof Array ) {
|
|
|
|
|
patterns = {
|
|
|
|
|
main: asArray(patterns)
|
|
|
|
|
};
|
|
|
|
|
this.hostnameCache = {};
|
|
|
|
|
this.adCache = [];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for ( var key in patterns ) {
|
|
|
|
|
parsed[key] = [];
|
|
|
|
|
|
|
|
|
|
asArray(patterns[key]).forEach(pattern => {
|
|
|
|
|
var attrs = {};
|
|
|
|
|
|
|
|
|
|
pattern.split('\\;').forEach((attr, i) => {
|
|
|
|
|
if ( i ) {
|
|
|
|
|
// Key value pairs
|
|
|
|
|
attr = attr.split(':');
|
|
|
|
|
|
|
|
|
|
if ( attr.length > 1 ) {
|
|
|
|
|
attrs[attr.shift()] = attr.join(':');
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
attrs.string = attr;
|
|
|
|
|
/**
|
|
|
|
|
* Enclose string in array
|
|
|
|
|
*/
|
|
|
|
|
asArray(value) {
|
|
|
|
|
return typeof value === 'string' ? [ value ] : value;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
attrs.regex = new RegExp(attr.replace('/', '\/'), 'i'); // Escape slashes in regular expression
|
|
|
|
|
} catch (e) {
|
|
|
|
|
attrs.regex = new RegExp();
|
|
|
|
|
/**
|
|
|
|
|
* Parse apps.json patterns
|
|
|
|
|
*/
|
|
|
|
|
parsePatterns(patterns) {
|
|
|
|
|
var parsed = {};
|
|
|
|
|
|
|
|
|
|
// Convert string to object containing array containing string
|
|
|
|
|
if ( typeof patterns === 'string' || patterns instanceof Array ) {
|
|
|
|
|
patterns = {
|
|
|
|
|
main: this.asArray(patterns)
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
wappalyzer.log(e + ': ' + attr, 'error', 'core');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
for ( var key in patterns ) {
|
|
|
|
|
parsed[key] = [];
|
|
|
|
|
|
|
|
|
|
parsed[key].push(attrs);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
this.asArray(patterns[key]).forEach(pattern => {
|
|
|
|
|
var attrs = {};
|
|
|
|
|
|
|
|
|
|
// Convert back to array if the original pattern list was an array (or string)
|
|
|
|
|
if ( 'main' in parsed ) {
|
|
|
|
|
parsed = parsed.main;
|
|
|
|
|
}
|
|
|
|
|
pattern.split('\\;').forEach((attr, i) => {
|
|
|
|
|
if ( i ) {
|
|
|
|
|
// Key value pairs
|
|
|
|
|
attr = attr.split(':');
|
|
|
|
|
|
|
|
|
|
return parsed;
|
|
|
|
|
}
|
|
|
|
|
if ( attr.length > 1 ) {
|
|
|
|
|
attrs[attr.shift()] = attr.join(':');
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
attrs.string = attr;
|
|
|
|
|
|
|
|
|
|
function resolveExcludes(apps) {
|
|
|
|
|
var excludes = [];
|
|
|
|
|
try {
|
|
|
|
|
attrs.regex = new RegExp(attr.replace('/', '\/'), 'i'); // Escape slashes in regular expression
|
|
|
|
|
} catch (e) {
|
|
|
|
|
attrs.regex = new RegExp();
|
|
|
|
|
|
|
|
|
|
// Exclude app in detected apps only
|
|
|
|
|
Object.keys(apps).forEach(appName => {
|
|
|
|
|
var app = apps[appName];
|
|
|
|
|
this.log(e + ': ' + attr, 'error', 'core');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if ( app.props.excludes ) {
|
|
|
|
|
asArray(app.props.excludes).forEach(excluded => {
|
|
|
|
|
excludes.push(excluded);
|
|
|
|
|
parsed[key].push(attrs);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
// Remove excluded applications
|
|
|
|
|
Object.keys(apps).forEach(appName => {
|
|
|
|
|
if ( excludes.indexOf(appName) !== -1 ) {
|
|
|
|
|
delete apps[appName];
|
|
|
|
|
// Convert back to array if the original pattern list was an array (or string)
|
|
|
|
|
if ( 'main' in parsed ) {
|
|
|
|
|
parsed = parsed.main;
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function resolveImplies(apps, url) {
|
|
|
|
|
var checkImplies = true;
|
|
|
|
|
return parsed;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Implied applications
|
|
|
|
|
// Run several passes as implied apps may imply other apps
|
|
|
|
|
while ( checkImplies ) {
|
|
|
|
|
checkImplies = false;
|
|
|
|
|
resolveExcludes(apps) {
|
|
|
|
|
var excludes = [];
|
|
|
|
|
|
|
|
|
|
// Exclude app in detected apps only
|
|
|
|
|
Object.keys(apps).forEach(appName => {
|
|
|
|
|
var app = apps[appName];
|
|
|
|
|
|
|
|
|
|
if ( app && app.implies ) {
|
|
|
|
|
asArray(app.props.implies).forEach(implied => {
|
|
|
|
|
implied = parsePatterns(implied)[0];
|
|
|
|
|
|
|
|
|
|
if ( !wappalyzer.apps[implied.string] ) {
|
|
|
|
|
wappalyzer.log('Implied application ' + implied.string + ' does not exist', 'core', 'warn');
|
|
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( !( implied.string in apps ) ) {
|
|
|
|
|
apps[implied.string] = detected[url] && detected[url][implied.string] ? detected[url][implied.string] : new Application(implied.string, true);
|
|
|
|
|
|
|
|
|
|
checkImplies = true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Apply app confidence to implied app
|
|
|
|
|
Object.keys(app.confidence).forEach(id => {
|
|
|
|
|
apps[implied.string].confidence[id + ' implied by ' + appName] = app.confidence[id] * ( implied.confidence ? implied.confidence / 100 : 1 );
|
|
|
|
|
});
|
|
|
|
|
if ( app.props.excludes ) {
|
|
|
|
|
this.asArray(app.props.excludes).forEach(excluded => {
|
|
|
|
|
excludes.push(excluded);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
// Remove excluded applications
|
|
|
|
|
Object.keys(apps).forEach(appName => {
|
|
|
|
|
if ( excludes.indexOf(appName) !== -1 ) {
|
|
|
|
|
delete apps[appName];
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Cache detected applications
|
|
|
|
|
*/
|
|
|
|
|
function cacheDetectedApps(apps, url) {
|
|
|
|
|
if (!wappalyzer.driver.ping instanceof Function) return;
|
|
|
|
|
resolveImplies(apps, url) {
|
|
|
|
|
var checkImplies = true;
|
|
|
|
|
|
|
|
|
|
Object.keys(apps).forEach(appName => {
|
|
|
|
|
var app = apps[appName];
|
|
|
|
|
// Implied applications
|
|
|
|
|
// Run several passes as implied apps may imply other apps
|
|
|
|
|
while ( checkImplies ) {
|
|
|
|
|
checkImplies = false;
|
|
|
|
|
|
|
|
|
|
// Per URL
|
|
|
|
|
detected[url][appName] = app;
|
|
|
|
|
Object.keys(apps).forEach(appName => {
|
|
|
|
|
var app = apps[appName];
|
|
|
|
|
|
|
|
|
|
Object.keys(app.confidence).forEach(id => {
|
|
|
|
|
detected[url][appName].confidence[id] = app.confidence[id];
|
|
|
|
|
});
|
|
|
|
|
})
|
|
|
|
|
if ( app && app.implies ) {
|
|
|
|
|
this.asArray(app.props.implies).forEach(implied => {
|
|
|
|
|
implied = this.parsePatterns(implied)[0];
|
|
|
|
|
|
|
|
|
|
wappalyzer.ping();
|
|
|
|
|
}
|
|
|
|
|
if ( !this.apps[implied.string] ) {
|
|
|
|
|
this.log('Implied application ' + implied.string + ' does not exist', 'core', 'warn');
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Track detected applications
|
|
|
|
|
*/
|
|
|
|
|
function trackDetectedApps(apps, url, hostname, html) {
|
|
|
|
|
if (!wappalyzer.driver.ping instanceof Function) return;
|
|
|
|
|
|
|
|
|
|
Object.keys(apps).forEach(appName => {
|
|
|
|
|
var app = apps[appName];
|
|
|
|
|
|
|
|
|
|
if ( detected[url][appName].getConfidence() >= 100 ) {
|
|
|
|
|
if ( validation.hostname.test(hostname) && !validation.hostnameBlacklist.test(url) ) {
|
|
|
|
|
wappalyzer.robotsTxtAllows(url)
|
|
|
|
|
.then(() => {
|
|
|
|
|
if ( !( hostname in hostnameCache ) ) {
|
|
|
|
|
hostnameCache[hostname] = {
|
|
|
|
|
applications: {},
|
|
|
|
|
meta: {}
|
|
|
|
|
};
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( !( appName in hostnameCache[hostname].applications ) ) {
|
|
|
|
|
hostnameCache[hostname].applications[appName] = {
|
|
|
|
|
hits: 0
|
|
|
|
|
};
|
|
|
|
|
if ( !( implied.string in apps ) ) {
|
|
|
|
|
apps[implied.string] = this.detected[url] && this.detected[url][implied.string] ? this.detected[url][implied.string] : new Application(implied.string, true);
|
|
|
|
|
|
|
|
|
|
checkImplies = true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
hostnameCache[hostname].applications[appName].hits ++;
|
|
|
|
|
// Apply app confidence to implied app
|
|
|
|
|
Object.keys(app.confidence).forEach(id => {
|
|
|
|
|
apps[implied.string].confidence[id + ' implied by ' + appName] = app.confidence[id] * ( implied.confidence ? implied.confidence / 100 : 1 );
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( apps[appName].version ) {
|
|
|
|
|
hostnameCache[hostname].applications[appName].version = app.version;
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
.catch(() => console.log('Disallowed in robots.txt: ' + url))
|
|
|
|
|
}
|
|
|
|
|
/**
|
|
|
|
|
* Cache detected applications
|
|
|
|
|
*/
|
|
|
|
|
cacheDetectedApps(apps, url) {
|
|
|
|
|
if ( !( this.driver.ping instanceof Function ) ) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Additional information
|
|
|
|
|
if ( hostname in hostnameCache ) {
|
|
|
|
|
var match = html.match(/<html[^>]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"/i);
|
|
|
|
|
Object.keys(apps).forEach(appName => {
|
|
|
|
|
var app = apps[appName];
|
|
|
|
|
|
|
|
|
|
if ( match && match.length ) {
|
|
|
|
|
hostnameCache[hostname].meta['language'] = match[1];
|
|
|
|
|
}
|
|
|
|
|
// Per URL
|
|
|
|
|
this.detected[url][appName] = app;
|
|
|
|
|
|
|
|
|
|
Object.keys(app.confidence).forEach(id => {
|
|
|
|
|
this.detected[url][appName].confidence[id] = app.confidence[id];
|
|
|
|
|
});
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
this.ping();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
wappalyzer.ping();
|
|
|
|
|
}
|
|
|
|
|
/**
|
|
|
|
|
* Track detected applications
|
|
|
|
|
*/
|
|
|
|
|
trackDetectedApps(apps, url, hostname, html) {
|
|
|
|
|
if ( !( this.driver.ping instanceof Function ) ) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Analyze URL
|
|
|
|
|
*/
|
|
|
|
|
function analyzeUrl(app, url) {
|
|
|
|
|
var patterns = parsePatterns(app.props.url);
|
|
|
|
|
Object.keys(apps).forEach(appName => {
|
|
|
|
|
var app = apps[appName];
|
|
|
|
|
|
|
|
|
|
if ( patterns.length ) {
|
|
|
|
|
patterns.forEach(pattern => {
|
|
|
|
|
if ( pattern.regex.test(url) ) {
|
|
|
|
|
addDetected(app, pattern, 'url', url);
|
|
|
|
|
if ( this.detected[url][appName].getConfidence() >= 100 ) {
|
|
|
|
|
if ( validation.hostname.test(hostname) && !validation.hostnameBlacklist.test(url) ) {
|
|
|
|
|
this.robotsTxtAllows(url)
|
|
|
|
|
.then(() => {
|
|
|
|
|
if ( !( hostname in this.hostnameCache ) ) {
|
|
|
|
|
this.hostnameCache[hostname] = {
|
|
|
|
|
applications: {},
|
|
|
|
|
meta: {}
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( !( appName in this.hostnameCache[hostname].applications ) ) {
|
|
|
|
|
this.hostnameCache[hostname].applications[appName] = {
|
|
|
|
|
hits: 0
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
this.hostnameCache[hostname].applications[appName].hits ++;
|
|
|
|
|
|
|
|
|
|
if ( apps[appName].version ) {
|
|
|
|
|
this.hostnameCache[hostname].applications[appName].version = app.version;
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
.catch(() => console.log('Disallowed in robots.txt: ' + url))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Analyze HTML
|
|
|
|
|
*/
|
|
|
|
|
function analyzeHtml(app, html) {
|
|
|
|
|
var patterns = parsePatterns(app.props.html);
|
|
|
|
|
// Additional information
|
|
|
|
|
if ( hostname in this.hostnameCache ) {
|
|
|
|
|
var match = html.match(/<html[^>]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"/i);
|
|
|
|
|
|
|
|
|
|
if ( patterns.length ) {
|
|
|
|
|
patterns.forEach(pattern => {
|
|
|
|
|
if ( pattern.regex.test(html) ) {
|
|
|
|
|
addDetected(app, pattern, 'html', html);
|
|
|
|
|
if ( match && match.length ) {
|
|
|
|
|
this.hostnameCache[hostname].meta['language'] = match[1];
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
this.ping();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Analyze script tag
|
|
|
|
|
*/
|
|
|
|
|
function analyzeScript(app, html) {
|
|
|
|
|
var regex = new RegExp('<script[^>]+src=("|\')([^"\']+)', 'ig');
|
|
|
|
|
var patterns = parsePatterns(app.props.script);
|
|
|
|
|
/**
|
|
|
|
|
* Analyze URL
|
|
|
|
|
*/
|
|
|
|
|
analyzeUrl(app, url) {
|
|
|
|
|
var patterns = this.parsePatterns(app.props.url);
|
|
|
|
|
|
|
|
|
|
if ( patterns.length ) {
|
|
|
|
|
patterns.forEach(pattern => {
|
|
|
|
|
var match;
|
|
|
|
|
if ( patterns.length ) {
|
|
|
|
|
patterns.forEach(pattern => {
|
|
|
|
|
if ( pattern.regex.test(url) ) {
|
|
|
|
|
this.addDetected(app, pattern, 'url', url);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Analyze HTML
|
|
|
|
|
*/
|
|
|
|
|
analyzeHtml(app, html) {
|
|
|
|
|
var patterns = this.parsePatterns(app.props.html);
|
|
|
|
|
|
|
|
|
|
while ( ( match = regex.exec(html) ) ) {
|
|
|
|
|
if ( pattern.regex.test(match[2]) ) {
|
|
|
|
|
addDetected(app, pattern, 'script', match[2]);
|
|
|
|
|
if ( patterns.length ) {
|
|
|
|
|
patterns.forEach(pattern => {
|
|
|
|
|
if ( pattern.regex.test(html) ) {
|
|
|
|
|
this.addDetected(app, pattern, 'html', html);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Analyze meta tag
|
|
|
|
|
*/
|
|
|
|
|
function analyzeMeta(app, html) {
|
|
|
|
|
var regex = /<meta[^>]+>/ig;
|
|
|
|
|
var patterns = parsePatterns(app.props.meta);
|
|
|
|
|
var content;
|
|
|
|
|
var match;
|
|
|
|
|
|
|
|
|
|
while ( patterns && ( match = regex.exec(html) ) ) {
|
|
|
|
|
for ( var meta in patterns ) {
|
|
|
|
|
if ( new RegExp('(name|property)=["\']' + meta + '["\']', 'i').test(match) ) {
|
|
|
|
|
content = match.toString().match(/content=("|')([^"']+)("|')/i);
|
|
|
|
|
|
|
|
|
|
patterns[meta].forEach(pattern => {
|
|
|
|
|
if ( content && content.length === 4 && pattern.regex.test(content[2]) ) {
|
|
|
|
|
addDetected(app, pattern, 'meta', content[2], meta);
|
|
|
|
|
/**
|
|
|
|
|
* Analyze script tag
|
|
|
|
|
*/
|
|
|
|
|
analyzeScript(app, html) {
|
|
|
|
|
var regex = new RegExp('<script[^>]+src=("|\')([^"\']+)', 'ig');
|
|
|
|
|
var patterns = this.parsePatterns(app.props.script);
|
|
|
|
|
|
|
|
|
|
if ( patterns.length ) {
|
|
|
|
|
patterns.forEach(pattern => {
|
|
|
|
|
var match;
|
|
|
|
|
|
|
|
|
|
while ( ( match = regex.exec(html) ) ) {
|
|
|
|
|
if ( pattern.regex.test(match[2]) ) {
|
|
|
|
|
this.addDetected(app, pattern, 'script', match[2]);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Analyze meta tag
|
|
|
|
|
*/
|
|
|
|
|
analyzeMeta(app, html) {
|
|
|
|
|
var regex = /<meta[^>]+>/ig;
|
|
|
|
|
var patterns = this.parsePatterns(app.props.meta);
|
|
|
|
|
var content;
|
|
|
|
|
var match;
|
|
|
|
|
|
|
|
|
|
while ( patterns && ( match = regex.exec(html) ) ) {
|
|
|
|
|
for ( var meta in patterns ) {
|
|
|
|
|
if ( new RegExp('(name|property)=["\']' + meta + '["\']', 'i').test(match) ) {
|
|
|
|
|
content = match.toString().match(/content=("|')([^"']+)("|')/i);
|
|
|
|
|
|
|
|
|
|
patterns[meta].forEach(pattern => {
|
|
|
|
|
if ( content && content.length === 4 && pattern.regex.test(content[2]) ) {
|
|
|
|
|
this.addDetected(app, pattern, 'meta', content[2], meta);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* analyze response headers
|
|
|
|
|
*/
|
|
|
|
|
function analyzeHeaders(app, headers) {
|
|
|
|
|
var patterns = parsePatterns(app.props.headers);
|
|
|
|
|
/**
|
|
|
|
|
* analyze response headers
|
|
|
|
|
*/
|
|
|
|
|
analyzeHeaders(app, headers) {
|
|
|
|
|
var patterns = this.parsePatterns(app.props.headers);
|
|
|
|
|
|
|
|
|
|
if ( headers ) {
|
|
|
|
|
Object.keys(patterns).forEach(header => {
|
|
|
|
|
patterns[header].forEach(pattern => {
|
|
|
|
|
header = header.toLowerCase();
|
|
|
|
|
if ( headers ) {
|
|
|
|
|
Object.keys(patterns).forEach(header => {
|
|
|
|
|
patterns[header].forEach(pattern => {
|
|
|
|
|
header = header.toLowerCase();
|
|
|
|
|
|
|
|
|
|
if ( header in headers && pattern.regex.test(headers[header]) ) {
|
|
|
|
|
addDetected(app, pattern, 'headers', headers[header], header);
|
|
|
|
|
}
|
|
|
|
|
if ( header in headers && pattern.regex.test(headers[header]) ) {
|
|
|
|
|
this.addDetected(app, pattern, 'headers', headers[header], header);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Analyze environment variables
|
|
|
|
|
*/
|
|
|
|
|
function analyzeEnv(app, envs) {
|
|
|
|
|
var patterns = parsePatterns(app.props.env);
|
|
|
|
|
|
|
|
|
|
if ( patterns.length ) {
|
|
|
|
|
patterns.forEach(pattern => {
|
|
|
|
|
Object.keys(envs).forEach(env => {
|
|
|
|
|
if ( pattern.regex.test(envs[env]) ) {
|
|
|
|
|
addDetected(app, pattern, 'env', envs[env]);
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
});
|
|
|
|
|
/**
|
|
|
|
|
* Analyze environment variables
|
|
|
|
|
*/
|
|
|
|
|
analyzeEnv(app, envs) {
|
|
|
|
|
var patterns = this.parsePatterns(app.props.env);
|
|
|
|
|
|
|
|
|
|
if ( patterns.length ) {
|
|
|
|
|
patterns.forEach(pattern => {
|
|
|
|
|
Object.keys(envs).forEach(env => {
|
|
|
|
|
if ( pattern.regex.test(envs[env]) ) {
|
|
|
|
|
this.addDetected(app, pattern, 'env', envs[env]);
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Analyze robots.txt
|
|
|
|
|
*/
|
|
|
|
|
function analyzeRobotsTxt(app, robotsTxt) {
|
|
|
|
|
var patterns = parsePatterns(app.props.robotsTxt);
|
|
|
|
|
/**
|
|
|
|
|
* Analyze robots.txt
|
|
|
|
|
*/
|
|
|
|
|
analyzeRobotsTxt(app, robotsTxt) {
|
|
|
|
|
var patterns = this.parsePatterns(app.props.robotsTxt);
|
|
|
|
|
|
|
|
|
|
if ( patterns.length ) {
|
|
|
|
|
patterns.forEach(pattern => {
|
|
|
|
|
if ( pattern.regex.test(robotsTxt) ) {
|
|
|
|
|
addDetected(app, pattern, 'robotsTxt', robotsTxt);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
if ( patterns.length ) {
|
|
|
|
|
patterns.forEach(pattern => {
|
|
|
|
|
if ( pattern.regex.test(robotsTxt) ) {
|
|
|
|
|
this.addDetected(app, pattern, 'robotsTxt', robotsTxt);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Mark application as detected, set confidence and version
|
|
|
|
|
*/
|
|
|
|
|
function addDetected(app, pattern, type, value, key) {
|
|
|
|
|
app.detected = true;
|
|
|
|
|
/**
|
|
|
|
|
* Mark application as detected, set confidence and version
|
|
|
|
|
*/
|
|
|
|
|
addDetected(app, pattern, type, value, key) {
|
|
|
|
|
app.detected = true;
|
|
|
|
|
|
|
|
|
|
// Set confidence level
|
|
|
|
|
app.confidence[type + ' ' + ( key ? key + ' ' : '' ) + pattern.regex] = pattern.confidence || 100;
|
|
|
|
|
// Set confidence level
|
|
|
|
|
app.confidence[type + ' ' + ( key ? key + ' ' : '' ) + pattern.regex] = pattern.confidence || 100;
|
|
|
|
|
|
|
|
|
|
// Detect version number
|
|
|
|
|
if ( pattern.version ) {
|
|
|
|
|
var versions = [];
|
|
|
|
|
var version = pattern.version;
|
|
|
|
|
var matches = pattern.regex.exec(value);
|
|
|
|
|
// Detect version number
|
|
|
|
|
if ( pattern.version ) {
|
|
|
|
|
var versions = [];
|
|
|
|
|
var version = pattern.version;
|
|
|
|
|
var matches = pattern.regex.exec(value);
|
|
|
|
|
|
|
|
|
|
if ( matches ) {
|
|
|
|
|
matches.forEach((match, i) => {
|
|
|
|
|
// Parse ternary operator
|
|
|
|
|
var ternary = new RegExp('\\\\' + i + '\\?([^:]+):(.*)$').exec(version);
|
|
|
|
|
if ( matches ) {
|
|
|
|
|
matches.forEach((match, i) => {
|
|
|
|
|
// Parse ternary operator
|
|
|
|
|
var ternary = new RegExp('\\\\' + i + '\\?([^:]+):(.*)$').exec(version);
|
|
|
|
|
|
|
|
|
|
if ( ternary && ternary.length === 3 ) {
|
|
|
|
|
version = version.replace(ternary[0], match ? ternary[1] : ternary[2]);
|
|
|
|
|
}
|
|
|
|
|
if ( ternary && ternary.length === 3 ) {
|
|
|
|
|
version = version.replace(ternary[0], match ? ternary[1] : ternary[2]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Replace back references
|
|
|
|
|
version = version.replace(new RegExp('\\\\' + i, 'g'), match || '');
|
|
|
|
|
});
|
|
|
|
|
// Replace back references
|
|
|
|
|
version = version.replace(new RegExp('\\\\' + i, 'g'), match || '');
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if ( version && versions.indexOf(version) === -1 ) {
|
|
|
|
|
versions.push(version);
|
|
|
|
|
}
|
|
|
|
|
if ( version && versions.indexOf(version) === -1 ) {
|
|
|
|
|
versions.push(version);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( versions.length ) {
|
|
|
|
|
// Use the longest detected version number
|
|
|
|
|
app.version = versions.reduce((a, b) => a.length > b.length ? a : b);
|
|
|
|
|
if ( versions.length ) {
|
|
|
|
|
// Use the longest detected version number
|
|
|
|
|
app.version = versions.reduce((a, b) => a.length > b.length ? a : b);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@ -558,5 +564,5 @@ class Application {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ( typeof module === 'object' ) {
|
|
|
|
|
module.exports = wappalyzer;
|
|
|
|
|
module.exports = Wappalyzer;
|
|
|
|
|
}
|
|
|
|
|
|
Daniel Zdziebko
7 years ago
committed by
