Migrations
/
wappalyzer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Asynchronous regex execution

Browse Source
main
Elbert Alias 7 years ago
parent 0a63ef2ddd
commit 74bb8fa56a
5 changed files with 112 additions and 96 deletions
Show Stats Download Patch File Download Diff File

17
src/drivers/npm/driver.js
Unescape View File

@ -131,15 +131,16 @@ class Driver {
html,
scripts,
js
});
const links = Array.from(browser.document.getElementsByTagName('a'))
.filter(link => link.protocol === 'http:' || link.protocol === 'https:')
.filter(link => link.hostname === this.origPageUrl.hostname)
.filter(link => extensions.test(link.pathname))
.map(link => { link.hash = ''; return url.parse(link.href) });
})
.then(() => {
const links = Array.from(browser.document.getElementsByTagName('a'))
.filter(link => link.protocol === 'http:' || link.protocol === 'https:')
.filter(link => link.hostname === this.origPageUrl.hostname)
.filter(link => extensions.test(link.pathname))
.map(link => { link.hash = ''; return url.parse(link.href) });
return resolve(links);
return resolve(links);
});
});
}

9
src/drivers/webextension/js/content.js
Unescape View File

@ -3,11 +3,12 @@
if ( typeof browser !== 'undefined' && typeof document.body !== 'undefined' ) {
try {
var html = new XMLSerializer().serializeToString(document);
var html = new XMLSerializer().serializeToString(document).split('\n');
if ( html.length > 100 * 1024 ) {
html = html.substring(0, 50 * 1024) + html.substring(html.length - 50 * 1024, html.length);
}
html = html
.slice(0, 1000).concat(html.slice(html.length - 1000))
.map(line => line.substring(0, 1000))
.join('\n');
const scripts = Array.prototype.slice
.apply(document.scripts)

8
src/drivers/webextension/js/driver.js
Unescape View File

@ -103,10 +103,10 @@ getOption('version')
getOption('upgradeMessage', true)
.then(upgradeMessage => {
if ( upgradeMessage ) {
// openTab({
// url: wappalyzer.config.websiteURL + 'upgraded?v' + version,
// background: true
// });
openTab({
url: wappalyzer.config.websiteURL + 'upgraded?v' + version,
background: true
});
}
});
}

2
src/drivers/webextension/js/inject.js
Unescape View File

@ -20,7 +20,7 @@
for ( let index in patterns[appName][chain] ) {
const value = detectJs(chain);
if ( value ) {
if ( value && patterns[appName][chain].hasOwnProperty(index) ) {
js[appName][chain][index] = value;
}
}

172
src/wappalyzer.js
Unescape View File

@ -38,7 +38,13 @@ class Wappalyzer {
this.driver.log(message, source || '', type || 'debug');
}
asyncForEach(iterable, iterator) {
return Promise.all(iterable.map(item => new Promise(resolve => setTimeout(() => resolve(iterator(item)), 1))));
}
analyze(url, data, context) {
const promises = [];
var apps = {};
if ( typeof data.html !== 'string' ) {
@ -62,52 +68,55 @@ class Wappalyzer {
this.analyzeUrl(app, url);
if ( data.html ) {
this.analyzeHtml(app, data.html);
this.analyzeMeta(app, data.html);
promises.push(this.analyzeHtml(app, data.html));
promises.push(this.analyzeMeta(app, data.html));
}
if ( data.scripts ) {
this.analyzeScripts(app, data.scripts);
promises.push(this.analyzeScripts(app, data.scripts));
}
if ( data.headers ) {
this.analyzeHeaders(app, data.headers);
promises.push(this.analyzeHeaders(app, data.headers));
}
if ( data.env ) {
this.analyzeEnv(app, data.env);
}
if ( data.robotsTxt ) {
this.analyzeRobotsTxt(app, data.robotsTxt);
promises.push(this.analyzeEnv(app, data.env));
}
})
if ( data.js ) {
Object.keys(data.js).forEach(appName => {
this.analyzeJs(apps[appName], data.js[appName]);
promises.push(this.analyzeJs(apps[appName], data.js[appName]));
});
}
Object.keys(apps).forEach(appName => {
var app = apps[appName];
return new Promise(resolve => {
Promise.all(promises)
.then(() => {
Object.keys(apps).forEach(appName => {
var app = apps[appName];
if ( !app.detected || !app.getConfidence() ) {
delete apps[app.name];
}
});
if ( !app.detected || !app.getConfidence() ) {
delete apps[app.name];
}
});
this.resolveExcludes(apps);
this.resolveImplies(apps, url.canonical);
this.resolveExcludes(apps);
this.resolveImplies(apps, url.canonical);
this.cacheDetectedApps(apps, url.canonical);
this.trackDetectedApps(apps, url, language);
this.cacheDetectedApps(apps, url.canonical);
this.trackDetectedApps(apps, url, language);
if ( Object.keys(apps).length ) {
this.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url.canonical, 'core');
}
if ( Object.keys(apps).length ) {
this.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url.canonical, 'core');
}
this.driver.displayApps(this.detected[url.canonical], { language }, context);
this.driver.displayApps(this.detected[url.canonical], { language }, context);
resolve();
});
});
}
/**
@ -393,13 +402,15 @@ class Wappalyzer {
analyzeUrl(app, url) {
var patterns = this.parsePatterns(app.props.url);
if ( patterns.length ) {
patterns.forEach(pattern => {
if ( pattern.regex.test(url.canonical) ) {
this.addDetected(app, pattern, 'url', url.canonical);
}
});
if ( !patterns.length ) {
return Promise.resolve();
}
return this.asyncForEach(patterns, pattern => {
if ( pattern.regex.test(url.canonical) ) {
this.addDetected(app, pattern, 'url', url.canonical);
}
});
}
/**
@ -408,13 +419,15 @@ class Wappalyzer {
analyzeHtml(app, html) {
var patterns = this.parsePatterns(app.props.html);
if ( patterns.length ) {
patterns.forEach(pattern => {
if ( pattern.regex.test(html) ) {
this.addDetected(app, pattern, 'html', html);
}
});
if ( !patterns.length ) {
return Promise.resolve();
}
return this.asyncForEach(patterns, pattern => {
if ( pattern.regex.test(html) ) {
this.addDetected(app, pattern, 'html', html);
}
});
}
/**
@ -423,67 +436,75 @@ class Wappalyzer {
analyzeScripts(app, scripts) {
var patterns = this.parsePatterns(app.props.script);
if ( patterns.length ) {
patterns.forEach(pattern => {
var match;
if ( !patterns.length ) {
return Promise.resolve();
}
scripts.forEach(uri => {
if ( pattern.regex.test(uri) ) {
this.addDetected(app, pattern, 'script', uri);
}
});
return this.asyncForEach(patterns, pattern => {
var match;
scripts.forEach(uri => {
if ( pattern.regex.test(uri) ) {
this.addDetected(app, pattern, 'script', uri);
}
});
}
});
}
/**
* Analyze meta tag
*/
analyzeMeta(app, html) {
var regex = /<meta[^>]+>/ig;
var patterns = this.parsePatterns(app.props.meta);
const regex = /<meta[^>]+>/ig;
const patterns = this.parsePatterns(app.props.meta);
const promises = [];
var content = '';
var matches = [];
while ( patterns && ( matches = regex.exec(html) ) ) {
for ( var meta in patterns ) {
const r = new RegExp('(?:name|property)=["\']' + meta + '["\']', 'i');
if ( new RegExp('(?:name|property)=["\']' + meta + '["\']', 'i').test(matches[0]) ) {
if ( r.test(matches[0]) ) {
content = matches[0].match(/content=("|')([^"']+)("|')/i);
patterns[meta].forEach(pattern => {
promises.push(this.asyncForEach(patterns[meta], pattern => {
if ( content && content.length === 4 && pattern.regex.test(content[2]) ) {
this.addDetected(app, pattern, 'meta', content[2], meta);
}
});
}));
}
}
}
return promises ? Promise.all(promises) : Promise.resolve();
}
/**
* analyze response headers
*/
analyzeHeaders(app, headers) {
var patterns = this.parsePatterns(app.props.headers);
const patterns = this.parsePatterns(app.props.headers);
const promises = [];
if ( headers ) {
Object.keys(patterns).forEach(headerName => {
patterns[headerName].forEach(pattern => {
this.asyncForEach(patterns[headerName], pattern => {
headerName = headerName.toLowerCase();
if ( headerName in headers ) {
headers[headerName].forEach(headerValue => {
promises.push(headers[headerName].forEach(headerValue => {
if ( pattern.regex.test(headerValue) ) {
this.addDetected(app, pattern, 'headers', headerValue, headerName);
}
});
}));
}
});
});
}
return promises ? Promise.all(promises) : Promise.resolve();
}
/**
@ -493,45 +514,36 @@ class Wappalyzer {
var patterns = this.parsePatterns(app.props.env);
if ( patterns.length ) {
patterns.forEach(pattern => {
Object.keys(envs).forEach(env => {
if ( pattern.regex.test(envs[env]) ) {
this.addDetected(app, pattern, 'env', envs[env]);
}
})
});
return Promise.resolve();
}
return this.asyncForEach(patterns, pattern => {
Object.keys(envs).forEach(env => {
if ( pattern.regex.test(envs[env]) ) {
this.addDetected(app, pattern, 'env', envs[env]);
}
})
});
}
/**
* Analyze JavaScript variables
*/
analyzeJs(app, results) {
const promises = [];
Object.keys(results).forEach(string => {
Object.keys(results[string]).forEach(index => {
promises.push(this.asyncForEach(Object.keys(results[string]), index => {
const pattern = this.jsPatterns[app.name][string][index];
const value = results[string][index];
if ( pattern.regex.test(value) ) {
if ( pattern && pattern.regex.test(value) ) {
this.addDetected(app, pattern, 'js', value);
}
});
}));
});
}
/**
* Analyze robots.txt
*/
analyzeRobotsTxt(app, robotsTxt) {
var patterns = this.parsePatterns(app.props.robotsTxt);
if ( patterns.length ) {
patterns.forEach(pattern => {
if ( pattern.regex.test(robotsTxt) ) {
this.addDetected(app, pattern, 'robotsTxt', robotsTxt);
}
});
}
return promises ? Promise.all(promises) : Promise.resolve();
}
/**
@ -540,6 +552,8 @@ class Wappalyzer {
addDetected(app, pattern, type, value, key) {
app.detected = true;
console.log(app);
// Set confidence level
app.confidence[type + ' ' + ( key ? key + ' ' : '' ) + pattern.regex] = pattern.confidence || 100;

Write Preview
Loading…
Cancel
Save