Migrations
/
wappalyzer
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove async/await in NPM driver for ES6 compatibility

Browse Source
main
Elbert Alias 8 years ago
parent 3bda8533b2
commit 50818de3b8
1 changed files with 67 additions and 66 deletions
Show Stats Download Patch File Download Diff File

133
src/drivers/npm/driver.js
Unescape View File

@ -20,6 +20,13 @@ class Driver {
userAgent: 'Mozilla/5.0 (compatible; Wappalyzer)', userAgent: 'Mozilla/5.0 (compatible; Wappalyzer)',
}, options || {}); }, options || {});
this.options.debug = Boolean(this.options.debug);
this.options.recursive = Boolean(this.options.recursive);
this.options.delay = this.options.recursive ? parseInt(this.options.delay, 10) : 0;
this.options.maxDepth = parseInt(this.options.maxDepth, 10);
this.options.maxUrls = parseInt(this.options.maxUrls, 10);
this.options.maxWait = parseInt(this.options.maxWait, 10);
this.origPageUrl = url.parse(pageUrl); this.origPageUrl = url.parse(pageUrl);
this.analyzedPageUrls = []; this.analyzedPageUrls = [];
this.apps = []; this.apps = [];
@ -38,9 +45,7 @@ class Driver {
} }
log(message, source, type) { log(message, source, type) {
if ( Boolean(this.options.debug) ) { this.options.debug && console.log('[wappalyzer ' + type + ']', '[' + source + ']', message);
console.log('[wappalyzer ' + type + ']', '[' + source + ']', message);
}
} }
displayApps(detected) { displayApps(detected) {
@ -71,7 +76,7 @@ class Driver {
} }
fetch(pageUrl, index, depth) { fetch(pageUrl, index, depth) {
return new Promise(async resolve => { return new Promise(resolve => {
// Return when the URL is a duplicate or maxUrls has been reached // Return when the URL is a duplicate or maxUrls has been reached
if ( this.analyzedPageUrls.indexOf(pageUrl.href) !== -1 || this.analyzedPageUrls.length >= this.options.maxUrls ) { if ( this.analyzedPageUrls.indexOf(pageUrl.href) !== -1 || this.analyzedPageUrls.length >= this.options.maxUrls ) {
return resolve(); return resolve();
@ -81,80 +86,76 @@ class Driver {
this.wappalyzer.log('depth: ' + depth + '; delay: ' + ( this.options.delay * index ) + 'ms; url: ' + pageUrl.href, 'driver'); this.wappalyzer.log('depth: ' + depth + '; delay: ' + ( this.options.delay * index ) + 'ms; url: ' + pageUrl.href, 'driver');
// Be nice
if ( this.options.delay ) {
await this.sleep(this.options.delay * index);
}
const browser = new Browser({ const browser = new Browser({
userAgent: this.options.userAgent, userAgent: this.options.userAgent,
waitDuration: this.options.maxWait + 'ms', waitDuration: this.options.maxWait + 'ms',
}); });
browser.visit(pageUrl.href, error => { this.sleep(this.options.delay * index)
if ( !browser.resources['0'] || !browser.resources['0'].response ) { .then(() => {
this.wappalyzer.log('No response from server', 'browser', 'error'); browser.visit(pageUrl.href, error => {
if ( !browser.resources['0'] || !browser.resources['0'].response ) {
return resolve(); this.wappalyzer.log('No response from server', 'browser', 'error');
}
return resolve();
browser.wait() }
.catch(error => this.wappalyzer.log(error.message, 'browser', 'error'))
.finally(() => { browser.wait()
const headers = {}; .catch(error => this.wappalyzer.log(error.message, 'browser', 'error'))
.finally(() => {
browser.resources['0'].response.headers._headers.forEach(header => { const headers = {};
if ( !headers[header[0]] ){
headers[header[0]] = []; browser.resources['0'].response.headers._headers.forEach(header => {
} if ( !headers[header[0]] ){
headers[header[0]] = [];
headers[header[0]].push(header[1]); }
});
headers[header[0]].push(header[1]);
const vars = Object.getOwnPropertyNames(browser.window); });
const html = browser.html();
const scripts = Array.prototype.slice const vars = Object.getOwnPropertyNames(browser.window);
.apply(browser.document.scripts) const html = browser.html();
.filter(s => s.src) const scripts = Array.prototype.slice
.map(s => s.src); .apply(browser.document.scripts)
.filter(s => s.src)
this.wappalyzer.analyze(pageUrl.hostname, pageUrl.href, { .map(s => s.src);
headers,
html, this.wappalyzer.analyze(pageUrl.hostname, pageUrl.href, {
env: vars, headers,
scripts html,
}); env: vars,
scripts
const links = browser.body.getElementsByTagName('a'); });
resolve(links); const links = browser.body.getElementsByTagName('a');
resolve(links);
});
}); });
}); });
}); });
} }
async crawl(pageUrl, index = 1, depth = 1) { crawl(pageUrl, index = 1, depth = 1) {
try { return new Promise(resolve => {
var links = await this.fetch(pageUrl, index, depth); this.fetch(pageUrl, index, depth)
.then(links => {
if ( this.options.recursive && depth < this.options.maxDepth && links ) { if ( links && Boolean(this.options.recursive) && depth < this.options.maxDepth ) {
links = Array.from(links).filter(link => link.hostname === this.origPageUrl.hostname); links = Array.from(links)
.filter(link => link.hostname === this.origPageUrl.hostname)
await Promise.all(links.map(async (link, index) => { .map(link => { link.hash = ''; return link });
link.hash = '';
return Promise.all(links.map((link, index) => this.crawl(link, index + 1, depth + 1)));
return this.crawl(link, index + 1, depth + 1); } else {
})); return Promise.resolve();
} }
})
return Promise.resolve(this.apps); .then(() => resolve(this.apps));
} catch (error) { });
return Promise.reject(error);
}
} }
sleep(ms) { sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms)); return ms ? new Promise(resolve => setTimeout(resolve, ms)) : Promise.resolve();
} }
}; };