diff --git a/src/drivers/npm/README.md b/src/drivers/npm/README.md index 7c1d950e6..f3c476e38 100644 --- a/src/drivers/npm/README.md +++ b/src/drivers/npm/README.md @@ -31,9 +31,10 @@ wappalyzer [options] -m, --max-urls=... Exit when num URLs have been analysed -w, --max-wait=... Wait no more than ms milliseconds for page resources to load -P, --pretty Pretty-print JSON output --p, --probe Perform a deeper scan by requesting common files +-p, --probe Perform a deeper scan by performing additional requests and inspecting DNS records -r, --recursive Follow links on pages (crawler) -a, --user-agent=... Set the user agent string +-n, --no-scripts Disabled JavaScript on web pages ``` @@ -64,6 +65,7 @@ const options = { userAgent: 'Wappalyzer', htmlMaxCols: 2000, htmlMaxRows: 2000, + noScripts: false, }; const wappalyzer = new Wappalyzer(options) diff --git a/src/drivers/npm/cli.js b/src/drivers/npm/cli.js index 197a6a122..d6e53d5a2 100755 --- a/src/drivers/npm/cli.js +++ b/src/drivers/npm/cli.js @@ -21,6 +21,7 @@ const aliases = { P: 'pretty', r: 'recursive', w: 'maxWait', + n: 'noScripts', } while (true) { @@ -69,10 +70,11 @@ Options: -D, --max-depth=... Don't analyse pages more than num levels deep -m, --max-urls=... Exit when num URLs have been analysed -w, --max-wait=... Wait no more than ms milliseconds for page resources to load - -p, --probe Perform a deeper scan by making additional network requests + -p, --probe Perform a deeper scan by performing additional requests and inspecting DNS records -P, --pretty Pretty-print JSON output -r, --recursive Follow links on pages (crawler) -a, --user-agent=... Set the user agent string + -n, --no-scripts Disabled JavaScript on web pages `) process.exit(1) diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 826384929..26feedd2b 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -282,6 +282,7 @@ class Driver { maxWait: 30000, recursive: false, probe: false, + noScripts: false, ...options, } @@ -294,6 +295,7 @@ class Driver { this.options.maxWait = parseInt(this.options.maxWait, 10) this.options.htmlMaxCols = parseInt(this.options.htmlMaxCols, 10) this.options.htmlMaxRows = parseInt(this.options.htmlMaxRows, 10) + this.options.noScripts = Boolean(+this.options.noScripts) this.destroyed = false } @@ -382,9 +384,6 @@ class Site { this.pages = [] - this.dnsChecked = false - this.dns = [] - this.cache = {} this.probed = false @@ -477,6 +476,8 @@ class Site { this.pages.push(page) + page.setJavaScriptEnabled(!this.options.noScripts) + page.setDefaultTimeout(this.options.maxWait) await page.setRequestInterception(true) @@ -512,7 +513,9 @@ class Site { if ( (responseReceived && request.isNavigationRequest()) || request.frame() !== page.mainFrame() || - !['document', 'script'].includes(request.resourceType()) + !['document', ...(this.options.noScripts ? [] : ['script'])].includes( + request.resourceType() + ) ) { request.abort('blockedbyclient') } else { @@ -582,7 +585,9 @@ class Site { 'Timeout (navigation)' ) - await sleep(1000) + if (!this.options.noScripts) { + await sleep(1000) + } // page.on('console', (message) => this.log(message.text())) @@ -690,7 +695,9 @@ class Site { ) // JavaScript - const js = await this.promiseTimeout(getJs(page), [], 'Timeout (js)') + const js = this.options.noScripts + ? [] + : await this.promiseTimeout(getJs(page), [], 'Timeout (js)') // DOM const dom = await this.promiseTimeout(getDom(page), [], 'Timeout (dom)') @@ -728,56 +735,6 @@ class Site { html = batches.join('\n') } - // DNS - if (!this.dnsChecked) { - this.dnsChecked = true - - const records = {} - const resolve = (func, hostname) => { - return this.promiseTimeout( - func(hostname).catch((error) => { - if (error.code !== 'ENODATA') { - this.error(error) - } - - return [] - }), - [], - 'Timeout (dns)' - ) - } - - const domain = url.hostname.replace(/^www\./, '') - - ;[records.cname, records.ns, records.mx, records.txt, records.soa] = - await Promise.all([ - resolve(dns.resolveCname, url.hostname), - resolve(dns.resolveNs, domain), - resolve(dns.resolveMx, domain), - resolve(dns.resolveTxt, domain), - resolve(dns.resolveSoa, domain), - ]) - - this.dns = Object.keys(records).reduce((dns, type) => { - dns[type] = dns[type] || [] - - Array.prototype.push.apply( - dns[type], - Array.isArray(records[type]) - ? records[type].map((value) => { - return typeof value === 'object' - ? Object.values(value).join(' ') - : value - }) - : [Object.values(records[type]).join(' ')] - ) - - return dns - }, {}) - - await this.onDetect(url, await analyze({ dns: this.dns })) - } - // Validate response if ( url.protocol !== 'file:' && @@ -797,7 +754,6 @@ class Site { cookies, scripts, meta, - dns: this.dns, } await this.onDetect( @@ -971,6 +927,52 @@ class Site { this.error(`get ${path}: ${error.message || error}`) } } + + // DNS + const records = {} + const resolve = (func, hostname) => { + return this.promiseTimeout( + func(hostname).catch((error) => { + if (error.code !== 'ENODATA') { + this.error(error) + } + + return [] + }), + [], + 'Timeout (dns)' + ) + } + + const domain = url.hostname.replace(/^www\./, '') + + ;[records.cname, records.ns, records.mx, records.txt, records.soa] = + await Promise.all([ + resolve(dns.resolveCname, url.hostname), + resolve(dns.resolveNs, domain), + resolve(dns.resolveMx, domain), + resolve(dns.resolveTxt, domain), + resolve(dns.resolveSoa, domain), + ]) + + const dnsRecords = Object.keys(records).reduce((dns, type) => { + dns[type] = dns[type] || [] + + Array.prototype.push.apply( + dns[type], + Array.isArray(records[type]) + ? records[type].map((value) => { + return typeof value === 'object' + ? Object.values(value).join(' ') + : value + }) + : [Object.values(records[type]).join(' ')] + ) + + return dns + }, {}) + + await this.onDetect(url, await analyze({ dns: dnsRecords })) } async batch(links, depth, batch = 0) { diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index 7b78b806d..b6349d89f 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -102,10 +102,10 @@ const Driver = { 'https://www.wappalyzer.com/installed/?utm_source=installed&utm_medium=extension&utm_campaign=wappalyzer' ) } else if (version !== previous && upgradeMessage) { - // open( - // `https://www.wappalyzer.com/upgraded/?utm_source=upgraded&utm_medium=extension&utm_campaign=wappalyzer`, - // false - // ) + open( + `https://www.wappalyzer.com/upgraded/?utm_source=upgraded&utm_medium=extension&utm_campaign=wappalyzer`, + false + ) } await setOption('version', version)