From f4fc3e66a6b52c0577b5d99de6ebc18faeba8bae Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Mon, 2 Jan 2023 10:33:32 +1100 Subject: [PATCH] Add localStorage and sessionStorage options --- src/drivers/npm/README.md | 49 +++++++++------ src/drivers/npm/cli.js | 29 ++++++++- src/drivers/npm/driver.js | 126 +++++++++++++++++++++++++------------- 3 files changed, 141 insertions(+), 63 deletions(-) diff --git a/src/drivers/npm/README.md b/src/drivers/npm/README.md index d73ec227a..5f4181513 100644 --- a/src/drivers/npm/README.md +++ b/src/drivers/npm/README.md @@ -21,22 +21,27 @@ wappalyzer [options] #### Options ``` --b, --batch-size=... Process links in batches --d, --debug Output debug messages --t, --delay=ms Wait for ms milliseconds between requests --h, --help This text ---html-max-cols=... Limit the number of HTML characters per line processed ---html-max-rows=... Limit the number of HTML lines processed --D, --max-depth=... Don't analyse pages more than num levels deep --m, --max-urls=... Exit when num URLs have been analysed --w, --max-wait=... Wait no more than ms milliseconds for page resources to load --P, --pretty Pretty-print JSON output --p, --probe Perform a deeper scan by performing additional requests and inspecting DNS records ---proxy=... Proxy URL, e.g. 'http://user:pass@proxy:8080' --r, --recursive Follow links on pages (crawler) --a, --user-agent=... Set the user agent string --n, --no-scripts Disabled JavaScript on web pages --N, --no-redirect Disable cross-domain redirects +-b, --batch-size=... Process links in batches +-d, --debug Output debug messages +-t, --delay=ms Wait for ms milliseconds between requests +-h, --help This text +-H, --header Extra header to send with requests +--html-max-cols=... Limit the number of HTML characters per line processed +--html-max-rows=... Limit the number of HTML lines processed +-D, --max-depth=... Don't analyse pages more than num levels deep +-m, --max-urls=... Exit when num URLs have been analysed +-w, --max-wait=... Wait no more than ms milliseconds for page resources to load +-p, --probe=[basic|full] Perform a deeper scan by performing additional requests and inspecting DNS records +-P, --pretty Pretty-print JSON output +--proxy=... Proxy URL, e.g. 'http://user:pass@proxy:8080' +-r, --recursive Follow links on pages (crawler) +-a, --user-agent=... Set the user agent string +-n, --no-scripts Disabled JavaScript on web pages +-N, --no-redirect Disable cross-domain redirects +-e, --extended Output additional information +--local-storage=... JSON object to use as local storage +--session-storage=... JSON object to use as session storage + ``` @@ -51,9 +56,9 @@ $ npm i wappalyzer ### Usage ```javascript -const Wappalyzer = require('wappalyzer'); +const Wappalyzer = require('wappalyzer') -const url = 'https://www.wappalyzer.com'; +const url = 'https://www.wappalyzer.com' const options = { debug: false, @@ -81,7 +86,13 @@ const wappalyzer = new Wappalyzer(options) // Optionally set additional request headers const headers = {} - const site = await wappalyzer.open(url, headers) + // Optionally set local and/or session storage + const storage = { + local: {} + session: {} + } + + const site = await wappalyzer.open(url, headers, storage) // Optionally capture and output errors site.on('error', console.error) diff --git a/src/drivers/npm/cli.js b/src/drivers/npm/cli.js index 36dc896bb..357fbf975 100755 --- a/src/drivers/npm/cli.js +++ b/src/drivers/npm/cli.js @@ -90,6 +90,8 @@ Options: -n, --no-scripts Disabled JavaScript on web pages -N, --no-redirect Disable cross-domain redirects -e, --extended Output additional information + --local-storage=... JSON object to use as local storage + --session-storage=... JSON object to use as session storage `) process.exit(options.help ? 0 : 1) } @@ -119,13 +121,38 @@ if (options.header) { ) } +const storage = { + local: {}, + session: {}, +} + +for (const type of Object.keys(storage)) { + if (options[`${type}Storage`]) { + try { + storage[type] = JSON.parse(options[`${type}Storage`]) + + if ( + !options[`${type}Storage`] || + !Object.keys(options[`${type}Storage`]).length + ) { + throw new Error('Object has no properties') + } + } catch (error) { + // eslint-disable-next-line no-console + console.log(`${type}Storage error: ${error.message || error}`) + + process.exit(1) + } + } +} + ;(async function () { const wappalyzer = new Wappalyzer(options) try { await wappalyzer.init() - const site = await wappalyzer.open(url, headers) + const site = await wappalyzer.open(url, headers, storage) const results = await site.analyze() diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index bd84ef2a7..f51d288df 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -410,8 +410,42 @@ class Driver { } } - open(url, headers = {}) { - return new Site(url.split('#')[0], headers, this) + async open(url, headers = {}, storage = {}) { + const site = new Site(url.split('#')[0], headers, this) + + if (storage.local || storage.session) { + this.log('Setting storage...') + + const page = await site.newPage(site.originalUrl) + + await page.setRequestInterception(true) + + page.on('request', (request) => + request.respond({ + status: 200, + contentType: 'text/plain', + body: 'ok', + }) + ) + + await page.goto(url) + + await page.evaluate((storage) => { + ;['local', 'session'].forEach((type) => { + Object.keys(storage[type] || {}).forEach((key) => { + window[`${type}Storage`].setItem(key, storage[type][key]) + }) + }) + }, storage) + + try { + await page.close() + } catch { + // Continue + } + } + + return site } log(message, source = 'driver') { @@ -542,50 +576,10 @@ class Site { status: 0, } - if (!this.browser) { - await this.initDriver() - - if (!this.browser) { - throw new Error('Browser closed') - } - } - - let page - - try { - page = await this.browser.newPage() - - if (!page || page.isClosed()) { - throw new Error('Page did not open') - } - } catch (error) { - error.message += ` (${url})` - - this.error(error) - - await this.initDriver() - - page = await this.browser.newPage() - } - - this.pages.push(page) - - page.setJavaScriptEnabled(!this.options.noScripts) - - page.setDefaultTimeout(this.options.maxWait) + const page = await this.newPage(url) await page.setRequestInterception(true) - await page.setUserAgent(this.options.userAgent) - - page.on('dialog', (dialog) => dialog.dismiss()) - - page.on('error', (error) => { - error.message += ` (${url})` - - this.error(error) - }) - let responseReceived = false page.on('request', async (request) => { @@ -1044,6 +1038,52 @@ class Site { } } + async newPage(url) { + if (!this.browser) { + await this.initDriver() + + if (!this.browser) { + throw new Error('Browser closed') + } + } + + let page + + try { + page = await this.browser.newPage() + + if (!page || page.isClosed()) { + throw new Error('Page did not open') + } + } catch (error) { + error.message += ` (${url})` + + this.error(error) + + await this.initDriver() + + page = await this.browser.newPage() + } + + this.pages.push(page) + + page.setJavaScriptEnabled(!this.options.noScripts) + + page.setDefaultTimeout(this.options.maxWait) + + await page.setUserAgent(this.options.userAgent) + + page.on('dialog', (dialog) => dialog.dismiss()) + + page.on('error', (error) => { + error.message += ` (${url})` + + this.error(error) + }) + + return page + } + async analyze(url = this.originalUrl, index = 1, depth = 1) { if (this.options.recursive) { await sleep(this.options.delay * index)