Add localStorage and sessionStorage options

src/drivers/npm/README.md

@@ -21,22 +21,27 @@ wappalyzer <url> [options]
 #### Options
 
 ```
--b, --batch-size=...     Process links in batches
--d, --debug              Output debug messages
--t, --delay=ms           Wait for ms milliseconds between requests
--h, --help               This text
---html-max-cols=...      Limit the number of HTML characters per line processed
---html-max-rows=...      Limit the number of HTML lines processed
--D, --max-depth=...      Don't analyse pages more than num levels deep
--m, --max-urls=...       Exit when num URLs have been analysed
--w, --max-wait=...       Wait no more than ms milliseconds for page resources to load
--P, --pretty             Pretty-print JSON output
--p, --probe              Perform a deeper scan by performing additional requests and inspecting DNS records
---proxy=...              Proxy URL, e.g. 'http://user:pass@proxy:8080'
--r, --recursive          Follow links on pages (crawler)
--a, --user-agent=...     Set the user agent string
--n, --no-scripts         Disabled JavaScript on web pages
--N, --no-redirect        Disable cross-domain redirects
+-b, --batch-size=...       Process links in batches
+-d, --debug                Output debug messages
+-t, --delay=ms             Wait for ms milliseconds between requests
+-h, --help                 This text
+-H, --header               Extra header to send with requests
+--html-max-cols=...        Limit the number of HTML characters per line processed
+--html-max-rows=...        Limit the number of HTML lines processed
+-D, --max-depth=...        Don't analyse pages more than num levels deep
+-m, --max-urls=...         Exit when num URLs have been analysed
+-w, --max-wait=...         Wait no more than ms milliseconds for page resources to load
+-p, --probe=[basic|full]   Perform a deeper scan by performing additional requests and inspecting DNS records
+-P, --pretty               Pretty-print JSON output
+--proxy=...                Proxy URL, e.g. 'http://user:pass@proxy:8080'
+-r, --recursive            Follow links on pages (crawler)
+-a, --user-agent=...       Set the user agent string
+-n, --no-scripts           Disabled JavaScript on web pages
+-N, --no-redirect          Disable cross-domain redirects
+-e, --extended             Output additional information
+--local-storage=...        JSON object to use as local storage
+--session-storage=...      JSON object to use as session storage
+
 ```
 
 
@@ -51,9 +56,9 @@ $ npm i wappalyzer
 ### Usage
 
 ```javascript
-const Wappalyzer = require('wappalyzer');
+const Wappalyzer = require('wappalyzer')
 
-const url = 'https://www.wappalyzer.com';
+const url = 'https://www.wappalyzer.com'
 
 const options = {
   debug: false,
@@ -81,7 +86,13 @@ const wappalyzer = new Wappalyzer(options)
     // Optionally set additional request headers
     const headers = {}
 
-    const site = await wappalyzer.open(url, headers)
+    // Optionally set local and/or session storage
+    const storage = {
+      local: {}
+      session: {}
+    }
+
+    const site = await wappalyzer.open(url, headers, storage)
 
     // Optionally capture and output errors
     site.on('error', console.error)

src/drivers/npm/cli.js

@@ -90,6 +90,8 @@ Options:
   -n, --no-scripts           Disabled JavaScript on web pages
   -N, --no-redirect          Disable cross-domain redirects
   -e, --extended             Output additional information
+  --local-storage=...        JSON object to use as local storage
+  --session-storage=...      JSON object to use as session storage
 `)
   process.exit(options.help ? 0 : 1)
 }
@@ -119,13 +121,38 @@ if (options.header) {
   )
 }
 
+const storage = {
+  local: {},
+  session: {},
+}
+
+for (const type of Object.keys(storage)) {
+  if (options[`${type}Storage`]) {
+    try {
+      storage[type] = JSON.parse(options[`${type}Storage`])
+
+      if (
+        !options[`${type}Storage`] ||
+        !Object.keys(options[`${type}Storage`]).length
+      ) {
+        throw new Error('Object has no properties')
+      }
+    } catch (error) {
+      // eslint-disable-next-line no-console
+      console.log(`${type}Storage error: ${error.message || error}`)
+
+      process.exit(1)
+    }
+  }
+}
+
 ;(async function () {
   const wappalyzer = new Wappalyzer(options)
 
   try {
     await wappalyzer.init()
 
-    const site = await wappalyzer.open(url, headers)
+    const site = await wappalyzer.open(url, headers, storage)
 
     const results = await site.analyze()
 

src/drivers/npm/driver.js

@@ -410,8 +410,42 @@ class Driver {
     }
   }
 
-  open(url, headers = {}) {
-    return new Site(url.split('#')[0], headers, this)
+  async open(url, headers = {}, storage = {}) {
+    const site = new Site(url.split('#')[0], headers, this)
+
+    if (storage.local || storage.session) {
+      this.log('Setting storage...')
+
+      const page = await site.newPage(site.originalUrl)
+
+      await page.setRequestInterception(true)
+
+      page.on('request', (request) =>
+        request.respond({
+          status: 200,
+          contentType: 'text/plain',
+          body: 'ok',
+        })
+      )
+
+      await page.goto(url)
+
+      await page.evaluate((storage) => {
+        ;['local', 'session'].forEach((type) => {
+          Object.keys(storage[type] || {}).forEach((key) => {
+            window[`${type}Storage`].setItem(key, storage[type][key])
+          })
+        })
+      }, storage)
+
+      try {
+        await page.close()
+      } catch {
+        // Continue
+      }
+    }
+
+    return site
   }
 
   log(message, source = 'driver') {
@@ -542,50 +576,10 @@ class Site {
       status: 0,
     }
 
-    if (!this.browser) {
-      await this.initDriver()
-
-      if (!this.browser) {
-        throw new Error('Browser closed')
-      }
-    }
-
-    let page
-
-    try {
-      page = await this.browser.newPage()
-
-      if (!page || page.isClosed()) {
-        throw new Error('Page did not open')
-      }
-    } catch (error) {
-      error.message += ` (${url})`
-
-      this.error(error)
-
-      await this.initDriver()
-
-      page = await this.browser.newPage()
-    }
-
-    this.pages.push(page)
-
-    page.setJavaScriptEnabled(!this.options.noScripts)
-
-    page.setDefaultTimeout(this.options.maxWait)
+    const page = await this.newPage(url)
 
     await page.setRequestInterception(true)
 
-    await page.setUserAgent(this.options.userAgent)
-
-    page.on('dialog', (dialog) => dialog.dismiss())
-
-    page.on('error', (error) => {
-      error.message += ` (${url})`
-
-      this.error(error)
-    })
-
     let responseReceived = false
 
     page.on('request', async (request) => {
@@ -1044,6 +1038,52 @@ class Site {
     }
   }
 
+  async newPage(url) {
+    if (!this.browser) {
+      await this.initDriver()
+
+      if (!this.browser) {
+        throw new Error('Browser closed')
+      }
+    }
+
+    let page
+
+    try {
+      page = await this.browser.newPage()
+
+      if (!page || page.isClosed()) {
+        throw new Error('Page did not open')
+      }
+    } catch (error) {
+      error.message += ` (${url})`
+
+      this.error(error)
+
+      await this.initDriver()
+
+      page = await this.browser.newPage()
+    }
+
+    this.pages.push(page)
+
+    page.setJavaScriptEnabled(!this.options.noScripts)
+
+    page.setDefaultTimeout(this.options.maxWait)
+
+    await page.setUserAgent(this.options.userAgent)
+
+    page.on('dialog', (dialog) => dialog.dismiss())
+
+    page.on('error', (error) => {
+      error.message += ` (${url})`
+
+      this.error(error)
+    })
+
+    return page
+  }
+
   async analyze(url = this.originalUrl, index = 1, depth = 1) {
     if (this.options.recursive) {
       await sleep(this.options.delay * index)