Fix Puppeteer JavaScript processing, add --pretty option to NPM driver

src/drivers/npm/README.md

@@ -28,18 +28,20 @@ wappalyzer [url] [options]
 ### Options
 
 ```
---password           Password to be used for basic HTTP authentication
---proxy              Proxy URL, e.g. 'http://user:pass@proxy:8080'
---username           Username to be used for basic HTTP authentication
+--browser=str        Specify which headless browser to use (zombie or puppeteer)
+--password=str       Password to be used for basic HTTP authentication
+--proxy=str          Proxy URL, e.g. 'http://user:pass@proxy:8080'
+--username=str       Username to be used for basic HTTP authentication
 --chunk-size=num     Process links in chunks.
---debug=0|1          Output debug messages.
+--debug              Output debug messages.
 --delay=ms           Wait for ms milliseconds between requests.
 --html-max-cols=num  Limit the number of HTML characters per line processed.
 --html-max-rows=num  Limit the number of HTML lines processed.
 --max-depth=num      Don't analyse pages more than num levels deep.
 --max-urls=num       Exit when num URLs have been analysed.
 --max-wait=ms        Wait no more than ms milliseconds for page resources to load.
---recursive=0|1      Follow links on pages (crawler).
+--pretty             Pretty-print JSON output
+--recursive          Follow links on pages (crawler).
 --user-agent=str     Set the user agent string.
 ```
 
@@ -52,6 +54,7 @@ const Wappalyzer = require('wappalyzer');
 const url = 'https://www.wappalyzer.com';
 
 const options = {
+  // browser: 'puppeteer',
   debug: false,
   delay: 500,
   maxDepth: 3,
@@ -65,9 +68,6 @@ const options = {
 
 const wappalyzer = new Wappalyzer(url, options);
 
-// Optional: set the browser to use
-// wappalyzer.Browser = Wappalyzer.browsers.zombie;
-
 // Optional: capture log output
 // wappalyzer.on('log', params => {
 //   const { message, source, type } = params;
@@ -79,14 +79,14 @@ const wappalyzer = new Wappalyzer(url, options);
 // });
 
 wappalyzer.analyze()
-  .then(json => {
+  .then((json) => {
     process.stdout.write(`${JSON.stringify(json, null, 2)}\n`);
 
     process.exit(0);
   })
-  .catch(error => {
+  .catch((error) => {
     process.stderr.write(`${error}\n`);
 
     process.exit(1);
-});
+  });
 ```

src/drivers/npm/browsers/puppeteer.js

@@ -4,15 +4,16 @@ const Browser = require('../browser');
 function getJs() {
   const dereference = (obj, level = 0) => {
     try {
-      if (level > 5 || typeof obj === 'function') {
-        return undefined;
+      // eslint-disable-next-line no-undef
+      if (level > 5 || (level && obj === window)) {
+        return '[Removed]';
       }
 
       if (Array.isArray(obj)) {
         obj = obj.map(item => dereference(item, level + 1));
       }
 
-      if (typeof obj === 'object' && obj !== null) {
+      if (typeof obj === 'function' || (typeof obj === 'object' && obj !== null)) {
         const newObj = {};
 
         Object.keys(obj).forEach((key) => {
@@ -39,7 +40,7 @@ class PuppeteerBrowser extends Browser {
     super(options);
 
     this.browser = () => puppeteer.launch({
-      args: ['--no-sandbox', '--headless', '--disable-gpu'],
+      args: ['--no-sandbox', '--headless', '--disable-gpu', '--ignore-certificate-errors'],
     });
   }
 
@@ -72,6 +73,8 @@ class PuppeteerBrowser extends Browser {
       }
     });
 
+    page.on('console', ({ _type, _text, _location }) => this.log(`${_text} (${_location.url}: ${_location.lineNumber})`, _type));
+
     await page.setUserAgent(this.options.userAgent);
 
     try {

src/drivers/npm/cli.js

@@ -1,45 +1,47 @@
 #!/usr/bin/env node
 
 const Wappalyzer = require('./driver');
-let Browser = require('./browsers/zombie');
 
 const args = process.argv.slice(2);
 
-const url = args.shift() || '';
-
-if (!url) {
-  process.stderr.write('No URL specified\n');
-
-  process.exit(1);
-}
-
 const options = {};
 
+let url;
 let arg;
 
-do {
+while (true) { // eslint-disable-line no-constant-condition
   arg = args.shift();
 
-  const matches = /--([^=]+)=(.+)/.exec(arg);
+  if (!arg) {
+    break;
+  }
+
+  const matches = /--([^=]+)(?:=(.+))?/.exec(arg);
 
   if (matches) {
     const key = matches[1].replace(/-\w/g, _matches => _matches[1].toUpperCase());
-    const value = matches[2];
+    const value = matches[2] || true;
 
     options[key] = value;
+  } else {
+    url = arg;
   }
-} while (arg);
+}
 
-if (options.browser && options.browser === 'puppeteer') {
-  // eslint-disable-next-line global-require
-  Browser = require('./browsers/puppeteer');
+if (!url) {
+  process.stderr.write('No URL specified\n');
+
+  process.exit(1);
 }
 
+// eslint-disable-next-line import/no-dynamic-require
+const Browser = require(`./browsers/${options.browser || 'zombie'}`);
+
 const wappalyzer = new Wappalyzer(Browser, url, options);
 
 wappalyzer.analyze()
   .then((json) => {
-    process.stdout.write(`${JSON.stringify(json)}\n`);
+    process.stdout.write(`${JSON.stringify(json, null, options.pretty ? 2 : null)}\n`);
 
     process.exit(0);
   })

src/drivers/npm/driver.js

@@ -138,6 +138,7 @@ class Driver {
 
   log(message, source, type) {
     if (this.options.debug) {
+      // eslint-disable-next-line no-console
       console.log(`[wappalyzer ${type}]`, `[${source}]`, message);
     }
 

src/drivers/npm/index.js

@@ -1,16 +1,12 @@
 const Driver = require('./driver');
-const ZombieBrowser = require('./browsers/zombie');
 
 class Wappalyzer {
   constructor(pageUrl, options) {
-    this.browser = ZombieBrowser;
+    // eslint-disable-next-line import/no-dynamic-require, global-require
+    const Browser = require(`./browsers/${options.browser || 'zombie'}`);
 
-    return new Driver(this.browser, pageUrl, options);
+    return new Driver(Browser, pageUrl, options);
   }
 }
 
-Wappalyzer.browsers = {
-  zombie: ZombieBrowser,
-};
-
 module.exports = Wappalyzer;