wappalyzer/src/drivers/npm/browsers/puppeteer.js

const {
  AWS_LAMBDA_FUNCTION_NAME,
  CHROME_BIN,
} = process.env;

let chromium;
let puppeteer;

if (AWS_LAMBDA_FUNCTION_NAME) {
  // eslint-disable-next-line global-require, import/no-unresolved
  chromium = require('chrome-aws-lambda');

  ({ puppeteer } = chromium);
} else {
  // eslint-disable-next-line global-require
  puppeteer = require('puppeteer');
}

const Browser = require('../browser');

function getJs() {
  const dereference = (obj, level = 0) => {
    try {
      // eslint-disable-next-line no-undef
      if (level > 5 || (level && obj === window)) {
        return '[Removed]';
      }

      if (Array.isArray(obj)) {
        obj = obj.map(item => dereference(item, level + 1));
      }

      if (typeof obj === 'function' || (typeof obj === 'object' && obj !== null)) {
        const newObj = {};

        Object.keys(obj).forEach((key) => {
          newObj[key] = dereference(obj[key], level + 1);
        });

        return newObj;
      }

      return obj;
    } catch (error) {
      return undefined;
    }
  };

  // eslint-disable-next-line no-undef
  return dereference(window);
}

class PuppeteerBrowser extends Browser {
  constructor(options) {
    options.maxWait = options.maxWait || 60;

    super(options);
  }

  async visit(url) {
    let done = false;
    let browser;

    try {
      await new Promise(async (resolve, reject) => {
        try {
          browser = await puppeteer.launch(chromium ? {
            args: [...chromium.args, '--ignore-certificate-errors'],
            defaultViewport: chromium.defaultViewport,
            executablePath: await chromium.executablePath,
            headless: chromium.headless,
          } : {
            args: ['--no-sandbox', '--headless', '--disable-gpu', '--ignore-certificate-errors'],
            executablePath: CHROME_BIN,
          });

          browser.on('disconnected', () => {
            if (!done) {
              reject(new Error('browser: disconnected'));
            }
          });

          const page = await browser.newPage();

          page.setDefaultTimeout(this.options.maxWait * 1.1);

          await page.setRequestInterception(true);

          page.on('error', error => reject(new Error(`page error: ${error.message || error}`)));

          let responseReceived = false;

          page.on('request', (request) => {
            try {
              if (
                responseReceived
                && request.isNavigationRequest()
                && request.frame() === page.mainFrame()
                && request.url() !== url
              ) {
                this.log(`abort navigation to ${request.url()}`);

                request.abort('aborted');
              } else if (!done) {
                if (!['document', 'script'].includes(request.resourceType())) {
                  request.abort();
                } else {
                  request.continue();
                }
              }
            } catch (error) {
              reject(new Error(`page error: ${error.message || error}`));
            }
          });

          page.on('response', (response) => {
            try {
              if (!this.statusCode) {
                this.statusCode = response.status();

                this.headers = {};

                const headers = response.headers();

                Object.keys(headers).forEach((key) => {
                  this.headers[key] = Array.isArray(headers[key]) ? headers[key] : [headers[key]];
                });

                this.contentType = headers['content-type'] || null;
              }

              if (response.status() < 300 || response.status() > 399) {
                responseReceived = true;
              }
            } catch (error) {
              reject(new Error(`page error: ${error.message || error}`));
            }
          });

          page.on('console', ({ _type, _text, _location }) => {
            if (!/Failed to load resource: net::ERR_FAILED/.test(_text)) {
              this.log(`${_text} (${_location.url}: ${_location.lineNumber})`, _type);
            }
          });

          if (this.options.userAgent) {
            await page.setUserAgent(this.options.userAgent);
          }

          try {
            await Promise.race([
              page.goto(url, { waitUntil: 'domcontentloaded' }),
              // eslint-disable-next-line no-shadow
              new Promise((resolve, reject) => setTimeout(() => reject(new Error('timeout')), this.options.maxWait)),
            ]);
          } catch (error) {
            throw new Error(error.message || error.toString());
          }

          // eslint-disable-next-line no-undef
          const links = await page.evaluateHandle(() => Array.from(document.getElementsByTagName('a')).map(({
            hash, hostname, href, pathname, protocol, rel,
          }) => ({
            hash,
            hostname,
            href,
            pathname,
            protocol,
            rel,
          })));

          this.links = await links.jsonValue();

          // eslint-disable-next-line no-undef
          const scripts = await page.evaluateHandle(() => Array.from(document.getElementsByTagName('script')).map(({
            src,
          }) => src));

          this.scripts = (await scripts.jsonValue()).filter(script => script);

          this.js = await page.evaluate(getJs);

          this.cookies = (await page.cookies()).map(({
            name, value, domain, path,
          }) => ({
            name, value, domain, path,
          }));

          this.html = await page.content();

          resolve();
        } catch (error) {
          reject(new Error(`visit error: ${error.message || error}`));
        }
      });
    } catch (error) {
      this.log(`visit error: ${error.message || error} (${url})`, 'error');

      throw new Error(error.message || error.toString());
    } finally {
      done = true;

      if (browser) {
        try {
          await browser.close();

          this.log('browser close ok');
        } catch (error) {
          this.log(`browser close error: ${error.message || error}`, 'error');
        }
      }
    }

    this.log(`visit ok (${url})`);
  }
}

module.exports = PuppeteerBrowser;