From 30ac5b413b9294a21a5f8e285e9a12fc9e204730 Mon Sep 17 00:00:00 2001
From: Elbert Alias <elbert@alias.io>
Date: Fri, 5 Jan 2018 14:43:15 +1100
Subject: [PATCH] Add catch statement

---
 src/drivers/npm/driver.js |  1 +
 src/wappalyzer.js         | 12 +++++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js
index b3ffee9a5..6b6df682b 100644
--- a/src/drivers/npm/driver.js
+++ b/src/drivers/npm/driver.js
@@ -261,6 +261,7 @@ class Driver {
 
     return new Promise(resolve => {
       this.fetch(pageUrl, index, depth)
+        .catch(() => {})
         .then(links => {
           if ( links && Boolean(this.options.recursive) && depth < this.options.maxDepth ) {
             return Promise.all(links.map((link, index) => this.crawl(link, index + 1, depth + 1)));
diff --git a/src/wappalyzer.js b/src/wappalyzer.js
index 3b4d8f798..95c933eab 100644
--- a/src/wappalyzer.js
+++ b/src/wappalyzer.js
@@ -122,17 +122,19 @@ class Wappalyzer {
    */
   robotsTxtAllows(url) {
     return new Promise((resolve, reject) => {
-      var parsed = this.parseUrl(url.canonical);
+      var parsed = this.parseUrl(url);
 
       if ( parsed.protocol !== 'http:' && parsed.protocol !== 'https:' ) {
-        reject();
+        return reject();
       }
 
       this.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
         .then(robotsTxt => {
-          robotsTxt.forEach(disallow => parsed.pathname.indexOf(disallow) === 0 && reject());
-
-          resolve();
+          if (robotsTxt.some(disallowedPath => parsed.pathname.indexOf(disallowedPath) === 0)) {
+            return reject();
+          } else {
+            return resolve();
+          }
         });
     });
   };