Add catch statement

src/drivers/npm/driver.js

@@ -261,6 +261,7 @@ class Driver {
 
     return new Promise(resolve => {
       this.fetch(pageUrl, index, depth)
+        .catch(() => {})
         .then(links => {
           if ( links && Boolean(this.options.recursive) && depth < this.options.maxDepth ) {
             return Promise.all(links.map((link, index) => this.crawl(link, index + 1, depth + 1)));

src/wappalyzer.js

@@ -122,17 +122,19 @@ class Wappalyzer {
    */
   robotsTxtAllows(url) {
     return new Promise((resolve, reject) => {
-      var parsed = this.parseUrl(url.canonical);
+      var parsed = this.parseUrl(url);
 
       if ( parsed.protocol !== 'http:' && parsed.protocol !== 'https:' ) {
-        reject();
+        return reject();
       }
 
       this.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
         .then(robotsTxt => {
-          robotsTxt.forEach(disallow => parsed.pathname.indexOf(disallow) === 0 && reject());
+          if (robotsTxt.some(disallowedPath => parsed.pathname.indexOf(disallowedPath) === 0)) {
-
+            return reject();
-          resolve();
+          } else {
+            return resolve();
+          }
         });
     });
   };