Disable robots.txt fetching when tracking is disabled

src/drivers/webextension/js/driver.js

@@ -266,38 +266,43 @@ wappalyzer.driver.displayApps = (detected, context) => {
  */
 wappalyzer.driver.getRobotsTxt = (host, secure = false) => {
   return new Promise((resolve, reject) => {
-    getOption('robotsTxtCache')
+    getOption('tracking', true)
-      .then(robotsTxtCache => {
+      .then(tracking => {
-        robotsTxtCache = robotsTxtCache || {};
+        if ( !tracking ) {
-
+          return resolve([]);
-        if ( host in robotsTxtCache ) {
+        }
-          resolve(robotsTxtCache[host]);
-        } else {
-          var url = 'http' + ( secure ? 's' : '' ) + '://' + host + '/robots.txt';
-
-          fetch('http' + ( secure ? 's' : '' ) + '://' + host + '/robots.txt')
-            .then(response => {
-              if ( !response.ok ) {
-                if ( response.status === 404 ) {
-                  return '';
-                } else {
-                  throw 'GET ' + response.url + ' was not ok';
-                }
-              }
-
-              return response.text();
-            })
-            .then(robotsTxt => {
-              robotsTxtCache[host] = wappalyzer.parseRobotsTxt(robotsTxt);
 
-              setOption('robotsTxtCache', robotsTxtCache);
+        getOption('robotsTxtCache')
+          .then(robotsTxtCache => {
+            robotsTxtCache = robotsTxtCache || {};
 
+            if ( host in robotsTxtCache ) {
               resolve(robotsTxtCache[host]);
-
+            } else {
-              var hostname = host.replace(/:[0-9]+$/, '')
+              const url = 'http' + ( secure ? 's' : '' ) + '://' + host + '/robots.txt';
-            })
+
-            .catch(reject);
+              fetch('http' + ( secure ? 's' : '' ) + '://' + host + '/robots.txt')
-        }
+                .then(response => {
+                  if ( !response.ok ) {
+                    if ( response.status === 404 ) {
+                      return '';
+                    } else {
+                      throw 'GET ' + response.url + ' was not ok';
+                    }
+                  }
+
+                  return response.text();
+                })
+                .then(robotsTxt => {
+                  robotsTxtCache[host] = wappalyzer.parseRobotsTxt(robotsTxt);
+
+                  setOption('robotsTxtCache', robotsTxtCache);
+
+                  resolve(robotsTxtCache[host]);
+                })
+                .catch(reject);
+            }
+          });
       });
   });
 };