Refactoring

src/drivers/webextension/js/driver.js

@@ -279,8 +279,12 @@ wappalyzer.driver.getRobotsTxt = (host, secure = false) => {
           fetch('http' + ( secure ? 's' : '' ) + '://' + host + '/robots.txt')
             .then(response => {
               if ( !response.ok ) {
+                if (response.status === 404) {
+                    return '';
+                } else {
                     throw 'GET ' + response.url + ' was not ok';
                 }
+              }
 
               return response.text();
             })

src/drivers/webextension/js/iframe.js

@@ -131,7 +131,7 @@ var exports = {};
 			}
 		},
 
-		ifTrackingEnabled: function(callback, elseCallback) {
+		askIfTrackingEnabled: function(callback, elseCallback) {
 
 			this.sendToBackground(
 				'is_tracking_enabled',
@@ -181,7 +181,7 @@ var exports = {};
 				video_assets: opt_video_assets,
 				assets: opt_assets,
 				version: '3',
-				mrev: 'a80971b-c',
+				mrev: '082d7cb-d',
 				msgNum: this.msgNum,
 				timestamp: new Date().getTime(),
 				pageVis: document.visibilityState,
@@ -890,7 +890,7 @@ var exports = {};
 	var _pageTags;
 	var INIT_MS_BW_SEARCHES = 2000;
 	var PAGE_TAG_RE = new RegExp('gpt|oascentral');
-	var POST_MSG_ID = '1498662251-16965-9321-27566-5783';
+	var POST_MSG_ID = '1501281986-4236-27733-5465-12184';
 	var AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)');
 
 	function getPageTags(doc) {
@@ -1136,12 +1136,12 @@ if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
 	window.adparser = {
 		init: exports.coordinator.init,
 		addPostMessageListener: exports.coordinator.addPostMessageListener,
-		ifTrackingEnabled: exports.utils.ifTrackingEnabled,
+		askIfTrackingEnabled: exports.utils.askIfTrackingEnabled,
 		sendToBackground: exports.utils.sendToBackground
 	};
 } else {
 	exports.coordinator.addPostMessageListener();
-	exports.utils.ifTrackingEnabled(
+	exports.utils.askIfTrackingEnabled(
 		function() {
 			exports.coordinator.init(function() {});
 		},
@@ -1156,7 +1156,7 @@ if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
 
 	if (  window === window.top  ) {
 		adparser.addPostMessageListener();
-		adparser.ifTrackingEnabled(
+		adparser.askIfTrackingEnabled(
 			function() {
 				adparser.init(onAdFound);
 			},

src/drivers/webextension/js/network.js

@@ -114,23 +114,35 @@
 		}
 	}
 
-	function ifTrackingEnabled(ifCallback, elseCallback) {
+	function ifTrackingEnabled(url, ifCallback, elseCallback) {
+
+		var fullIfCallback = function() {
+			allowedByRobotsTxt(url, ifCallback, elseCallback);
+		};
 
 		browser.storage.local.get('tracking').then(function(item) {
 
 			if ( item.hasOwnProperty('tracking') ) {
 				if ( item.tracking ) {
-					ifCallback();
+					fullIfCallback();
 				} else {
 					elseCallback();
 				}
 			} else {
-				ifCallback();
+				fullIfCallback();
 			}
 		});
 
 	}
 
+	function allowedByRobotsTxt(url, ifCallback, elseCallback) {
+		if (  ! url.startsWith('chrome://')  ) {
+			wappalyzer.robotsTxtAllows(url).then(ifCallback, elseCallback);
+		} else {
+			elseCallback();
+		}
+	}
+
 	function isPixelRequest(request) {
 		return (request.type === 'image' || request.responseStatus === 204) &&
 				request.size <= 1000;
@@ -207,6 +219,7 @@
 			this.cleanupCollector(tabId);
 
 			ifTrackingEnabled(
+				details.url,
 				function() {
 					if ( !areListenersRegistered ) {
 
@@ -792,12 +805,13 @@
 	browserProxy.runtime.onMessage.addListener(function(request, sender, sendResponse) {
 		if ( request === 'is_tracking_enabled' ) {
 			ifTrackingEnabled(
+				sender.tab.url,
 				function() {
-					sendResponse({'tracking_enabled': true});
+					try {sendResponse({'tracking_enabled': true});}
-				},
+					catch(err) {} },
 				function() {
-					sendResponse({'tracking_enabled': false});
+					try {sendResponse({'tracking_enabled': false});}
-				}
+					catch(err) {}}
 			);
 		}
 		return true;

src/wappalyzer.js

@@ -116,7 +116,7 @@ wappalyzer.robotsTxtAllows = url => {
     wappalyzer.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
       .then(robotsTxt => {
         robotsTxt.forEach(disallow => {
-          if ( parsed.pathname.search(disallow) === 0 ) {
+          if ( parsed.pathname.indexOf(disallow) === 0 ) {
             reject();
           }
         });