From ef0d8ad593ee4299afcc23260ee04b2c80300915 Mon Sep 17 00:00:00 2001
From: Elbert Alias <elbert@alias.io>
Date: Sat, 29 Jul 2017 12:30:58 +1000
Subject: [PATCH] Refactoring

---
 src/drivers/webextension/js/driver.js  |  6 +++++-
 src/drivers/webextension/js/iframe.js  | 12 +++++------
 src/drivers/webextension/js/network.js | 28 +++++++++++++++++++-------
 src/wappalyzer.js                      |  2 +-
 4 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js
index a90e2805d..30ea87adf 100644
--- a/src/drivers/webextension/js/driver.js
+++ b/src/drivers/webextension/js/driver.js
@@ -279,7 +279,11 @@ wappalyzer.driver.getRobotsTxt = (host, secure = false) => {
           fetch('http' + ( secure ? 's' : '' ) + '://' + host + '/robots.txt')
             .then(response => {
               if ( !response.ok ) {
-                throw 'GET ' + response.url + ' was not ok';
+                if (response.status === 404) {
+                    return '';
+                } else {
+                    throw 'GET ' + response.url + ' was not ok';
+                }
               }
 
               return response.text();
diff --git a/src/drivers/webextension/js/iframe.js b/src/drivers/webextension/js/iframe.js
index 25a5c5f7b..156dae30b 100644
--- a/src/drivers/webextension/js/iframe.js
+++ b/src/drivers/webextension/js/iframe.js
@@ -131,7 +131,7 @@ var exports = {};
 			}
 		},
 
-		ifTrackingEnabled: function(callback, elseCallback) {
+		askIfTrackingEnabled: function(callback, elseCallback) {
 
 			this.sendToBackground(
 				'is_tracking_enabled',
@@ -181,7 +181,7 @@ var exports = {};
 				video_assets: opt_video_assets,
 				assets: opt_assets,
 				version: '3',
-				mrev: 'a80971b-c',
+				mrev: '082d7cb-d',
 				msgNum: this.msgNum,
 				timestamp: new Date().getTime(),
 				pageVis: document.visibilityState,
@@ -890,7 +890,7 @@ var exports = {};
 	var _pageTags;
 	var INIT_MS_BW_SEARCHES = 2000;
 	var PAGE_TAG_RE = new RegExp('gpt|oascentral');
-	var POST_MSG_ID = '1498662251-16965-9321-27566-5783';
+	var POST_MSG_ID = '1501281986-4236-27733-5465-12184';
 	var AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)');
 
 	function getPageTags(doc) {
@@ -1136,12 +1136,12 @@ if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
 	window.adparser = {
 		init: exports.coordinator.init,
 		addPostMessageListener: exports.coordinator.addPostMessageListener,
-		ifTrackingEnabled: exports.utils.ifTrackingEnabled,
+		askIfTrackingEnabled: exports.utils.askIfTrackingEnabled,
 		sendToBackground: exports.utils.sendToBackground
 	};
 } else {
 	exports.coordinator.addPostMessageListener();
-	exports.utils.ifTrackingEnabled(
+	exports.utils.askIfTrackingEnabled(
 		function() {
 			exports.coordinator.init(function() {});
 		},
@@ -1156,7 +1156,7 @@ if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
 
 	if (  window === window.top  ) {
 		adparser.addPostMessageListener();
-		adparser.ifTrackingEnabled(
+		adparser.askIfTrackingEnabled(
 			function() {
 				adparser.init(onAdFound);
 			},
diff --git a/src/drivers/webextension/js/network.js b/src/drivers/webextension/js/network.js
index 24c182b40..8a0421b88 100644
--- a/src/drivers/webextension/js/network.js
+++ b/src/drivers/webextension/js/network.js
@@ -114,23 +114,35 @@
 		}
 	}
 
-	function ifTrackingEnabled(ifCallback, elseCallback) {
+	function ifTrackingEnabled(url, ifCallback, elseCallback) {
+
+		var fullIfCallback = function() {
+			allowedByRobotsTxt(url, ifCallback, elseCallback);
+		};
 
 		browser.storage.local.get('tracking').then(function(item) {
 
 			if ( item.hasOwnProperty('tracking') ) {
 				if ( item.tracking ) {
-					ifCallback();
+					fullIfCallback();
 				} else {
 					elseCallback();
 				}
 			} else {
-				ifCallback();
+				fullIfCallback();
 			}
 		});
 
 	}
 
+	function allowedByRobotsTxt(url, ifCallback, elseCallback) {
+		if (  ! url.startsWith('chrome://')  ) {
+			wappalyzer.robotsTxtAllows(url).then(ifCallback, elseCallback);
+		} else {
+			elseCallback();
+		}
+	}
+
 	function isPixelRequest(request) {
 		return (request.type === 'image' || request.responseStatus === 204) &&
 				request.size <= 1000;
@@ -207,6 +219,7 @@
 			this.cleanupCollector(tabId);
 
 			ifTrackingEnabled(
+				details.url,
 				function() {
 					if ( !areListenersRegistered ) {
 
@@ -792,12 +805,13 @@
 	browserProxy.runtime.onMessage.addListener(function(request, sender, sendResponse) {
 		if ( request === 'is_tracking_enabled' ) {
 			ifTrackingEnabled(
+				sender.tab.url,
 				function() {
-					sendResponse({'tracking_enabled': true});
-				},
+					try {sendResponse({'tracking_enabled': true});}
+					catch(err) {} },
 				function() {
-					sendResponse({'tracking_enabled': false});
-				}
+					try {sendResponse({'tracking_enabled': false});}
+					catch(err) {}}
 			);
 		}
 		return true;
diff --git a/src/wappalyzer.js b/src/wappalyzer.js
index 0ae01cea9..42cea86f2 100644
--- a/src/wappalyzer.js
+++ b/src/wappalyzer.js
@@ -116,7 +116,7 @@ wappalyzer.robotsTxtAllows = url => {
     wappalyzer.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
       .then(robotsTxt => {
         robotsTxt.forEach(disallow => {
-          if ( parsed.pathname.search(disallow) === 0 ) {
+          if ( parsed.pathname.indexOf(disallow) === 0 ) {
             reject();
           }
         });