Chrome driver now detects headers automatically

README.md

@@ -18,7 +18,5 @@ Refer to the [wiki](https://github.com/ElbertF/Wappalyzer/wiki) for
 *Licensed under the [GPL](https://github.com/ElbertF/Wappalyzer/blob/master/LICENSE).*
 
 Donate Bitcoin: 16gb4uGDAjaeRJwKVmKr2EXa8x2fmvT8EQ - *Thanks!*
-![QR Code](https://wappalyzer.com/sites/default/themes/wappalyzer/images/bitcoinqrcode.png)
-
-
 
+![QR Code](https://wappalyzer.com/sites/default/themes/wappalyzer/images/bitcoinqrcode.png)

drivers/chrome/_locales/en/messages.json

@@ -5,11 +5,8 @@
 	"options":                  { "message": "Wappalyzer Options" },
 	"optionsSave":              { "message": "Save options" },
 	"optionsSaved":             { "message": "Saved" },
-	"optionAutoAnalyzeHeaders": { "message": "Analyze headers automatically on click" },
 	"optionUpgradeMessage":     { "message": "Tell me about upgrades" },
 	"optionTracking":           { "message": "Anonymously send reports on detected applications to wappalyzer.com for research" },
-	"analyzeHeaders":           { "message": "Analyze headers" },
-	"analyzeHeadersDone":       { "message": "Completed" },
 	"nothingToDo":              { "message": "Nothing to do here." },
 	"noAppsDetected":           { "message": "No applications detected." },
 	"categoryName1":            { "message": "CMS" },
@@ -17,10 +14,6 @@
 	"categoryName3":            { "message": "Database Manager" },
 	"categoryName4":            { "message": "Documentation Tool" },
 	"categoryName5":            { "message": "Widget" },
-	"categoryName6":            { "message": "eCommerce" },
-	"categoryName7":            { "message": "Photo Gallery" },
-	"categoryName8":            { "message": "Wiki" },
-	"categoryName9":            { "message": "Hosting Panel" },
 	"categoryName10":           { "message": "Analytics" },
 	"categoryName11":           { "message": "Blog" },
 	"categoryName12":           { "message": "Javascript Framework" },

drivers/chrome/_locales/es/messages.json

@@ -5,11 +5,8 @@
 	"options":                  { "message": "Opciones de Wappalyzer" },
 	"optionsSave":              { "message": "Guardar opciones" },
 	"optionsSaved":             { "message": "Guardado" },
-	"optionAutoAnalyzeHeaders": { "message": "Analizar cabeceras automáticamente al hacer clic" },
 	"optionUpgradeMessage":     { "message": "Indicarme actualizaciones" },
 	"optionTracking":           { "message": "Enviar informes anónimos sobre las aplicaciones detectadas a wappalyzer.com para análisis" },
-	"analyzeHeaders":           { "message": "Analizar cabeceras" },
-	"analyzeHeadersDone":       { "message": "Completado" },
 	"nothingToDo":              { "message": "Nada que hacer aquí." },
 	"noAppsDetected":           { "message": "Aplicaciones no detectadas." },
 	"categoryName1":            { "message": "Gestor de Contenido" },

drivers/chrome/_locales/fr/messages.json

@@ -1,10 +1,4 @@
 {
-   "analyzeHeaders": {
-      "message": "Analyser les en-têtes"
-   },
-   "analyzeHeadersDone": {
-      "message": "Fait"
-   },
    "categoryName1": {
       "message": "CMS"
    },
@@ -122,9 +116,6 @@
    "nothingToDo": {
       "message": "Rien à faire ici."
    },
-   "optionAutoAnalyzeHeaders": {
-      "message": "Analyser les en-têtes automatiquement"
-   },
    "optionTracking": {
       "message": "Envoyer anonymement des rapports sur les applications détectées à wappalyzer.com pour la recherche"
    },

drivers/chrome/css/popup.css

@@ -7,6 +7,10 @@ body {
 	min-width: 200px;
 }
 
+a:focus {
+	outline: 0;
+}
+
 img {
 	display: inline-block;
 	height: 16px;
@@ -58,35 +62,22 @@ img {
 	text-align: center;
 }
 
-#buttons {
+#footer {
+	border-top: 1px solid #ccc;
 	margin-top: 12px;
 	overflow: hidden;
+	padding-top: 6px;
 }
 
-	#buttons button {
-		height: 32px;
-	}
-
-#analyze-headers {
-	float: left;
-	width: 158px;
+#footer a {
+	color: #999;
+	text-decoration: none;
 }
 
-	#analyze-headers.pending {
-		background-image: url('../images/pending2.gif');
-		background-position: center center;
-		background-repeat: no-repeat;
-		text-indent: -999px;
+#footer a:hover {
+	color: #333;
 }
 
 #options {
-	background-image: url('../images/options.png');
-	background-position: center center;
-	background-repeat: no-repeat;
 	float: right;
-	height: 32px;
-  -webkit-padding-end: 0;
-  -webkit-padding-start: 0;
-	min-width: 32px;
-	width: 32px;
 }

drivers/chrome/js/driver.js

@@ -8,7 +8,9 @@
 	var w = wappalyzer,
 		firstRun = false,
 		upgraded = false,
-		tab, tabCache = {};
+		tab,
+		tabCache = {},
+		headersCache = {};
 
 	w.driver = {
 		/**
@@ -61,6 +63,10 @@
 			} catch(e) { }
 
 			chrome.extension.onRequest.addListener(function(request, sender, sendResponse) {
+				var
+					hostname,
+					a = document.createElement('a');
+
 				if ( typeof request.id != 'undefined' ) {
 					w.log('request: ' + request.id);
 
@@ -72,18 +78,16 @@
 						case 'analyze':
 							tab = sender.tab;
 
-							var hostname, a = document.createElement('a');
-
 							a.href = tab.url;
 
 							hostname = a.hostname;
 
-							w.analyze(hostname, tab.url, request.subject);
-
-							for ( subject in request.subject ) {
-								tabCache[tab.id].analyzed.push(subject);
+							if ( headersCache[tab.url] !== undefined ) {
+								request.subject.headers = headersCache[tab.url];
 							}
 
+							w.analyze(hostname, tab.url, request.subject);
+
 							break;
 						case 'fetch_headers':
 							chrome.tabs.executeScript(request.tab.id, { file: 'js/headers.js' });
@@ -115,6 +119,35 @@
 				tabCache[tabId] = null;
 			});
 
+			// Live intercept headers using webRequest API
+			chrome.webRequest.onCompleted.addListener(function(details) {
+				var responseHeaders = {};
+
+				if ( details.responseHeaders ) {
+					var uri = details.url.replace(/#.*$/, ''); // Remove hash
+
+					details.responseHeaders.forEach(function(header) {
+						responseHeaders[header.name.toLowerCase()] = header.value || '' + header.binaryValue;
+					});
+
+					if ( headersCache.length > 50 ) {
+						headersCache = {};
+					}
+
+					if ( /text\/html/.test(responseHeaders['content-type']) ) {
+						if ( headersCache[details.url] === undefined ) {
+							headersCache[details.url] = {};
+						}
+
+						for ( var header in responseHeaders ) {
+							headersCache[uri][header] = responseHeaders[header];
+						}
+					}
+
+					w.log(JSON.stringify({ uri: uri, headers: responseHeaders }));
+				}
+			}, { urls: [ 'http://*/*', 'https://*/*' ], types: [ 'main_frame' ] }, [ 'responseHeaders' ]);
+
 			if ( firstRun ) {
 				w.driver.goToURL({ url: w.config.websiteURL + 'installed', medium: 'install' });
 
@@ -122,16 +155,16 @@
 			}
 
 			if ( upgraded ) {
-				w.driver.goToURL({ url: w.config.websiteURL + 'upgraded', medium: 'upgrade' });
+				w.driver.goToURL({ url: w.config.websiteURL + 'upgraded', medium: 'upgrade', background: true });
 
 				upgraded = false;
 			}
 		},
 
 		goToURL: function(args) {
-			var url = args.url + ( typeof args.medium === 'undefined' ? '' :  '?utm_source=chrome&utm_medium=' + args.medium + '&utm_campaign=extensions');
+			var url = args.url + ( typeof args.medium === 'undefined' ? '' :  '?pk_campaign=chrome&pk_kwd=' + args.medium);
 
-			window.open(url);
+			chrome.tabs.create({ url: url, active: args.background === undefined || !args.background });
 		},
 
 		/**
@@ -143,8 +176,7 @@
 			if ( tabCache[tab.id] == null ) {
 				tabCache[tab.id] = {
 					count: 0,
-					appsDetected: [],
-					analyzed: []
+					appsDetected: []
 					};
 			}
 
@@ -159,6 +191,7 @@
 					for ( appName in w.detected[tab.url] ) {
 						w.apps[appName].cats.forEach(function(cat) {
 							if ( cat == match && !found ) {
+								//chrome.browserAction.setIcon({ tabId: tab.id, '19': 'images/icons/' + appName + '.png' });
 								chrome.browserAction.setIcon({ tabId: tab.id, path: 'images/icons/' + appName + '.png' });
 
 								found = true;

drivers/chrome/js/headers.js

@@ -1,46 +0,0 @@
-(function() {
-	var c = {
-		init: function() {
-			c.log('init');
-
-			c.getResponseHeaders();
-		},
-
-		log: function(message) {
-			chrome.extension.sendRequest({ id: 'log', message: '[ content.js ] ' + message });
-		},
-
-		getResponseHeaders: function() {
-			var xhr = new XMLHttpRequest();
-
-			xhr.open('GET', window.location, true);
-
-			xhr.onreadystatechange = function() {
-				if ( xhr.readyState === 4 && xhr.status ) {
-					var headers = xhr.getAllResponseHeaders().split("\n");
-
-					if ( headers.length > 0 && headers[0] != '' ) {
-						c.log('responseHeaders: ' + xhr.getAllResponseHeaders());
-
-						var responseHeaders = {};
-
-						headers.forEach(function(line) {
-							if ( line ) {
-								name  = line.substring(0, line.indexOf(': ')).toLowerCase();
-								value = line.substring(line.indexOf(': ') + 2, line.length - 1);
-
-								responseHeaders[name] = value;
-							}
-						});
-
-						chrome.extension.sendRequest({ id: 'analyze', subject: { headers: responseHeaders } });
-					}
-				}
-			}
-
-			xhr.send();
-		}
-	}
-
-	c.init();
-})();

drivers/chrome/js/options.js

@@ -9,7 +9,7 @@ document.addEventListener('DOMContentLoaded', function() {
 
 			d.getElementById('github'    ).addEventListener('click', function() { window.open(wappalyzer.config.githubURL);  });
 			d.getElementById('twitter'   ).addEventListener('click', function() { window.open(wappalyzer.config.twitterURL); });
-			d.getElementById('wappalyzer').addEventListener('click', function() { window.open(wappalyzer.config.websiteURL + '?utm_source=chrome&utm_medium=options&utm_campaign=extensions'); });
+			d.getElementById('wappalyzer').addEventListener('click', function() { window.open(wappalyzer.config.websiteURL + '?pk_campaign=chrome&pk_kwd=options'); });
 
 			d.getElementById('options-save').addEventListener('click', options.save);
 		},

drivers/chrome/js/popup.js

@@ -1,40 +1,19 @@
 document.addEventListener('DOMContentLoaded', function() {
 	var
 		d              = document,
-		analyzeHeaders = d.getElementById('analyze-headers'),
-		detectedApps   = d.getElementById('detected-apps')
-		;
+		detectedApps   = d.getElementById('detected-apps');
 
 	var popup = {
-		pollHeaders: null,
-
 		init: function() {
 			d.getElementById('options').addEventListener('click', function() {
 				window.open(chrome.extension.getURL('options.html'));
 			});
 
-			analyzeHeaders.innerHTML = chrome.i18n.getMessage('analyzeHeaders');
-			analyzeHeaders.removeAttribute('disabled');
-
 			chrome.tabs.getSelected(null, function(tab) {
 				if ( tab.url.match(/https?:\/\//) ) {
 					detectedApps.innerHTML = '<div class="empty">' + chrome.i18n.getMessage('noAppsDetected') + '</div>';
-
-					analyzeHeaders.addEventListener('click', function() {
-						analyzeHeaders.setAttribute('disabled', 'disabled');
-
-						chrome.extension.sendRequest({ id: 'fetch_headers', tab: tab });
-
-						popup.pollHeaders = setInterval(popup.displayApps, 100);
-					});
-
-					if ( parseInt(localStorage['autoAnalyzeHeaders']) ) {
-						analyzeHeaders.click();
-					}
 				} else {
 					detectedApps.innerHTML = '<div class="empty">' + chrome.i18n.getMessage('nothingToDo') + '</div>';
-
-					analyzeHeaders.setAttribute('disabled', 'disabled');
 				}
 			});
 
@@ -46,15 +25,7 @@ document.addEventListener('DOMContentLoaded', function() {
 
 			chrome.tabs.getSelected(null, function(tab) {
 				chrome.extension.sendRequest({ id: 'get_apps', tab: tab }, function(response) {
-					if ( response.tabCache.analyzed.indexOf('headers') > 0 ) {
-						if ( popup.pollHeaders != null ) {
-							clearTimeout(popup.pollHeaders);
-
-							analyzeHeaders.innerHTML = chrome.i18n.getMessage('analyzeHeadersDone');
-						}
-					}
-
-					if ( response.tabCache.count > 0 ) {
+					if ( response.tabCache && response.tabCache.count > 0 ) {
 						detectedApps.innerHTML = '';
 
 						for ( appName in response.tabCache.appsDetected ) {
@@ -63,14 +34,14 @@ document.addEventListener('DOMContentLoaded', function() {
 
 							html =
 								'<div class="detected-app">' +
-									'<a target="_blank" href="https://wappalyzer.com/applications/' + appName.toLowerCase().replace(/ /g, '-').replace(/[^\w-]/g, '') + '?utm_source=chrome&utm_medium=popup&utm_campaign=extensions">' +
+									'<a target="_blank" href="https://wappalyzer.com/applications/' + appName.toLowerCase().replace(/ /g, '-').replace(/[^\w-]/g, '') + '?pk_campaign=chrome&pk_kwd=popup">' +
 										'<img src="images/icons/' + appName + '.png"/>' +
 										'<span class="label">' + appName + ( version ? ' ' + version : '' ) + ( confidence < 100 ? ' (' + confidence + '% sure)' : '' ) + '</span>' +
 									'</a>';
 
 							response.apps[appName].cats.forEach(function(cat) {
 								html +=
-									'<a target="_blank" href="https://wappalyzer.com/categories/' + response.categories[cat] + '?utm_source=chrome&utm_medium=popup&utm_campaign=extensions">' +
+									'<a target="_blank" href="https://wappalyzer.com/categories/' + response.categories[cat] + '?pk_campaign=chrome&pk_kwd=popup">' +
 										'<span class="category">' + chrome.i18n.getMessage('categoryName' + cat) + '</span>' +
 									'</a>';
 							});

drivers/chrome/manifest.json

@@ -1,7 +1,7 @@
 { "name": "Wappalyzer",
-	"homepage_url": "https://wappalyzer.com?utm_source=chrome&utm_medium=context&utm_campaign=extensions",
+	"homepage_url": "https://wappalyzer.com?pk_campaign=chrome&pk_kwd=context",
 	"description": "Identifies software on the web",
-	"version": "2.29",
+	"version": "2.30",
 	"default_locale": "en",
 	"manifest_version": 2,
 	"icons": {
@@ -9,7 +9,10 @@
 		"128": "images/icon_128.png"
 	},
 	"browser_action": {
-		"default_icon": "images/icon_32.png",
+		"default_icon": {
+			"19": "images/icon_19.png",
+			"38": "images/icon_38.png"
+		},
 		"default_title": "Wappalyzer - click for details",
 		"default_popup": "popup.html"
 	},
@@ -23,6 +26,6 @@
 		"js/inject.js"
 	],
 	"options_page": "options.html",
-	"permissions": [ "tabs", "http://*/*", "https://*/*" ],
+	"permissions": [ "tabs", "webRequest", "http://*/*", "https://*/*" ],
 	"content_security_policy": "script-src 'self' https://ssl.google-analytics.com; object-src 'self'"
 }

drivers/chrome/options.html

@@ -18,10 +18,6 @@
 	<body>
 		<h1 data-i18n="options"></h1>
 
-		<p>
-			<label for="option-auto-analyze-headers"><input id="option-auto-analyze-headers" type="checkbox"> <span data-i18n="optionAutoAnalyzeHeaders"></span></label>
-		</p>
-
 		<p>
 			<label for="option-upgrade-message"><input id="option-upgrade-message" type="checkbox"> <span data-i18n="optionUpgradeMessage"></span></label>
 		</p>

drivers/chrome/popup.html

@@ -14,9 +14,8 @@
 	<body>
 		<div id="detected-apps"></div>
 
-		<div id="buttons">
-			<button id="analyze-headers" data-i18n="analyzeHeaders" disabled="disabled"></button>
-			<button id="options"></button>
+		<div id="footer">
+			<a href="javascript: void(0);" id="options">Options</a>
 		</div>
 	</body>
 </html>

drivers/firefox/lib/driver.js

@@ -164,8 +164,6 @@
 		init: function(callback) {
 			var json = JSON.parse(data.load('apps.json'));
 
-			console.log('xxxx');
-
 			if ( sp.prefs.urlbar ) {
 				createPanel();
 			} else {
@@ -178,7 +176,7 @@
 				if ( !ss.storage.version ) {
 					w.driver.goToURL({ url: w.config.websiteURL + 'installed', medium: 'install' });
 				} else if ( version !== ss.storage.version ) {
-					w.driver.goToURL({ url: w.config.websiteURL + 'upgraded', medium: 'upgrade' });
+					w.driver.goToURL({ url: w.config.websiteURL + 'upgraded', medium: 'upgrade', background: true });
 				}
 
 				ss.storage.version = version;
@@ -247,9 +245,9 @@
 		},
 
 		goToURL: function(args) {
-			var url = args.url + ( typeof args.medium === 'undefined' ? '' :  '?utm_source=firefox&utm_medium=' + args.medium + '&utm_campaign=extensions');
+			var url = args.url + ( typeof args.medium === 'undefined' ? '' : '?pk_campaign=chrome&pk_kwd=' + args.medium);
 
-			tabs.open(url);
+			tabs.open({ url: url, inBackground: args.background !== undefined && args.background });
 		},
 
 		displayApps: function() {

drivers/firefox/package.json

@@ -1,9 +1,10 @@
 {
   "name": "wappalyzer",
   "title": "Wappalyzer",
+	"homepage": "https://wappalyzer.com",
 	"icon": "images/icon48_hot.png",
 	"icon64": "images/icon64_hot.png",
-  "id": "ec8030f7-c20a-464f-9b0e-13a3a9e97384",
+  "id": "wappalyzer@crunchlabz.com",
   "description": "Identifies software on the web",
   "author": "Elbert Alias",
   "license": "GPLv3",