sync to master

drivers/phantomjs/js/driver.js

@@ -1,10 +1,24 @@
 (function() {
-	var url, json, app, categories, page;
+	var
+		url,
+		originalUrl,
+		args  = [],
+		debug = false;
 
 	try {
-		if ( require('system').args.length > 1 ) {
-			url = require('system').args[1];
-		} else {
+		require('system').args.forEach(function(arg) {
+			switch ( arg ) {
+				case '-v':
+				case '--verbose':
+					debug = true;
+
+					break;
+				default:
+					url = originalUrl = arg;
+			}
+		});
+
+		if ( !url ) {
 			throw new Error('Usage: phantomjs ' + require('system').args[0] + ' <url>');
 		}
 
@@ -17,23 +31,59 @@
 			 * Log messages to console
 			 */
 			log: function(args) {
-				console.log(args.message);
+				if ( debug || args.type !== 'debug' ) {
+					console.log(args.message);
+				}
 			},
 
 			/**
 			 * Display apps
 			 */
 			displayApps: function() {
-				var count = wappalyzer.detected[url] ? Object.keys(wappalyzer.detected[url]).length.toString() : '0';
-
-				console.log(count);
+				var
+					app,
+					apps  = [],
+					cats  = [],
+					count = wappalyzer.detected[url] ? Object.keys(wappalyzer.detected[url]).length : 0;
+
+				wappalyzer.log('driver.displayApps');
+
+				if ( count ) {
+					for ( app in wappalyzer.detected[url] ) {
+
+						wappalyzer.apps[app].cats.forEach(function(cat) {
+							cats.push(wappalyzer.categories[cat]);
+						});
+
+						apps.push({
+							url:         originalUrl,
+							finalUrl:    url,
+							application: app,
+							confidence:  wappalyzer.detected[url][app].confidenceTotal,
+							version:     wappalyzer.detected[url][app].version,
+							categories:  cats
+						});
+					}
+
+					console.log(JSON.stringify(apps));
+				}
 			},
 
 			/**
 			 * Initialize
 			 */
 			init: function() {
-				json = JSON.parse(require('fs').read('apps.json'));
+				var
+					page, hostname,
+					headers = {};
+					a       = document.createElement('a'),
+					json    = JSON.parse(require('fs').read('apps.json'));
+
+				wappalyzer.log('driver.init');
+
+				a.href = url.replace(/#.*$/, '');
+
+				hostname = a.hostname;
 
 				wappalyzer.apps       = json.apps;
 				wappalyzer.categories = json.categories;
@@ -41,19 +91,58 @@
 				page = require('webpage').create();
 
 				page.onConsoleMessage = function(message) {
-					console.log(message);
+					wappalyzer.log(message);
+				};
+
+				page.onResourceReceived = function(response) {
+					if ( response.url.replace(/\/$/, '') === url.replace(/\/$/, '') ) {
+						if ( response.redirectURL ) {
+							url = response.redirectURL;
+
+							return;
+						}
+
+						if ( response.stage === 'end' && response.contentType.indexOf('text/html') !== -1 ) {
+							response.headers.forEach(function(header) {
+								headers[header.name.toLowerCase()] = header.value;
+							});
+						}
+					}
 				};
 
 				page.open(url, function(status) {
-					var a, hostname;
+					var html, environmentVars;
+
+					if ( status === 'fail' ) {
+						return;
+					}
+
+					html = page.content;
+
+					if ( html.length > 50000 ) {
+						html = html.substring(0, 25000) + html.substring(html.length - 25000, html.length);
+					}
+
+					// Collect environment variables
+					environmentVars = page.evaluate(function() {
+						var i, environmentVars;
+
+						for ( i in window ) {
+							environmentVars += i + ' ';
+						}
 
-					a = document.createElement('a');
+						return environmentVars;
+					});
 
-					a.href = url.replace(/#.*$/, '');
+					wappalyzer.log({ message: 'environmentVars: ' + environmentVars });
 
-					hostname = a.hostname;
+					environmentVars = environmentVars.split(' ').slice(0, 500);
 
-					wappalyzer.analyze(hostname, url, { html: page.content });
+					wappalyzer.analyze(hostname, url, {
+						html:    html,
+						headers: headers,
+						env:     environmentVars
+					});
 
 					phantom.exit();
 				});

share/apps.json

@@ -190,7 +190,7 @@
 		"Akamai": {
 			"website": "akamai.com",
 			"cats": [ 31 ],
-			"headers": { "X-Akamai-Transformed": ".*" }
+			"headers": { "X-Akamai-Transformed": "" }
 		},
 		"Allegro RomPager": {
 			"website": "allegrosoft.com/embedded-web-server-s2",
@@ -312,7 +312,7 @@
 			"website": "jfrog.com/open-source/#os-arti",
 			"cats": [ 47 ],
 			"env": "^ArtifactoryUpdates$",
-			"script": [ "wicket/resource/org\\.artifactory." ],
+			"script": [ "wicket/resource/org\\.artifactory\\." ],
 			"html": [ "<span class=\"version\">Artifactory(?: Pro)?(?: Power Pack)?(?: ([\\d.]+))?\\;version:\\1" ]
 		},
 		"AsciiDoc": {
@@ -342,8 +342,8 @@
 			"website": "www.atlassian.com/software/confluence/overview/team-collaboration-software",
 			"cats": [ 8 ],
 			"html": "Powered by <a href=[^>]+atlassian\\.com/software/confluence(?:[^>]+>Atlassian Confluence</a> ([\\d.]+))?\\;version:\\1",
-			"headers": { "X-Confluence-Request-Time": ".*" },
-			"meta": { "confluence-request-time": ".*" },
+			"headers": { "X-Confluence-Request-Time": "" },
+			"meta": { "confluence-request-time": "" },
 			"implies": "Java"
 		},
 		"Atlassian Jira": {
@@ -547,7 +547,7 @@
 		"Cargo": {
 			"website": "cargocollective.com",
 			"cats": [ 1 ],
-			"meta": { "cargo_title": ".*" },
+			"meta": { "cargo_title": "" },
 			"script": "/cargo\\.",
 			"html": "<link [^>]+Cargo feed",
 			"implies": "PHP"
@@ -575,7 +575,7 @@
 		"CacheFly": {
 			"website": "www.cachefly.com",
 			"cats": [ 31 ],
-			"headers": { "Server": "^CFS ", "X-CF1": ".+", "X-CF2": ".+" }
+			"headers": { "Server": "^CFS ", "X-CF1": "", "X-CF2": "" }
 		},
 		"cgit": {
 			"website": "git.zx2c4.com/cgit/",
@@ -688,7 +688,7 @@
 		"Commerce Server": {
 			"website": "commerceserver.net",
 			"cats": [ 6 ],
-			"headers": { "COMMERCE-SERVER-SOFTWARE": ".+" },
+			"headers": { "COMMERCE-SERVER-SOFTWARE": "" },
 			"implies": "Microsoft ASP.NET"
 		},
 		"CompaqHTTPServer": {
@@ -867,11 +867,6 @@
 			"meta": { "generator": "DataLife Engine" },
 			"implies": [ "PHP", "Apache" ]
 		},
-		"DAV": {
-			"website": "???",
-			"cats": [ 33 ],
-			"headers": { "Server": "\\bDAV\\b(?:/([\\d.]+))?\\;version:\\1" }
-		},
 		"David Webbox": {
 			"website": "www.tobit.com",
 			"cats": [ 22 ],
@@ -986,7 +981,7 @@
 			"cats": [ 1 ],
 			"meta": { "generator": "DotNetNuke" },
 			"script": "/js/dnncore\\.js",
-			"headers": { "DNNOutputCache": ".+", "X-Compressed-By": "DotNetNuke", "Set-Cookie": "DotNetNukeAnonymous=" },
+			"headers": { "DNNOutputCache": "", "X-Compressed-By": "DotNetNuke", "Set-Cookie": "DotNetNukeAnonymous=" },
 			"html": "<!-- by DotNetNuke Corporation",
 			"env": "^DotNetNuke$",
 			"implies": "Microsoft ASP.NET"
@@ -1013,7 +1008,7 @@
 			"cats": [ 1 ],
 			"script": "drupal\\.js",
 			"html": "<(?:link|style)[^>]+sites/(?:default|all)/(?:themes|modules)/",
-			"headers": { "X-Drupal-Cache": ".*", "X-Generator": "Drupal(?:\\s([\\d.]+))?\\;version:\\1", "Expires": "19 Nov 1978" },
+			"headers": { "X-Drupal-Cache": "", "X-Generator": "Drupal(?:\\s([\\d.]+))?\\;version:\\1", "Expires": "19 Nov 1978" },
 			"meta": { "generator": "Drupal(?:\\s([\\d.]+))?\\;version:\\1" },
 			"env": "^Drupal$",
 			"implies": "PHP"
@@ -1254,7 +1249,7 @@
 			"cats": [ 1 ],
 			"meta": { "generator": "FlexCMP" },
 			"html": "<!--[^>]+FlexCMP[^>v]+v\\. ([\\d.]+)\\;version:\\1",
-			"headers": { "X-Powered-By": "FlexCMP.+\\[v\\. ([\\d.]+)\\;version:\\1", "X-Flex-Lang": ".+" }
+			"headers": { "X-Powered-By": "FlexCMP.+\\[v\\. ([\\d.]+)\\;version:\\1", "X-Flex-Lang": "" }
 		},
 		"FluxBB": {
 			"website": "fluxbb.org",
@@ -1348,7 +1343,7 @@
 			"website": "ghost.org",
 			"cats": [ 11 ],
 			"meta": { "generator": "Ghost(?:\\s([\\d.]+))?\\;version:\\1" },
-			"headers": { "X-Ghost-Cache-Status": ".*" }
+			"headers": { "X-Ghost-Cache-Status": "" }
 		},
 		"git": {
 			"website": "git-scm.com",
@@ -1518,7 +1513,8 @@
 		"Happy ICS Server": {
 			"website": "???",
 			"cats": [ 22 ],
-			"headers": { "Server": "Happy ICS Server(?:/([\\d.]+))?\\;version:\\1" }
+			"headers": { "Server": "Happy ICS Server(?:/([\\d.]+))?\\;version:\\1" },
+			"implies": "OmniTouch 8660 My Teamwork"
 		},
 		"HeadJS": {
 			"website": "headjs.com",
@@ -1599,7 +1595,7 @@
 		"HTTP-Server": {
 			"website": "???",
 			"cats": [ 22 ],
-			"headers": { "Server": "\\bHTTP-Server(?: ?/?V?([\\d.]+))?\\;version:\\1" }
+			"headers": { "Server": "(?:^|[^-])\\bHTTP-Server(?: ?/?V?([\\d.]+))?\\;version:\\1" }
 		},
 		"HubSpot": {
 			"website": "hubspot.com",
@@ -1632,7 +1628,7 @@
 		"IBM WebSphere Portal": {
 			"website": "ibm.com/software/websphere/portal",
 			"cats": [ 1 ],
-			"headers": { "IBM-Web2-Location": ".*", "Itx-Generated-Timestamp": ".+" },
+			"headers": { "IBM-Web2-Location": "", "Itx-Generated-Timestamp": "" },
 			"url": "/wps/",
 			"implies": "Java"
 		},
@@ -1645,7 +1641,7 @@
 		"iCongo": {
 			"website": "hybris.com/icongo",
 			"cats": [ 6 ],
-			"meta": { "iCongo": ".*" },
+			"meta": { "iCongo": "" },
 			"implies": "Adobe ColdFusion"
 		},
 		"IIS": {
@@ -2203,14 +2199,14 @@
 			"cats": [ 1 ],
 			"env": "^eidosBase$\\;confidence:99",
 			"html": "<!-- Methode uuid: \"[a-f\\d]+\" ?-->",
-			"meta": { "eomportal-instanceid": "\\d+", "eomportal-id": "\\d+", "eomportal-loid": "[\\d.]+", "eomportal-uuid": "[a-f\\d]+", "eomportal-lastUpdate": ".*" }
+			"meta": { "eomportal-instanceid": "\\d+", "eomportal-id": "\\d+", "eomportal-loid": "[\\d.]+", "eomportal-uuid": "[a-f\\d]+", "eomportal-lastUpdate": "" }
 		},
 		"Microsoft ASP.NET": {
 			"website": "www.asp.net",
 			"cats": [ 18 ],
 			"url": "\\.aspx(?:$|\\?)",
 			"html": "<input[^>]+name=\"__VIEWSTATE",
-			"headers": { "X-Powered-By": "ASP\\.NET\\;confidence:50", "X-AspNet-Version": "(.+)\\;version:\\1", "Set-Cookie": "(ASPSESSION|ASP\\.NET_SessionId)" },
+			"headers": { "X-Powered-By": "ASP\\.NET\\;confidence:50", "X-AspNet-Version": "(.+)\\;version:\\1", "Set-Cookie": "ASPSESSION|ASP\\.NET_SessionId" },
 			"implies": "IIS\\;confidence:50"
 		},
 		"Microsoft HTTPAPI": {
@@ -2222,7 +2218,7 @@
 			"website": "sharepoint.microsoft.com",
 			"cats": [ 1 ],
 			"meta": { "generator": "Microsoft SharePoint" },
-			"headers": { "MicrosoftSharePointTeamServices": "(.*)\\;version:\\1", "X-SharePointHealthScore": ".*", "SPRequestGuid": ".*", "SharePointHealthScore": ".*" },
+			"headers": { "MicrosoftSharePointTeamServices": "(.*)\\;version:\\1", "X-SharePointHealthScore": "", "SPRequestGuid": "", "SharePointHealthScore": "" },
 			"env": "^_spBodyOnLoadCalled$"
 		},
 		"mini_httpd": {
@@ -2295,6 +2291,12 @@
 			"headers": { "Server": "mod_auth_pam(?:/([\\d\\.]+))?\\;version:\\1" },
 			"implies": "Apache"
 		},
+		"mod_dav": {
+			"website": "webdav.org/mod_dav",
+			"cats": [ 33 ],
+			"headers": { "Server": "\\b(?:mod_)?DAV\\b(?:/([\\d.]+))?\\;version:\\1" },
+			"implies": "Apache"
+		},
 		"mod_fastcgi": {
 			"website": "www.fastcgi.com/mod_fastcgi/docs/mod_fastcgi.html",
 			"cats": [ 33 ],
@@ -2547,6 +2549,10 @@
 			"headers": { "Set-Cookie": "october_session=" },
 			"implies": "Laravel"
 		},
+		"OmniTouch 8660 My Teamwork": {
+			"website": "enterprise.alcatel-lucent.com",
+			"cats": [ 19 ]
+		},
 		"Open AdStream": {
 			"website": "xaxis.com",
 			"cats": [ 36 ],
@@ -2579,7 +2585,7 @@
 		"openEngine": {
 			"website": "openengine.de/html/pages/de/",
 			"cats": [ 1 ],
-			"meta": { "openEngine": ".*" }
+			"meta": { "openEngine": "" }
 		},
 		"OpenGrok": {
 			"website": "hub.opensolaris.org/bin/view/Project+opengrok/WebHome",
@@ -2644,8 +2650,7 @@
 		"Oracle HTTP Server": {
 			"website": "oracle.com",
 			"cats": [ 22 ],
-			"headers": { "Server": "Oracle-HTTP-Server(?:/([\\d.]+))?\\;version:\\1" },
-			"excludes": [ "HTTP-Server" ]
+			"headers": { "Server": "Oracle-HTTP-Server(?:/([\\d.]+))?\\;version:\\1" }
 		},
 		"Oracle Recommendations On Demand": {
 			"website": "www.oracle.com/us/products/applications/commerce/recommendations-on-demand/index.html",
@@ -2988,7 +2993,7 @@
 		"RackCache": {
 			"website": "github.com/rtomayko/rack-cache",
 			"cats": [ 23 ],
-			"headers": { "X-Rack-Cache": ".+" },
+			"headers": { "X-Rack-Cache": "" },
 			"implies": "Ruby"
 		},
 		"RAID HTTPServer": {
@@ -3248,7 +3253,7 @@
 			"website": "simgroep.nl/internet/portfolio-contentbeheer_41623/",
 			"cats": [ 1 ],
 			"script": "/sim(?:site|core)/js",
-			"meta": { "SIM.medium": ".*" }
+			"meta": { "SIM.medium": "" }
 		},
 		"Site Meter": {
 			"website": "www.sitemeter.com",
@@ -3366,14 +3371,14 @@
 		"Solodev": {
 			"website": "www.solodev.com",
 			"cats": [ 1 ],
-			"headers": { "solodev_session": ".*" },
+			"headers": { "solodev_session": "" },
 			"html": "<div class='dynamicDiv' id='dd\\.\\d\\.\\d'>",
 			"implies": "PHP"
 		},
 		"SPDY": {
 			"website": "chromium.org/spdy",
 			"cats": [ 19 ],
-			"headers": { "X-Firefox-Spdy": ".*" }
+			"headers": { "X-Firefox-Spdy": "" }
 		},
 		"Sphinx": {
 			"website": "sphinx.pocoo.org",
@@ -3396,7 +3401,7 @@
 			"website": "www.spip.net",
 			"cats": [ 1 ],
 			"meta": { "generator": "(?:^|\\s)SPIP(?:\\s([\\d.]+(?:\\s\\[\\d+\\])?))?\\;version:\\1" },
-			"headers": { "X-Spip-Cache": ".*" }
+			"headers": { "X-Spip-Cache": "" }
 		},
 		"Spree": {
 			"website": "spreecommerce.com",
@@ -3510,7 +3515,7 @@
 			"cats": [ 18 ],
 			"meta": { "generator": "Swiftlet" },
 			"html": "Powered by <a href=\"[^>]+Swiftlet",
-			"headers": { "X-Swiftlet-Cache": ".*", "X-Powered-By": "Swiftlet", "X-Generator": "Swiftlet" },
+			"headers": { "X-Swiftlet-Cache": "", "X-Powered-By": "Swiftlet", "X-Generator": "Swiftlet" },
 			"implies": "PHP"
 		},
 		"Synology DiskStation": {
@@ -3610,7 +3615,7 @@
 			"cats": [ 11 ],
 			"html": "<iframe src=\"[^>]+tumblr\\.com",
 			"url": "^https?://(?:www\\.)?[^/]+\\.tumblr\\.com/",
-			"headers": { "X-Tumblr-User": ".*" }
+			"headers": { "X-Tumblr-User": "" }
 		},
 		"TweenMax": {
 			"website": "greensock.com/tweenmax",
@@ -3762,7 +3767,7 @@
 		"Varnish": {
 			"website": "www.varnish-cache.org",
 			"cats": [ 23 ],
-			"headers": { "X-Varnish": ".+", "X-Varnish-Age": ".+", "X-Varnish-Cache": ".+", "X-Varnish-Action": ".+", "X-Varnish-Hostname": ".+", "Via": "Varnish" }
+			"headers": { "X-Varnish": "", "X-Varnish-Age": "", "X-Varnish-Cache": "", "X-Varnish-Action": "", "X-Varnish-Hostname": "", "Via": "Varnish" }
 		},
 		"vBulletin": {
 			"website": "www.vbulletin.com",
@@ -3978,7 +3983,7 @@
 			"website": "wix.com",
 			"cats": [ 1 ],
 			"script": "static\\.wixstatic\\.com",
-			"headers": { "X-Wix-Dispatcher-Cache-Hit": ".+", "Set-Cookie": "Domain=\\.wix\\.com" },
+			"headers": { "X-Wix-Dispatcher-Cache-Hit": "", "Set-Cookie": "Domain=\\.wix\\.com" },
 			"env": "^wix(?:Events|Data|Errors)"
 		},
 		"Wolf CMS": {