wappalyzer/src/wappalyzer.js

/**
 * Wappalyzer v4
 *
 * Created by Elbert Alias <elbert@alias.io>
 *
 * License: GPLv3 http://www.gnu.org/licenses/gpl-3.0.txt
 */

var wappalyzer = (function() {
	//'use strict';

	/**
	 * Application class
	 */
	var Application = function(app, detected) {
		this.app             = app;
		this.confidence      = {};
		this.confidenceTotal = 0;
		this.detected        = Boolean(detected);
		this.excludes        = [];
		this.version         = '';
		this.versions        = [];
	};

	Application.prototype = {
		/**
		 * Calculate confidence total
		 */
		getConfidence: function() {
			var total = 0, id;

			for ( id in this.confidence ) {
				total += this.confidence[id];
			}

			return this.confidenceTotal = Math.min(total, 100);
		},

		/**
		 * Resolve version number (find the longest version number that contains all shorter detected version numbers)
		 */
		getVersion: function() {
			var i, resolved;

			if ( !this.versions.length ) {
				return;
			}

			this.versions.sort(function(a, b) {
				return a.length - b.length;
			});

			resolved = this.versions[0];

			for ( i = 1; i < this.versions.length; i++ ) {
				if ( this.versions[i].indexOf(resolved) === -1 ) {
					break;
				}

				resolved = this.versions[i];
			}

			return this.version = resolved;
		},

		setDetected: function(pattern, type, value, key) {
			this.detected = true;

			// Set confidence level
			this.confidence[type + ' ' + ( key ? key + ' ' : '' ) + pattern.regex] = pattern.confidence ? pattern.confidence : 100;

			// Detect version number
			if ( pattern.version ) {
				var
					version = pattern.version,
					matches = pattern.regex.exec(value);

				if ( matches ) {
					matches.forEach(function(match, i) {
						// Parse ternary operator
						var ternary = new RegExp('\\\\' + i + '\\?([^:]+):(.*)$').exec(version);

						if ( ternary && ternary.length === 3 ) {
							w.log({ match: match, i: i, ternary: ternary });

							version = version.replace(ternary[0], match ? ternary[1] : ternary[2]);

							w.log({ version: version });
						}

						// Replace back references
						version = version.replace(new RegExp('\\\\' + i, 'g'), match ? match : '');
					});

					if ( version && this.versions.indexOf(version) < 0 ) {
						this.versions.push(version);
					}

					this.getVersion();
				}
			}
		}
	};

	/**
	 * Call driver functions
	 */
	var driver = function(func, args) {
		if ( typeof w.driver[func] !== 'function' ) {
			w.log('not implemented: w.driver.' + func, 'warn');

			return;
		}

		if ( func !== 'log' ) {
			w.log('w.driver.' + func);
		}

		return w.driver[func](args);
	};

	/**
	 * Parse apps.json patterns
	 */
	var parsePatterns = function(patterns) {
		var
			key,
			parsed = {};

		// Convert array to object containing array
		if ( patterns instanceof Array ) {
			patterns = { main: patterns }
		}

		// Convert string to object containing array containing string
		if ( typeof patterns === 'string' ) {
			patterns = { main: [ patterns ] };
		}

		for ( key in patterns ) {
			parsed[key] = [];

			// Convert string to array containing string
			if ( typeof patterns[key] === 'string' ) {
				patterns[key] = [ patterns[key] ];
			}

			patterns[key].forEach(function(pattern) {
				var attrs = {};

				pattern.split('\\;').forEach(function(attr, i) {
					if ( i ) {
						// Key value pairs
						attr = attr.split(':');

						if ( attr.length > 1 ) {
							attrs[attr.shift()] = attr.join(':');
						}
					} else {
						attrs.string = attr;

						try {
							attrs.regex = new RegExp(attr.replace('/', '\/'), 'i'); // Escape slashes in regular expression
						} catch (e) {
							attrs.regex = new RegExp();

							w.log(e + ': ' + attr, 'error');
						}
					}
				});

				parsed[key].push(attrs);
			});
		}

		// Convert back to array if the original pattern list was an array (or string)
		if ( parsed.hasOwnProperty('main') ) {
			parsed = parsed.main;
		}

		return parsed;
	};

	/**
	 * Main script
	 */
	var w = {
		apps:     {},
		cats:     null,
		ping:     { hostnames: {} },
		adCache:  [],
		detected: {},

		config: {
			websiteURL: 'https://wappalyzer.com/',
			twitterURL: 'https://twitter.com/Wappalyzer',
			githubURL:  'https://github.com/AliasIO/Wappalyzer',
		},

		/**
		 * Log messages to console
		 */
		log: function(message, type) {
			if ( type === undefined ) {
				type = 'debug';
			}

			if ( typeof message === 'object' ) {
				message = JSON.stringify(message);
			}

			driver('log', { message: message, type: type });
		},

		/**
		 * Initialize
		 */
		init: function() {
			w.log('w.init');

			// Checks
			if ( w.driver === undefined ) {
				w.log('no driver, exiting');

				return;
			}

			// Initialize driver
			driver('init');
		},

		/**
		 * Analyze the request
		 */
		analyze: function(hostname, url, data) {
			var
				app, confirmMatch, type,
				apps = {};

			w.log('w.analyze');

			if ( w.apps === undefined || w.categories === undefined ) {
				w.log('apps.json not loaded, check for syntax errors');

				return;
			}

			// Remove hash from URL
			data.url = url = url.split('#')[0];

			if ( typeof data.html !== 'string' ) {
				data.html = '';
			}

			if ( w.detected[url] === undefined ) {
				w.detected[url] = {};
			}

			for ( app in w.apps ) {
				apps[app] = w.detected[url] && w.detected[url][app] ? w.detected[url][app] : new Application(app);

				for ( type in w.apps[app] ) {
					confirmMatch = function(pattern, value, key) {
						apps[app].setDetected(pattern, type, value, key);
					}

					switch ( type ) {
						case 'url':
							if ( url ) {
								w.analyzeUrl(parsePatterns(w.apps[app][type]), url, confirmMatch);
							}

							break;
						case 'html':
							if ( data.html ) {
								w.analyzeHtml(parsePatterns(w.apps[app][type]), data.html, confirmMatch);
							}

							break;
						case 'script':
							if ( data.html ) {
								w.analyzeScript(parsePatterns(w.apps[app][type]), data.html, confirmMatch);
							}

							break;
						case 'meta':
							if ( data.html ) {
								w.analyzeMeta(parsePatterns(w.apps[app][type]), data.html, confirmMatch);
							}

							break;
						case 'headers':
							if ( data.hasOwnProperty('headers') && data.headers ) {
								w.analyzeHeaders(parsePatterns(w.apps[app][type]), data.headers, confirmMatch);
							}

							break;
						case 'env':
							if ( data.hasOwnProperty('env') && data.env ) {
								w.analyzeEnv(parsePatterns(w.apps[app][type]), data.env, confirmMatch);
							}

							break;
						default:
					}
				}
			}

			for ( app in apps ) {
				if ( !apps[app].detected ) {
					delete apps[app];
				}
			}

			w.resolveExcludes(apps);
			w.resolveImplies(apps, url);

			w.cacheDetectedApps(apps, url);
			w.trackDetectedApps(apps, url, hostname, data.html);

			w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url);

			driver('displayApps');
		},

		resolveExcludes: function(apps) {
			var
				app,
				excludes = [];

			// Exclude app in detected apps only
			for ( app in apps ) {
				if ( w.apps[app].excludes ) {
					if ( typeof w.apps[app].excludes === 'string' ) {
						w.apps[app].excludes = [ w.apps[app].excludes ];
					}

					w.apps[app].excludes.forEach(function(excluded) {
						excludes.push(excluded);
					});
				}
			}

			// Remove excluded applications
			for ( app in apps ) {
				if ( excludes.indexOf(app) !== -1 ) {
					delete apps[app];
				}
			}
		},

		resolveImplies: function(apps, url) {
			var
				confidence,
				id,
				checkImplies = true;

			// Implied applications
			// Run several passes as implied apps may imply other apps
			while ( checkImplies ) {
				checkImplies = false;

				for ( app in apps ) {
					confidence = apps[app].confidence;

					if ( w.apps[app] && w.apps[app].implies ) {
						// Cast strings to an array
						if ( typeof w.apps[app].implies === 'string' ) {
							w.apps[app].implies = [ w.apps[app].implies ];
						}

						w.apps[app].implies.forEach(function(implied) {
							implied = parsePatterns(implied)[0];

							if ( !w.apps[implied.string] ) {
								w.log('Implied application ' + implied.string + ' does not exist', 'warn');

								return;
							}

							if ( !apps.hasOwnProperty(implied.string) ) {
								apps[implied.string] = w.detected[url] && w.detected[url][implied.string] ? w.detected[url][implied.string] : new Application(implied.string, true);

								checkImplies = true;
							}

							// Apply app confidence to implied app
							for ( id in confidence ) {
								apps[implied.string].confidence[id + ' implied by ' + app] = confidence[id] * ( implied.confidence ? implied.confidence / 100 : 1 );
							}
						});
					}
				}
			}
		},

		/**
		 * Cache detected applications
		 */
		cacheDetectedApps: function(apps, url) {
			var app, id, confidence;

			for ( app in apps ) {
				confidence = apps[app].confidence;

				// Per URL
				w.detected[url][app] = apps[app];

				for ( id in confidence ) {
					w.detected[url][app].confidence[id] = confidence[id];
				}
			}
		},

		/**
		 * Track detected applications
		 */
		trackDetectedApps: function(apps, url, hostname, html) {
			var app, match;

			for ( app in apps ) {
				if ( w.detected[url][app].getConfidence() >= 100 ) {
					if ( /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/.test(hostname) && !/((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/.test(url) ) {
						if ( !w.ping.hostnames.hasOwnProperty(hostname) ) {
							w.ping.hostnames[hostname] = {
								applications: {},
								meta: {}
							};
						}

						if ( !w.ping.hostnames[hostname].applications.hasOwnProperty(app) ) {
							w.ping.hostnames[hostname].applications[app] = {
								hits: 0
							};
						}

						w.ping.hostnames[hostname].applications[app].hits ++;

						if ( apps[app].version ) {
							w.ping.hostnames[hostname].applications[app].version = apps[app].version;
						}
					} else {
						w.log('Ignoring hostname "' + hostname + '"');
					}
				}
			}

			// Additional information
			if ( w.ping.hostnames.hasOwnProperty(hostname) ) {
				match = html.match(/<html[^>]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"/i);

				if ( match && match.length ) {
					w.ping.hostnames[hostname].meta['language'] = match[1];
				}
			}

			if ( Object.keys(w.ping.hostnames).length >= 50 || w.adCache.length >= 50 ) {
				driver('ping');
			}
		},

		/**
		 * Analyze URL
		 */
		analyzeUrl: function(patterns, url, confirmMatch) {
			patterns.forEach(function(pattern) {
				if ( pattern.regex.test(url) ) {
					confirmMatch(pattern, url);
				}
			});
		},

		/**
		 * Analyze HTML
		 */
		analyzeHtml: function(patterns, html, confirmMatch) {
			patterns.forEach(function(pattern) {
				if ( pattern.regex.test(html) ) {
					confirmMatch(pattern, html);
				}
			});
		},

		/**
		 * Analyze script tag
		 */
		analyzeScript: function(patterns, html, confirmMatch) {
			var regex = new RegExp('<script[^>]+src=("|\')([^"\']+)', 'ig');

			patterns.forEach(function(pattern) {
				var match;

				while ( match = regex.exec(html) ) {
					if ( pattern.regex.test(match[2]) ) {
						confirmMatch(pattern, match[2]);
					}
				}
			});
		},

		/**
		 * Analyze meta tag
		 */
		analyzeMeta: function(patterns, html, confirmMatch) {
			var
				content, match, meta,
				regex = /<meta[^>]+>/ig;

			while ( match = regex.exec(html) ) {
				for ( meta in patterns ) {
					if ( new RegExp('(name|property)=["\']' + meta + '["\']', 'i').test(match) ) {
						content = match.toString().match(/content=("|')([^"']+)("|')/i);

						patterns[meta].forEach(function(pattern) {
							if ( content && content.length === 4 && pattern.regex.test(content[2]) ) {
								confirmMatch(pattern, content[2], meta);
							}
						});
					}
				}
			}
		},

		/**
		 * analyze response headers
		 */
		analyzeHeaders: function(patterns, headers, confirmMatch) {
			var header;

			for ( header in patterns ) {
				patterns[header].forEach(function(pattern) {
					header = header.toLowerCase();

					if ( headers.hasOwnProperty(header) && pattern.regex.test(headers[header]) ) {
						confirmMatch(pattern, headers[header], header);
					}
				});
			}
		},

		/**
		 * Analyze environment variables
		 */
		analyzeEnv: function(patterns, envs, confirmMatch) {
			var env;

			patterns.forEach(function(pattern) {
				for ( env in envs ) {
					if ( pattern.regex.test(envs[env]) ) {
						confirmMatch(pattern, envs[env]);
					}
				}
			});
		}
	};

	return w;
})();

// CommonJS package
// See http://wiki.commonjs.org/wiki/CommonJS
if ( typeof exports === 'object' ) {
	exports.wappalyzer = wappalyzer;
}