Merge remote-tracking branch 'AliasIO/master'

src/apps.json

@@ -182,18 +182,6 @@
 			"script": "addthis\\.com/js/",
 			"website": "www.addthis.com"
 		},
-		"Adobe CQ5": {
-			"cats": [
-				1
-			],
-			"html": [
-				"<div class=\"[^\"]*parbase",
-				"<div[^>]+data-component-path=\"[^\"+]jcr:"
-			],
-			"implies": "Java",
-			"script": "/etc/designs/",
-			"website": "adobe.com/products/cq.html"
-		},
 		"Adobe ColdFusion": {
 			"cats": [
 				18
@@ -208,6 +196,18 @@
 			"url": "\\.cfm(?:$|\\?)",
 			"website": "adobe.com/products/coldfusion-family.html"
 		},
+		"Adobe Experience Manager": {
+			"cats": [
+				1
+			],
+			"html": [
+				"<div class=\"[^\"]*parbase",
+				"<div[^>]+data-component-path=\"[^\"+]jcr:"
+			],
+			"implies": "Java",
+			"script": "/etc/designs/",
+			"website": "www.adobe.com/au/marketing-cloud/enterprise-content-management.html"
+		},
 		"Adobe GoLive": {
 			"cats": [
 				20
@@ -1426,7 +1426,7 @@
 		},
 		"DataTables": {
 			"cats": [
-			  12
+				12
 			],
 			"implies": "jQuery",
 			"script": "dataTables.*\\.js",
@@ -1697,6 +1697,13 @@
 			},
 			"website": "www.dynamicweb.dk"
 		},
+		"Dynatrace": {
+			"cats": [
+				10
+			],
+			"script": "dtagent.*\\.js",
+			"website": "dynatrace.com"
+		},
 		"E-Merchant": {
 			"cats": [
 				6
@@ -2724,6 +2731,14 @@
 			"script": "jscripts/ips_",
 			"website": "www.invisionpower.com"
 		},
+		"Immutable.js": {
+			"cats": [
+				12
+			],
+			"env": "^Immutable$",
+			"script": "^immutable\\.(?:min\\.)?js$",
+			"website": "facebook.github.io/immutable-js/"
+		},
 		"ImpressCMS": {
 			"cats": [
 				1
@@ -3397,8 +3412,7 @@
 			],
 			"env": [
 				"^fyre$",
-				"^FyreLoader$",
+				"^FyreLoader$"
-				"^LF$"
 			],
 			"html": "<[^>]+(?:id|class)=\"livefyre",
 			"script": "livefyre_init\\.js",
@@ -3483,25 +3497,25 @@
 			],
 			"website": "mrincworld.com"
 		},
-		"M.R. Inc Webserver": {
+		"M.R. Inc SiteFrame": {
 			"cats": [
-				22
+				18
 			],
 			"headers": {
-				"Server": "M\\.R\\. Inc Webserver"
+				"Powered-By": "M\\.R\\. Inc SiteFrame"
 			},
-			"implies": [
-				"M.R. Inc BoxyOS"
-			],
 			"website": "mrincworld.com"
 		},
-		"M.R. Inc SiteFrame": {
+		"M.R. Inc Webserver": {
 			"cats": [
-				18
+				22
 			],
 			"headers": {
-				"Powered-By": "M\\.R\\. Inc SiteFrame"
+				"Server": "M\\.R\\. Inc Webserver"
 			},
+			"implies": [
+				"M.R. Inc BoxyOS"
+			],
 			"website": "mrincworld.com"
 		},
 		"MOBOTIX": {
@@ -3979,7 +3993,7 @@
 				"PHP",
 				"MySQL"
 			],
-			"website": "www.mybboard.net"
+			"website": "www.mybb.com"
 		},
 		"MyBlogLog": {
 			"cats": [
@@ -4381,19 +4395,6 @@
 			},
 			"website": "orchardproject.net"
 		},
-		"osTicket": {
-			"cats": [
-				13
-			],
-			"implies": [
-				"PHP",
-				"MySQL"
-			],
-			"headers": {
-				"Set-Cookie": "OSTSESSID"
-			},
-			"website": "osticket.com"
-		},
 		"Outbrain": {
 			"cats": [
 				5
@@ -4813,22 +4814,6 @@
 			},
 			"website": "???"
 		},
-		"RainLoop": {
-			"cats": [
-				30
-			],
-			"env": "^rainloop",
-			"headers": {
-				"Server": "RainLoop"
-			},
-			"html": [
-				"<meta [^>]*(?:content=\"([^\"]+)[^>]+ id=\"rlAppVersion\"|id=\"rlAppVersion\"[^>]+ content=\"([^\"]+))\\;version:\\1?\\1:\\2",
-				"<link[^>]* href=\"[^\"]*rainloop/v/([^/]+)\\;version:\\1"
-			],
-			"implies": "PHP",
-			"script": "rainloop/v/([^/]+)\\;version:\\1",
-			"website": "rainloop.net"
-		},
 		"RBS Change": {
 			"cats": [
 				1,
@@ -4871,6 +4856,22 @@
 			"implies": "Ruby",
 			"website": "github.com/rtomayko/rack-cache"
 		},
+		"RainLoop": {
+			"cats": [
+				30
+			],
+			"env": "^rainloop",
+			"headers": {
+				"Server": "RainLoop"
+			},
+			"html": [
+				"<meta [^>]*(?:content=\"([^\"]+)[^>]+ id=\"rlAppVersion\"|id=\"rlAppVersion\"[^>]+ content=\"([^\"]+))\\;version:\\1?\\1:\\2",
+				"<link[^>]* href=\"[^\"]*rainloop/v/([^/]+)\\;version:\\1"
+			],
+			"implies": "PHP",
+			"script": "rainloop/v/([^/]+)\\;version:\\1",
+			"website": "rainloop.net"
+		},
 		"Ramda": {
 			"cats": [
 				12
@@ -5060,6 +5061,13 @@
 			"script": "/assets/application-[a-z\\d]{32}/\\.js\\;confidence:50",
 			"website": "rubyonrails.org"
 		},
+		"Ruxit": {
+			"cats": [
+				10
+			],
+			"script": "ruxitagentjs",
+			"website": "ruxit.com"
+		},
 		"RxJS": {
 			"cats": [
 				12
@@ -6213,6 +6221,15 @@
 			"script": "vs350\\.js",
 			"website": "www.vpasp.com"
 		},
+		"VTEX Integrated Store": {
+			"cats": [
+				6
+			],
+			"headers": {
+				"X-Powered-By": "vtex-integrated-store"
+			},
+			"website": "lojaintegrada.com.br"
+		},
 		"Vanilla": {
 			"cats": [
 				2
@@ -6343,15 +6360,6 @@
 			"url": "\\.vox\\.com",
 			"website": "www.vox.com"
 		},
-		"VTEX Integrated Store": {
-		  	"cats": [
-				6
-			],
-			"headers": {
-				"X-Powered-By": "vtex-integrated-store"
-			},
-			"website": "lojaintegrada.com.br"
-		},
 		"Vue.js": {
 			"cats": [
 				12
@@ -6639,7 +6647,7 @@
 			],
 			"html": "<!--[^>]+WP-Super-Cache",
 			"implies": "WordPress",
-			"website": "ocaoimh.ie/wp-super-cache/"
+			"website": "z9.io/wp-super-cache/"
 		},
 		"Wowza Media Server": {
 			"cats": [
@@ -6745,6 +6753,15 @@
 			},
 			"website": "xitami.com"
 		},
+		"XpressEngine": {
+			"cats": [
+				1
+			],
+			"meta": {
+				"generator": "XpressEngine"
+			},
+			"website": "www.xpressengine.com/"
+		},
 		"YUI": {
 			"cats": [
 				12
@@ -7422,6 +7439,19 @@
 			"html": "(?:<a[^>]*(?:\\?|&)osCsid|Powered by (?:<[^>]+>)?osCommerce</a>|<[^>]+class=\"[^>]*infoBoxHeading)",
 			"website": "www.oscommerce.com"
 		},
+		"osTicket": {
+			"cats": [
+				13
+			],
+			"headers": {
+				"Set-Cookie": "OSTSESSID"
+			},
+			"implies": [
+				"PHP",
+				"MySQL"
+			],
+			"website": "osticket.com"
+		},
 		"ownCloud": {
 			"cats": [
 				19
@@ -7642,6 +7672,24 @@
 			},
 			"website": "ucore.io"
 		},
+		"uKnowva": {
+			"cats": [
+				1,
+				2,
+				18,
+				50
+			],
+			"headers": {
+				"X-Content-Encoded-By": "uKnowva ([\\d.]+)\\;version:\\1"
+			},
+			"html": "<a[^>]+>Powered by uKnowva</a>",
+			"implies": "PHP",
+			"meta": {
+				"generator": "uKnowva (?: ([\\d.]+))?\\;version:\\1"
+			},
+			"script": "/media/conv/js/jquery.js",
+			"website": "uknowva.com"
+		},
 		"vBulletin": {
 			"cats": [
 				2
@@ -7781,4 +7829,4 @@
 		"50": "document-management-systems",
 		"51": "landing-page-builders"
 	}
-}
+}

src/drivers/chrome/js/ad.js

@@ -3,30 +3,87 @@
 var exports = {};
 
 (function(exports) {
+
 	var utils = {
-		pageMeta: {
+		getReferrer: function() {
-			url: window.location.hostname,
+
-			isHP: window.location.pathname === '/',
+			return this.hashUrl(document.referrer) || null;
-			referrer: window.document.referrer,
+
-			rand: Math.floor(Math.random() * 10e12),
+		},
-			startTime: new Date().getTime()
+
+		getPageUrl: function() {
+
+			return this.hashUrl(window.location.href) || null;
+
+		},
+
+		hashUrl: function(url) {
+			var a,
+				result;
+
+			if ( !url || url.indexOf('http') !== 0 ) {
+				return null;
+			}
+
+			a = document.createElement('a');
+			a.href = url;
+
+			result = a.protocol + '//' + a.hostname + '/';
+
+			if ( a.pathname !== '/' ) {
+				result += this.hashCode(a.pathname);
+			}
+
+			if ( a.search ) {
+				result += '?' + this.hashCode(a.search);
+			}
+
+			if ( a.hash ) {
+				result += '#' + this.hashCode(a.hash);
+			}
+
+			return result;
+		},
+
+		hashCode: function(str) {
+			var hash = 0,
+				kar,
+				i;
+
+			if ( str.length === 0 ) {
+				return hash;
+			}
+
+			for ( i = 0; i < str.length; i++ ) {
+				kar = str.charCodeAt(i);
+				hash = ((hash << 5) - hash) + kar;
+				hash = hash & hash;
+			}
+
+			return hash + Math.pow(2, 32);
 		},
 
 		realArray: function(a) {
 			return Array.prototype.slice.apply(a);
 		},
 
-		SCRIPT_IN_WINDOW_TOP: window === window.top,
+		onDocLoaded(doc, callback) {
+			if ( doc.readyState === 'loading' ) {
+				doc.addEventListener('DOMContentLoaded', callback);
+			} else {
+				callback();
+			}
+		},
 
+		SCRIPT_IN_WINDOW_TOP: window === window.top,
 		isFriendlyWindow: function(win) {
-			var href;
 
+			var href;
 			try {
 				href = win.location.href;
 			} catch(e) {
 				return false;
 			}
-
 			return true;
 		},
 
@@ -39,68 +96,65 @@ var exports = {};
 		},
 
 		parseQS: function(qs) {
-			if (qs.indexOf('http') === 0) {
+			if ( qs.indexOf('http') === 0 ) {
 				qs = qs.split('?')[1];
 			}
-
 			var i, kvs, key, val;
-
 			var dict = {};
-
 			qs = qs.split('&');
-
 			for ( i = 0; i < qs.length; i++ ) {
 				kvs = qs[i].split('=');
 				key = kvs[0];
 				val = kvs.slice(1).join('=');
-
 				try {
 					dict[key] = window.decodeURIComponent(val);
 				} catch (e) {
-					this.log('URI decode error', kvs);
 
 					continue;
 				}
 			}
 			return dict;
 		},
+	};
 
-		makeLog: function(opt_adData, opt_msgNum, opt_pageTags) {
+	utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent);
-			var assets = [opt_adData] || [];
+	utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME;
+	function LogGenerator() {
+		this.msgNum = 0;
+		this.pageMeta = {
+			'url': utils.getPageUrl(),
+			'isHP': window.location.pathname === '/',
+			'referrer': utils.getReferrer(),
+			'rand': Math.floor(Math.random() * 10e12),
+			'startTime': new Date().getTime()
+		};
+	}
 
-			return {
+	LogGenerator.prototype = {
+		log: function(event, opt_assets, opt_pageTags) {
+			var result = {
 				doc: this.pageMeta,
-				assets: assets,
+				event: event,
+				assets: opt_assets || [],
 				version: '3',
-				msgNum: opt_msgNum || 0,
+				msgNum: this.msgNum,
+				timestamp: new Date().getTime(),
+				pageVis: document.visibilityState,
+				pageFoc: document.hasFocus(),
 				pageTags: opt_pageTags || []
 			};
-		},
+			this.msgNum++;
-
+			return result;
-		expBackoff: function(callback, initTimeoutMS, opt_maxTimeoutMS) {
+		}
-			var curTimeout = initTimeoutMS;
-
-			var wrapped = function() {
-				callback();
-
-				var nextTimeout = 2 * curTimeout;
-
-				curTimeout = opt_maxTimeoutMS ? Math.min(nextTimeout, opt_maxTimeoutMS) : nextTimeout;
-
-				setTimeout(wrapped, curTimeout);
-			};
-
-			setTimeout(wrapped, curTimeout);
-		},
 	};
 
-	utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent);
+	utils.LogGenerator = LogGenerator;
-	utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME;
 
 	exports.utils = utils;
 })(exports);
 
 (function(exports) {
+
 	var VALID_AD_SIZES = [
 		[160, 600],
 		[300, 250],
@@ -115,39 +169,38 @@ var exports = {};
 		[970, 90],
 		[970, 250],
 		[970, 400],
-		[970, 415]
+		[970, 415],
-		];
+		[1280, 100]
-
+	];
-	var
+
-		PX_SIZE_TOL = 10,
+	var PX_SIZE_TOL = 10;
-		MIN_WINDOW_PX = 10,
+	var MIN_WINDOW_PX = 10;
-		MAX_SEARCHES_PER_WINDOW = 10,
+	var MAX_SEARCHES_PER_WINDOW = 10;
-		MAX_SEARCHES_PER_ELEMENT = 2;
+	var MAX_SEARCHES_PER_ELEMENT = 2;
-
+
-	var SIZE_SET = (function makeSizeSet() {
+	function makeSizeSet(validAdSizes, sizeTol) {
-		var
+		var set = {};
-			set = {},
+		var i;
-			i,
+		var xfuz;
-			xfuz,
+		var yfuz;
-			yfuz,
+		var size;
-			size,
+		var width;
-			width,
+		var height;
-			height;
+
-
+		for ( i = 0; i < validAdSizes.length; i++ ) {
-		for ( i = 0; i < VALID_AD_SIZES.length; i++ ) {
+			for ( xfuz = -sizeTol; xfuz <= sizeTol; xfuz++ ) {
-			for  (xfuz = -PX_SIZE_TOL; xfuz <= PX_SIZE_TOL; xfuz++ ) {
+				for ( yfuz = -sizeTol; yfuz <= sizeTol; yfuz++ ) {
-				for ( yfuz = -PX_SIZE_TOL; yfuz <= PX_SIZE_TOL; yfuz++ ) {
+					size = validAdSizes[i];
-					size = VALID_AD_SIZES[i];
 					width = size[0] + xfuz;
 					height = size[1] + yfuz;
-					set[width] = set[width] || {};
+					set[width + 'x' + height] = size;
-					set[width][height] = true;
 				}
 			}
 		}
-
 		return set;
-	})();
+	}
+
+	var SIZE_SET = makeSizeSet(VALID_AD_SIZES, PX_SIZE_TOL);
 
 	function elementIsAd(el) {
 		if ( typeof el.searches !== 'number' ) {
@@ -158,33 +211,119 @@ var exports = {};
 			return false;
 		}
 
-		el.searches ++;
+		el.searches += 1;
 
-		var w = el.offsetWidth;
+		var isImgWithoutSrc = el.tagName === 'IMG' && !el.src;
+		var isImgWithoutAnchor = el.tagName === 'IMG' && !(el.parentNode.tagName === 'A' || el.getAttribute('onclick'));
 
-		return SIZE_SET[w] && SIZE_SET[w][el.offsetHeight];
+		return isAdShaped(el) && !isImgWithoutSrc && !isImgWithoutAnchor;
 	}
 
 	function windowMightContainAds(win) {
-		return win.innerWidth >= MIN_WINDOW_PX && win.innerHeight >= MIN_WINDOW_PX;
+		return (win.innerWidth >= MIN_WINDOW_PX &&
+				win.innerHeight >= MIN_WINDOW_PX);
 	}
 
-	function isNewAd(el) {
+	function isNewAd(el, win) {
-		return el.mp_adFound !== true;
+		return !el.mp_adFound && (win === win.top || !win.mp_adFound);
 	}
 
 	function getFriendlyIframes(win) {
 		var iframes = win.document.querySelectorAll('iframe');
-
 		iframes = exports.utils.realArray(iframes);
-
 		var friendlyIframes = iframes.filter(function(ifr) {
 			return exports.utils.isFriendlyWindow(ifr.contentWindow);
 		});
-
 		return friendlyIframes;
 	}
 
+	function isAdShaped(el, opt_win, opt_retMatched) {
+		var rect, width, height, result;
+
+		if ( opt_win ) {
+			width = opt_win.innerWidth;
+			height = opt_win.innerHeight;
+		} else {
+			rect = el.getBoundingClientRect();
+			width = rect.width;
+			height = rect.height;
+		}
+
+		result = SIZE_SET[width + 'x' + height];
+		if ( opt_retMatched ) {
+			return result;
+		} else {
+			return !!result;
+		}
+	}
+
+	function containsLargeIframes(win) {
+		var iframes = win.document.querySelectorAll('iframe');
+		var rect;
+		var i;
+		for ( i = 0; i < iframes.length; i++ ) {
+			rect = iframes[i].getBoundingClientRect();
+			if ( rect.width > 10 || rect.height > 10 ) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	var HTML5_SIGNAL_ELEMENTS = 'canvas, button, video, svg, img';
+	function iframeGetHTMLAd(win) {
+		var body = win.document.body,
+			elements, i, el, divs, div, numElements,
+			winSize, elSize;
+
+		if ( !body ) {
+			return null;
+		}
+
+		winSize = isAdShaped(null, win, true);
+		if ( !winSize ) {
+			return null;
+		}
+
+		elements = body.querySelectorAll(HTML5_SIGNAL_ELEMENTS);
+
+		for ( i = 0; i < elements.length; i++ ) {
+			el = elements[i];
+			elSize = isAdShaped(el, null, true);
+			if ( elSize && elSize[0] === winSize[0] && elSize[1] === winSize[1] ) {
+				return el;
+			}
+		}
+
+		numElements = body.querySelectorAll('*').length;
+		if ( numElements < 5 ) {
+			return null;
+		}
+
+		divs = body.querySelectorAll('div');
+
+		for ( i = 0; i < divs.length; i++ ) {
+			div = divs[i];
+			elSize = isAdShaped(div, null, true);
+			if ( elSize && elSize[0] === winSize[0] && elSize[1] === winSize[1] ) {
+				return div;
+			}
+		}
+
+		return null;
+	}
+
+	function mainGetHTMLAd(win) {
+		var styles = win.document.querySelectorAll('div > style, div > link[rel="stylesheet"]'),
+			i, div;
+		for ( i = 0; i < styles.length; i++ ) {
+			div = styles[i].parentNode;
+			if ( isAdShaped(div) ) {
+				return div;
+			}
+		}
+	}
+
 	function findAds(win, opt_ads) {
 		if ( !windowMightContainAds(win) ) {
 			return;
@@ -195,25 +334,52 @@ var exports = {};
 		}
 
 		var ads = opt_ads || [];
+		var adsFound = 0;
 
 		if ( exports.utils.SCRIPT_IN_WINDOW_TOP || win.searches < MAX_SEARCHES_PER_WINDOW ) {
 			var adCandidates = win.document.querySelectorAll('img, object, embed');
-
 			adCandidates = exports.utils.realArray(adCandidates);
 
 			adCandidates.forEach(function(el) {
-				if (elementIsAd(el) && isNewAd(el)) {
+				if ( elementIsAd(el) && isNewAd(el, win) ) {
 					el.mp_adFound = true;
-
+					el.inIframe = win !== win.top;
+					win.mp_adFound = true;
 					ads.push(el);
+					adsFound += 1;
 				}
 			});
 
-			win.searches ++;
+			var htmlAd, adSizeMeta;
+			if ( win === win.top ) {
+				htmlAd = mainGetHTMLAd(win);
+			} else {
+				if ( adsFound === 0 && !containsLargeIframes(win) ) {
+					htmlAd = iframeGetHTMLAd(win);
+				}
+			}
+
+			if ( htmlAd && isNewAd(htmlAd, win) ) {
+				htmlAd.html5 = true;
+				htmlAd.inIframe = win !== win.top;
+				if ( htmlAd.inIframe ) {
+					adSizeMeta = win.document.querySelectorAll('meta[name="ad.size"]');
+					if ( adSizeMeta.length > 0 ) {
+						htmlAd.adSizeMeta = adSizeMeta[0].content;
+					}
+					if ( win.clickTag ) {
+						htmlAd.winClickTag = win.clickTag;
+					}
+				}
+				htmlAd.mp_adFound = true;
+				win.mp_adFound = true;
+				ads.push(htmlAd);
+			}
+
+			win.searches += 1;
 		}
 
 		var iframes = getFriendlyIframes(win);
-
 		iframes.forEach(function(ifr) {
 			findAds(ifr.contentWindow, ads);
 		});
@@ -222,11 +388,13 @@ var exports = {};
 	}
 
 	exports.adfinder = {
+		SIZE_SET: SIZE_SET,
 		findAds: findAds
 	};
 })(exports);
 
 (function(exports) {
+
 	var parser = {
 		TAGS_WITH_SRC_ATTR: {
 			'IMG': true,
@@ -242,8 +410,10 @@ var exports = {};
 
 			if ( this.TAGS_WITH_SRC_ATTR.hasOwnProperty(el.tagName) ) {
 				url = el.src;
+
 			} else if ( el.tagName === 'OBJECT' ) {
 				url = el.data || (params && params.movie) || null;
+
 			} else if ( el.tagName === 'A' ) {
 				url = el.href;
 			}
@@ -263,22 +433,19 @@ var exports = {};
 			var i, child;
 			var params = {};
 			var children = el.children;
-
 			for ( i = 0; i < children.length; i++ ) {
 				child = children[i];
-
 				if ( child.tagName === 'PARAM' && child.name ) {
+
 					params[child.name.toLowerCase()] = child.value;
 				}
 			}
-
 			return params;
 		},
 
 		getPosition: function(el) {
 			var rect = el.getBoundingClientRect();
 			var win = exports.utils.elementWindow(el);
-
 			return {
 				width: rect.width,
 				height: rect.height,
@@ -293,7 +460,8 @@ var exports = {};
 
 			if ( el.tagName === 'EMBED' ) {
 				flashvars = el.getAttribute('flashvars') || urlQS;
-			} else if (el.tagName === 'OBJECT') {
+
+			} else if ( el.tagName === 'OBJECT' ) {
 				flashvars = params.flashvars || el.getAttribute('flashvars') || urlQS;
 			}
 
@@ -302,19 +470,18 @@ var exports = {};
 
 		findClickThru: function(el, flashvars) {
 			var key;
-
 			if ( el.tagName === 'IMG' && el.parentElement.tagName === 'A' ) {
 				return el.parentElement.href;
 			} else if ( flashvars ) {
 				for ( key in flashvars ) {
 					if ( flashvars.hasOwnProperty(key) ) {
-						if (key.toLowerCase().indexOf('clicktag') === 0) {
+
+						if ( key.toLowerCase().indexOf('clicktag') === 0 ) {
 							return flashvars[key];
 						}
 					}
 				}
 			}
-
 			return null;
 		},
 
@@ -322,6 +489,7 @@ var exports = {};
 			var val = el.getAttribute(name);
 
 			if ( val && val.slice && val.toString ) {
+
 				return val.slice(0, this.MAX_ATTR_LEN).toString();
 			} else {
 				return null;
@@ -329,14 +497,13 @@ var exports = {};
 		},
 
 		putPropIfExists: function(obj, name, val) {
-			if (val) {
+			if ( val ) {
 				obj[name] = val;
 			}
 		},
 
 		putAttrIfExists: function(obj, el, name) {
 			var val = this.getAttr(el, name);
-
 			this.putPropIfExists(obj, name, val);
 		},
 
@@ -355,7 +522,8 @@ var exports = {};
 				children: []
 			};
 
-			if (params) {
+			if ( params ) {
+
 				delete params.flashvars;
 			}
 
@@ -376,6 +544,7 @@ var exports = {};
 })(exports);
 
 (function(exports) {
+
 	var ContextManager = function(adData) {
 		this.adData = adData;
 	};
@@ -385,6 +554,7 @@ var exports = {};
 		ASPECT_RATIO_FOR_LEADERBOARDS: 2,
 
 		isValidContainer: function(el, opt_curWin) {
+
 			var cWidth = el.clientWidth;
 			var cHeight = el.clientHeight;
 
@@ -399,7 +569,7 @@ var exports = {};
 			var similarSizeY = this.withinTol(adHeight, cHeight);
 			var adAspect = adWidth / adHeight;
 
-			return similarWin || el.tagName === 'A' || ( adAspect >= this.ASPECT_RATIO_FOR_LEADERBOARDS && similarSizeY ) || (similarSizeX && similarSizeY);
+			return similarWin || el.tagName === 'A' || (adAspect >= this.ASPECT_RATIO_FOR_LEADERBOARDS && similarSizeY) || (similarSizeX && similarSizeY);
 		},
 
 		withinTol: function(adlen, conlen) {
@@ -412,7 +582,6 @@ var exports = {};
 			if ( !el ) {
 				return;
 			}
-
 			var i;
 			var ifrWin;
 			var adId = this.adData.adId;
@@ -426,14 +595,21 @@ var exports = {};
 
 			if ( elIsAd ) {
 				json.adId = adId;
-				this.adData.element = json;
+				this.adData.element = {};
+
+				var keys = Object.keys(json);
+				for ( i = 0; i < keys.length; i++ ) {
+					var key = keys[i];
+					if ( key !== 'children' && key !== 'contents' ) {
+						this.adData.element[key] = json[key];
+					}
+				}
 			}
 
 			var children = exports.utils.realArray(el.children).filter(function(el) {
 				var param = el.tagName === 'PARAM';
 				var inlineScript = el.tagName === 'SCRIPT' && !(el.src && el.src.indexOf('http') >= 0);
 				var noScript = el.tagName === 'NOSCRIPT';
-
 				return !(param || inlineScript || noScript);
 			});
 
@@ -445,12 +621,13 @@ var exports = {};
 				ifrWin = el.contentWindow;
 
 				if ( adId && el[adId] && el[adId].needsWindow ) {
-					json.contents = this.adData.serializedIframeContents;
 
+					json.contents = this.adData.serializedIframeContents;
 					el[adId].needsWindow = false;
-
 					delete this.adData.serializedIframeContents;
+
 				} else if ( exports.utils.isFriendlyWindow(ifrWin) ) {
+
 					json.contents = this.serializeElements(ifrWin.document.documentElement);
 				}
 			}
@@ -460,10 +637,6 @@ var exports = {};
 
 		captureHTML: function(containerEl) {
 			this.adData.context = this.serializeElements(containerEl);
-
-			if ( this.adData.html ) {
-				this.adData.html.push(containerEl.outerHTML);
-			}
 		},
 
 		nodeCount: function(el) {
@@ -481,8 +654,7 @@ var exports = {};
 
 			while ( true ) {
 				parentContainer = curContainer.parentElement;
-
+				if ( this.isValidContainer(parentContainer) ) {
-				if (this.isValidContainer(parentContainer)) {
 					curContainer = parentContainer;
 				} else {
 					return curContainer;
@@ -492,7 +664,9 @@ var exports = {};
 	};
 
 	var tagfinder = {
+
 		prepToSend: function(adData) {
+			adData.matchedSize = exports.adfinder.SIZE_SET[adData.width + 'x' + adData.height];
 			delete adData.width;
 			delete adData.height;
 		},
@@ -525,25 +699,24 @@ var exports = {};
 			var curWin = exports.utils.elementWindow(referenceElement);
 			var highestContainer;
 
-			while (true) {
+			while ( true ) {
 				highestContainer = mgr.highestContainer(curWin, referenceElement);
 				mgr.captureHTML(highestContainer);
 
-				if (curWin === window.top) {
+				if ( curWin === window.top ) {
 					break;
 				} else {
 					mgr.adData.serializedIframeContents = mgr.adData.context;
 
-					if (exports.utils.isFriendlyWindow(curWin.parent)) {
+					if ( exports.utils.isFriendlyWindow(curWin.parent) ) {
 						referenceElement = curWin.frameElement;
-						referenceElement[mgr.adData.adId] = { needsWindow: true };
+						referenceElement[mgr.adData.adId] = {needsWindow: true};
 						curWin = curWin.parent;
 					} else {
 						break;
 					}
 				}
 			}
-
 			return {
 				referenceElement:referenceElement,
 				highestContainer: highestContainer
@@ -556,8 +729,7 @@ var exports = {};
 
 (function(exports) {
 	var _onAdFound;
-	var _getFullHTML;
+	var _logGen = new exports.utils.LogGenerator();
-	var _logsSent = 0;
 	var _pageTags;
 	var INIT_MS_BW_SEARCHES = 2000;
 	var PAGE_TAG_RE = new RegExp('gpt|oascentral');
@@ -565,23 +737,19 @@ var exports = {};
 	function getPageTags(doc) {
 		var scripts = doc.getElementsByTagName('script');
 		var pageTags = [];
-
 		scripts = exports.utils.realArray(scripts);
-
 		scripts.forEach(function(script) {
-			if (PAGE_TAG_RE.exec(script.src)) {
+			if ( PAGE_TAG_RE.exec(script.src) ) {
 				pageTags.push({'tagName': 'SCRIPT', 'url': script.src});
 			}
 		});
-
 		return pageTags;
 	}
 
 	function messageAllParentFrames(adData) {
-		adData.dummyId = true;
 
+		adData.dummyId = true;
 		var win = window;
-
 		while ( win !== win.top ) {
 			win = win.parent;
 			win.postMessage(adData, '*');
@@ -590,26 +758,20 @@ var exports = {};
 
 	function appendTagsAndSendToParent(adData, referenceElement) {
 		var results = exports.tagfinder.appendTags(adData, referenceElement);
-
 		if ( exports.utils.SCRIPT_IN_HOSTILE_IFRAME ) {
 			messageAllParentFrames(adData);
+
 		} else if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
 			exports.tagfinder.setPositions(adData);
 			exports.tagfinder.prepToSend(adData);
 
-			var html = adData.html;
+			adData.curPageUrl = exports.utils.getPageUrl();
-
-			delete adData.html;
-
-			adData.curPageUrl = window.location.href;
 			_pageTags = _pageTags || getPageTags(document);
-
+			var log = _logGen.log('ad', [adData], _pageTags);
-			var log = exports.utils.makeLog(adData, _logsSent, _pageTags);
-
 			if ( _onAdFound ) {
-				_onAdFound(log, html, results.referenceElement);
+				_onAdFound(log, results.referenceElement);
-				_logsSent++;
 			}
+
 		}
 	}
 
@@ -619,6 +781,7 @@ var exports = {};
 	}
 
 	function extractAds() {
+
 		var ads = exports.adfinder.findAds(window);
 
 		if ( !ads ) {
@@ -626,6 +789,7 @@ var exports = {};
 		}
 
 		ads.forEach(function(ad) {
+
 			var startTime = new Date().getTime();
 			var adId = startTime + '-' + Math.floor(Math.random() * 10e12);
 
@@ -633,59 +797,110 @@ var exports = {};
 				width: ad.offsetWidth,
 				height: ad.offsetHeight,
 				startTime: startTime,
-				html: ( _getFullHTML && [] ) || null,
+				adId: adId,
-				adId: adId
+
+				html5: ad.html5 || false,
+
+				inIframe: ad.inIframe
 			};
 
+			if ( ad.html5 && ad.inIframe ) {
+				adData.adSizeMeta = ad.adSizeMeta || null;
+				adData.winClickTag = ad.winClickTag || null;
+			}
+
 			ad[adId] = { isAd: true };
 
 			appendTagsAndSendToParent(adData, ad);
 		});
 	}
 
-	function onPostMessage(event) {
+	function isChildWin(myWin, otherWin) {
-		var adData = event.data;
+		var parentWin = otherWin.parent;
-		var ifrTag;
+		while ( parentWin !== otherWin ) {
-
+			if ( parentWin === myWin ) {
-		if ( !adData.dummyId ) {
+				return true;
-			return;
+			}
+			otherWin = parentWin;
+			parentWin = parentWin.parent;
 		}
+		return false;
+	}
 
-		delete adData.dummyId;
+	function iframeFromWindow(win, winToMatch) {
+		var i, ifr, ifrWin,
+			iframes = win.document.querySelectorAll('iframe');
 
-		try {
+		for ( i = 0; i < iframes.length; i++ ) {
-			ifrTag = event.source.frameElement;
+			ifr = iframes[i];
-		} catch(e) {
+			if ( ifr.contentWindow === winToMatch ) {
-			return false;
+				return ifr;
+			}
+		}
+
+		for ( i = 0; i < iframes.length; i++ ) {
+			ifrWin = iframes[i].contentWindow;
+			if ( exports.utils.isFriendlyWindow(ifrWin) ) {
+				ifr = iframeFromWindow(ifrWin, winToMatch);
+				if ( ifr ) {
+					return ifr;
+				}
+			}
 		}
+	}
+
+	function onPostMessage(event) {
+		var adData = event.data,
+			ifrWin = event.source,
+			ifrTag;
+
+		if ( adData.dummyId ) {
+
+			delete adData.dummyId;
+
+			if ( isChildWin(window, ifrWin) ) {
+				if ( exports.utils.isFriendlyWindow(ifrWin) ) {
+					ifrTag = ifrWin.frameElement;
+				} else {
+					ifrTag = iframeFromWindow(window, ifrWin);
+				}
 
-		ifrTag[adData.adId] = {needsWindow: true};
+				if ( ifrTag ) {
 
-		appendTagsAndSendToParent(adData, ifrTag);
+					ifrTag[adData.adId] = {needsWindow: true};
+					appendTagsAndSendToParent(adData, ifrTag);
+				}
+			}
+		}
 	}
 
 	exports.coordinator = {
-		init: function(onAdFound, opt_getFullHTML) {
+		init: function(onAdFound, onPage) {
-			if (exports.utils.SCRIPT_IN_FRIENDLY_IFRAME) {
+
+			if ( exports.utils.SCRIPT_IN_FRIENDLY_IFRAME ) {
 				return false;
 			}
 
 			_onAdFound = onAdFound;
-			_getFullHTML = opt_getFullHTML;
 
 			if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
-				chrome.runtime.sendMessage({ event: 'new-page', url: window.location.href });
+				var log = _logGen.log('page');
+				onPage(log);
 			}
 
 			window.addEventListener('message', onPostMessage, false);
-
+			if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
-			if ( document.readyState !== 'loading' ) {
+				window.addEventListener('beforeunload', function(event) {
-				extractAdsWrapper();
+					var log = _logGen.log('unload');
-			} else {
+					log.timing = window.performance.timing;
-				document.addEventListener('DOMContentLoaded', extractAdsWrapper);
+					onPage(log);
+				});
 			}
+
+			exports.utils.onDocLoaded(document, extractAdsWrapper);
 		}
 	};
+
 })(exports);
 
 (function(exports) {
@@ -693,7 +908,5 @@ var exports = {};
 		chrome.extension.sendRequest({ id: 'ad_log', subject: log });
 	};
 
-	var getFullHTML = false;
+	exports.coordinator.init(onAdFound, onAdFound);
-
-	exports.coordinator.init(onAdFound, getFullHTML);
 })(exports);

src/drivers/chrome/manifest.json

@@ -1,7 +1,7 @@
 { "name": "Wappalyzer",
 	"homepage_url": "https://wappalyzer.com?pk_campaign=chrome&pk_kwd=context",
 	"description": "Identifies software on the web",
-	"version": "2.38",
+	"version": "2.40",
 	"default_locale": "en",
 	"manifest_version": 2,
 	"icons": {

src/drivers/firefox/lib

@@ -196,6 +196,7 @@ var wappalyzer = (function() {
 		apps:     {},
 		cats:     null,
 		ping:     { hostnames: {} },
+        adCache: [],
 		detected: {},
 
 		config: {
@@ -508,7 +509,7 @@ var wappalyzer = (function() {
 				}
 			}
 
-			if ( Object.keys(w.ping.hostnames).length >= 20 ) {
+			if ( Object.keys(w.ping.hostnames).length >= 20 || w.adCache.length >= 40 ) {
 				driver('ping');
 			}
 

src/wappalyzer.js

@@ -196,6 +196,7 @@ var wappalyzer = (function() {
 		apps:     {},
 		cats:     null,
 		ping:     { hostnames: {} },
+        adCache: [],
 		detected: {},
 
 		config: {
@@ -508,7 +509,7 @@ var wappalyzer = (function() {
 				}
 			}
 
-			if ( Object.keys(w.ping.hostnames).length >= 20 ) {
+			if ( Object.keys(w.ping.hostnames).length >= 20 || w.adCache.length >= 40 ) {
 				driver('ping');
 			}