From 1cca98b405a65479a8407854ce2a87ad8cbeca99 Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Sat, 23 Mar 2019 15:17:38 +1100 Subject: [PATCH] Build v5.7.4 --- src/apps.json | 11 +- src/drivers/npm/npm-shrinkwrap.json | 2 +- src/drivers/npm/package.json | 2 +- .../webextension/_locales/de/messages.json | 4 +- .../webextension/_locales/el/messages.json | 4 +- .../webextension/_locales/es/messages.json | 4 +- .../webextension/_locales/fa/messages.json | 4 +- .../webextension/_locales/fr/messages.json | 4 +- .../webextension/_locales/gr/messages.json | 4 +- .../webextension/_locales/id/messages.json | 4 +- .../webextension/_locales/it/messages.json | 4 +- .../webextension/_locales/pl/messages.json | 4 +- .../webextension/_locales/pt/messages.json | 4 +- .../webextension/_locales/pt_BR/messages.json | 4 +- .../webextension/_locales/ro/messages.json | 4 +- .../webextension/_locales/ru/messages.json | 4 +- .../webextension/_locales/sk/messages.json | 4 +- .../webextension/_locales/tr/messages.json | 4 +- .../webextension/_locales/uk/messages.json | 4 +- .../webextension/_locales/uz/messages.json | 4 +- .../webextension/_locales/zh_CN/messages.json | 4 +- .../webextension/_locales/zh_TW/messages.json | 4 +- src/drivers/webextension/js/lib/iframe.js | 2321 +++++++++-------- src/drivers/webextension/js/lib/network.js | 1616 ++++++------ src/drivers/webextension/manifest.json | 3 +- 25 files changed, 2068 insertions(+), 1963 deletions(-) diff --git a/src/apps.json b/src/apps.json index 4c95d0fce..cf1c1022c 100644 --- a/src/apps.json +++ b/src/apps.json @@ -1139,7 +1139,7 @@ 6 ], "headers": { - "X-SERVER": "BIGBANGSHOP*\w+" + "X-SERVER": "BIGBANGSHOP" }, "icon": "bigbangshop.svg", "website": "https://www.bigbangshop.com.br" @@ -5738,7 +5738,7 @@ "website": "http://liveinternet.ru/rating/" }, "LocalFocus": { - "cats": [ + "cats": [ 61 ], "html": "]+localfocus", @@ -11698,7 +11698,6 @@ "icon": "YouTube.png", "website": "http://www.youtube.com" }, - "iEXExchanger": { "cats": [ 1 @@ -11707,8 +11706,7 @@ "PHP", "Apache" ], - - "cookies": { + "cookies": { "iexexchanger_session": "" }, "meta": { @@ -11717,7 +11715,6 @@ "script": "iexexchanger", "website": "https://exchanger.iexbase.com" }, - "ZK": { "cats": [ 18 @@ -13379,4 +13376,4 @@ "priority": 7 } } -} \ No newline at end of file +} diff --git a/src/drivers/npm/npm-shrinkwrap.json b/src/drivers/npm/npm-shrinkwrap.json index 26c32eed6..43be434cc 100644 --- a/src/drivers/npm/npm-shrinkwrap.json +++ b/src/drivers/npm/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "wappalyzer", - "version": "5.7.3", + "version": "5.7.4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index ffbc1a38b..18fed21ea 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -2,7 +2,7 @@ "name": "wappalyzer", "description": "Uncovers the technologies used on websites", "homepage": "https://github.com/AliasIO/Wappalyzer", - "version": "5.7.3", + "version": "5.7.4", "author": "Elbert Alias", "license": "GPL-3.0", "repository": { diff --git a/src/drivers/webextension/_locales/de/messages.json b/src/drivers/webextension/_locales/de/messages.json index c8b72145d..a3735830c 100644 --- a/src/drivers/webextension/_locales/de/messages.json +++ b/src/drivers/webextension/_locales/de/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/el/messages.json b/src/drivers/webextension/_locales/el/messages.json index f336695de..0076ca754 100644 --- a/src/drivers/webextension/_locales/el/messages.json +++ b/src/drivers/webextension/_locales/el/messages.json @@ -71,5 +71,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/es/messages.json b/src/drivers/webextension/_locales/es/messages.json index 741b88c9a..134f044cb 100644 --- a/src/drivers/webextension/_locales/es/messages.json +++ b/src/drivers/webextension/_locales/es/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/fa/messages.json b/src/drivers/webextension/_locales/fa/messages.json index c5b190c30..51bd2a555 100644 --- a/src/drivers/webextension/_locales/fa/messages.json +++ b/src/drivers/webextension/_locales/fa/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/fr/messages.json b/src/drivers/webextension/_locales/fr/messages.json index a0f34b1ed..23dcfd19f 100644 --- a/src/drivers/webextension/_locales/fr/messages.json +++ b/src/drivers/webextension/_locales/fr/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/gr/messages.json b/src/drivers/webextension/_locales/gr/messages.json index 4ea87768e..bb73d8c94 100644 --- a/src/drivers/webextension/_locales/gr/messages.json +++ b/src/drivers/webextension/_locales/gr/messages.json @@ -71,5 +71,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/id/messages.json b/src/drivers/webextension/_locales/id/messages.json index e0b350859..1d5f7c2f3 100644 --- a/src/drivers/webextension/_locales/id/messages.json +++ b/src/drivers/webextension/_locales/id/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/it/messages.json b/src/drivers/webextension/_locales/it/messages.json index 7fba4c39d..171cc6c32 100644 --- a/src/drivers/webextension/_locales/it/messages.json +++ b/src/drivers/webextension/_locales/it/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/pl/messages.json b/src/drivers/webextension/_locales/pl/messages.json index e6eb2720f..2e5b0df13 100644 --- a/src/drivers/webextension/_locales/pl/messages.json +++ b/src/drivers/webextension/_locales/pl/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/pt/messages.json b/src/drivers/webextension/_locales/pt/messages.json index 21918a376..896b8f91f 100644 --- a/src/drivers/webextension/_locales/pt/messages.json +++ b/src/drivers/webextension/_locales/pt/messages.json @@ -76,5 +76,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/pt_BR/messages.json b/src/drivers/webextension/_locales/pt_BR/messages.json index 085701851..86402cf1e 100644 --- a/src/drivers/webextension/_locales/pt_BR/messages.json +++ b/src/drivers/webextension/_locales/pt_BR/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/ro/messages.json b/src/drivers/webextension/_locales/ro/messages.json index 09ef0b8b5..14e832699 100644 --- a/src/drivers/webextension/_locales/ro/messages.json +++ b/src/drivers/webextension/_locales/ro/messages.json @@ -71,5 +71,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/ru/messages.json b/src/drivers/webextension/_locales/ru/messages.json index 5e165a189..9438baef7 100644 --- a/src/drivers/webextension/_locales/ru/messages.json +++ b/src/drivers/webextension/_locales/ru/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/sk/messages.json b/src/drivers/webextension/_locales/sk/messages.json index 96eec610c..b9ee93169 100644 --- a/src/drivers/webextension/_locales/sk/messages.json +++ b/src/drivers/webextension/_locales/sk/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/tr/messages.json b/src/drivers/webextension/_locales/tr/messages.json index 1c886d971..f7a1c98ec 100644 --- a/src/drivers/webextension/_locales/tr/messages.json +++ b/src/drivers/webextension/_locales/tr/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/uk/messages.json b/src/drivers/webextension/_locales/uk/messages.json index 2f99dbb8c..5f6b8089d 100644 --- a/src/drivers/webextension/_locales/uk/messages.json +++ b/src/drivers/webextension/_locales/uk/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/uz/messages.json b/src/drivers/webextension/_locales/uz/messages.json index ae5e62cdb..edd7df369 100644 --- a/src/drivers/webextension/_locales/uz/messages.json +++ b/src/drivers/webextension/_locales/uz/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/zh_CN/messages.json b/src/drivers/webextension/_locales/zh_CN/messages.json index 86ba50ca1..de4afab07 100644 --- a/src/drivers/webextension/_locales/zh_CN/messages.json +++ b/src/drivers/webextension/_locales/zh_CN/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "Containers" }, "categoryName61": { "message": "SaaS" }, "categoryName62": { "message": "PaaS" }, - "categoryName63": { "message": "IaaS" } + "categoryName63": { "message": "IaaS" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/_locales/zh_TW/messages.json b/src/drivers/webextension/_locales/zh_TW/messages.json index de64b78fc..b1213d6ef 100644 --- a/src/drivers/webextension/_locales/zh_TW/messages.json +++ b/src/drivers/webextension/_locales/zh_TW/messages.json @@ -75,5 +75,7 @@ "categoryName60": { "message": "容器" }, "categoryName61": { "message": "軟體即服務（SaaS）" }, "categoryName62": { "message": "平台即服務（PaaS）" }, - "categoryName63": { "message": "基礎設施即服務（IaaS）" } + "categoryName63": { "message": "基礎設施即服務（IaaS）" }, + "categoryName64": { "message": "Reverse Proxy" }, + "categoryName65": { "message": "Load Balancer" } } diff --git a/src/drivers/webextension/js/lib/iframe.js b/src/drivers/webextension/js/lib/iframe.js index 491d26257..1c65cf074 100644 --- a/src/drivers/webextension/js/lib/iframe.js +++ b/src/drivers/webextension/js/lib/iframe.js @@ -1,1142 +1,1183 @@ +'use strict'; +(function(win) { -(function (win) { - const exports = {}; - - (function (exports) { - const utils = { - normalizeUrl(url) { - return this.hashUrl(url) || null; - }, - - getReferrer() { - return this.normalizeUrl(document.referrer); - }, - - getPageUrl() { - return this.normalizeUrl(window.location.href); - }, - hashUrl(url) { - let a, - result; - - if (!url || url.indexOf('http') !== 0) { - return null; - } - - a = document.createElement('a'); - a.href = url; - - result = `${a.protocol}//${a.hostname}/`; - - if (a.pathname && a.pathname !== '/') { - result += this.hashCode(a.pathname); - } - - if (a.search) { - result += `?${this.hashCode(a.search)}`; - } - - if (a.hash) { - result += `#${this.hashCode(a.hash)}`; - } - - return result; - }, - - hashCode(str) { - let hash = 0, - kar, - i; - - if (str.length === 0) { - return hash; - } - - for (i = 0; i < str.length; i++) { - kar = str.charCodeAt(i); - hash = ((hash << 5) - hash) + kar; - hash &= hash; - } - - return hash + Math.pow(2, 32); - }, - - realArray(a) { - return Array.prototype.slice.apply(a); - }, - - onDocLoaded(doc, callback) { - if (doc.readyState === 'loading') { - doc.addEventListener('DOMContentLoaded', callback); - } else { - callback(); - } - }, - - SCRIPT_IN_WINDOW_TOP: window === window.top, - - isFriendlyWindow(win) { - let href; - try { - href = win.location.href; - } catch (e) { - return false; - } - return true; - }, - - elementWindow(el) { - return el.ownerDocument.defaultView; - }, - - viewport(win) { - return { width: win.innerWidth, height: win.innerHeight }; - }, - - parseQS(qs) { - if (qs.indexOf('http') === 0) { - qs = qs.split('?')[1]; - } - let i, - kvs, - key, - val; - const dict = {}; - qs = qs.split('&'); - for (i = 0; i < qs.length; i++) { - kvs = qs[i].split('='); - key = kvs[0]; - val = kvs.slice(1).join('='); - try { - dict[key] = window.decodeURIComponent(val); - } catch (e) { - continue; - } - } - return dict; - }, - sendToBackground(message, event, responseMessage, onResponse) { - if (typeof browser !== 'undefined') { - const response = browser.runtime.sendMessage(message); - response.then(onResponse); - } else if (typeof chrome !== 'undefined') { - chrome.runtime.sendMessage(message, onResponse); - } else if (window.self.port) { - window.self.port.on(responseMessage, onResponse); - window.self.port.emit(event, message); - } - }, - - askIfTrackingEnabled(callback, elseCallback) { - this.sendToBackground( - 'is_tracking_enabled', - '', - 'tracking_enabled_response', - (message) => { - if (message && message.tracking_enabled) { - callback(); - } else { - elseCallback(); - } - } - ); - }, - }; - - utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent); - utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME; - - function LogGenerator() { - this.msgNum = 0; - this.pageMeta = { - url: utils.getPageUrl(), - isHP: window.location.pathname === '/', - referrer: utils.getReferrer(), - rand: Math.floor(Math.random() * 10e12), - startTime: new Date().getTime(), - }; - } - - LogGenerator.prototype = { - log(event, opt_assets, opt_pageTags) { - let opt_video_assets; - if (event === 'video' || event === 'invalid-video') { - opt_video_assets = opt_assets || []; - opt_assets = []; - } else { - opt_video_assets = []; - opt_assets = opt_assets || []; - } - const result = { - doc: this.pageMeta, - event, - video_assets: opt_video_assets, - assets: opt_assets, - version: '3', - mrev: 'fe20291-c', - msgNum: this.msgNum, - timestamp: new Date().getTime(), - pageVis: document.visibilityState, - pageFoc: document.hasFocus(), - pageTags: opt_pageTags || [], - }; - this.msgNum++; - return result; - }, - }; - - utils.LogGenerator = LogGenerator; - - exports.utils = utils; - }(exports)); - - (function (exports) { - const SizeMatcher = { - VALID_AD_SIZES: [ - [300, 50], - [320, 50], - [160, 600], - [300, 250], - [300, 600], - [300, 1050], - [336, 280], - [336, 850], - [468, 60], - [728, 90], - [728, 250], - [728, 270], - [970, 66], - [970, 90], - [970, 125], - [970, 250], - [970, 400], - [970, 415], - [1280, 100], - ], - - PX_SIZE_TOL: 10, - - getMatchedAdSize(width, height) { - if (!this.set) { - this.set = this._makeSizeSet(); - } - - return this.set[`${Math.round(width)}x${Math.round(height)}`]; - }, - - elementIsAdShaped(el) { - return !!this.getMatchedAdSizeForElement(el); - }, - - getMatchedAdSizeForElement(el) { - const rect = el.getBoundingClientRect(); - return this.getMatchedAdSize(rect.width, rect.height); - }, - - _makeSizeSet() { - const set = {}; - let i; - let xfuz; - let yfuz; - let size; - let width; - let height; - - for (i = 0; i < this.VALID_AD_SIZES.length; i++) { - for (xfuz = -this.PX_SIZE_TOL; xfuz <= this.PX_SIZE_TOL; xfuz++) { - for (yfuz = -this.PX_SIZE_TOL; yfuz <= this.PX_SIZE_TOL; yfuz++) { - size = this.VALID_AD_SIZES[i]; - width = size[0] + xfuz; - height = size[1] + yfuz; - set[`${width}x${height}`] = size; - } - } - } - return set; - }, - }; - - const Throttler = { - MAX_SEARCHES_PER_WINDOW: 10, - MAX_SEARCHES_PER_ELEMENT: 2, - - countSearch(el) { - if (typeof el.searches !== 'number') { - el.searches = 0; - } - - el.searches += 1; - }, - - throttle(el, max) { - if (typeof el.searches === 'number' && el.searches >= max) { - return true; - } - return false; - }, - - throttleElement(el) { - return this.throttle(el, this.MAX_SEARCHES_PER_ELEMENT); - }, - - throttleWin(win) { - return this.throttle(win, this.MAX_SEARCHES_PER_WINDOW); - }, - - getCount(el) { - return el.searches || 0; - }, - }; - - function TopSearcher(win) { - this.win = win; - this.doc = win.document; - } - - TopSearcher.prototype.search = function () { - let candidates = exports.utils.realArray(this.doc.querySelectorAll('img, object, embed')), - html5Ad, - ads = []; - - ads = ads.concat(candidates.filter((el) => { - if (!el.mpAdFound && !Throttler.throttleElement(el)) { - Throttler.countSearch(el); - if ((el.tagName !== 'IMG' || isStandardImage(el)) && SizeMatcher.elementIsAdShaped(el)) { - el.mpAdFound = true; - return true; - } - } - return false; - })); - - html5Ad = this._mainGetHTMLAd(); - if (html5Ad) { - html5Ad.html5 = true; - html5Ad.mpAdFound = true; - ads.push(html5Ad); - } - - return ads; - }; - - TopSearcher.prototype._mainGetHTMLAd = function () { - let styles = this.doc.querySelectorAll('div > style, div > link[rel="stylesheet"]'), - i, - div; - for (i = 0; i < styles.length; i++) { - div = styles[i].parentNode; - if (!div.mpAdFound && SizeMatcher.elementIsAdShaped(div) && this._jumpedOut(div)) { - return div; - } - } - }; - - TopSearcher.prototype._jumpedOut = function (el) { - let siblings, - ifrs; - siblings = exports.utils.realArray(el.parentNode.children); - ifrs = siblings.filter(el => el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0); - return ifrs.length > 0; - }; - - function IframeSearcher(win) { - this.MIN_AD_AREA = 14000; - this.MIN_WINDOW_PX = 10; - - this.win = win; - this.doc = win.document; - this.body = win.document.body; - this.winClickTag = win.clickTag; - this.adSizeMeta = this._getAdSizeMeta(); - this.numElementsInBody = (this.body && this.body.querySelectorAll('*').length) || 0; - - this.shouldSearchWindow = false; - if (!this.win.mpAdFound && this.body && !Throttler.throttleWin(this.win)) { - this.winWidth = this.win.innerWidth; - this.winHeight = this.win.innerHeight; - if (this._meetsMinAdSize(this.winWidth, this.winHeight) && !this._containsLargeIframes()) { - this.shouldSearchWindow = true; - } - } - } - - IframeSearcher.prototype.search = function () { - let ad; - - if (this.shouldSearchWindow) { - ad = this._search(); - if (ad) { - ad.mpAdFound = true; - win.mpAdFound = true; - return ad; - } - Throttler.countSearch(this.win); - } - - return null; - }; - - IframeSearcher.prototype._search = function () { - let _this = this, - stdCandidates, - html5Candidates, - stdEl, - html5El; - - stdCandidates = this.body.querySelectorAll('img, object, embed'); - - stdEl = getFirst(stdCandidates, (el) => { - if (!el.mpAdFound - && !Throttler.throttleElement(el) - && (el.tagName !== 'IMG' || isStandardImage(el)) - && _this._elementIsAtLeastAsBigAsWindow(el)) { - return true; - } - Throttler.countSearch(el); - return false; - }); - - if (stdEl) { - return stdEl; - } - - if (this._isHTML5Iframe()) { - html5Candidates = this.doc.querySelectorAll('body, canvas, button, video, svg, div'); - html5El = getFirst(html5Candidates, (el) => { - if (_this._elementIsAtLeastAsBigAsWindow(el)) { - return true; - } - Throttler.countSearch(el); - return false; - }); - } - - if (html5El) { - html5El.html5 = true; - html5El.winClickTag = this.winClickTag; - html5El.adSizeMeta = this.adSizeMeta; - return html5El; - } - - return null; - }; - - IframeSearcher.prototype._isHTML5Iframe = function () { - if (this.winClickTag || this.adSizeMeta) { - return true; - } - - if (this.doc.querySelectorAll('canvas', 'button', 'video', 'svg').length > 0) { - return true; - } - - if (this.numElementsInBody >= 5 && Throttler.getCount(this.win) > 0 && this.doc.querySelectorAll('div').length > 0) { - return true; - } - - return false; - }; - - IframeSearcher.prototype._elementIsAtLeastAsBigAsWindow = function (el) { - let rect = el.getBoundingClientRect(), - tol = 0.95; - - return rect.width >= (tol * this.winWidth) && rect.height >= (tol * this.winHeight); - }; - - IframeSearcher.prototype._meetsMinAdSize = function (width, height) { - return (width * height) >= this.MIN_AD_AREA; - }; - - IframeSearcher.prototype._containsLargeIframes = function () { - const iframes = this.doc.querySelectorAll('iframe'); - let rect; - let i; - for (i = 0; i < iframes.length; i++) { - rect = iframes[i].getBoundingClientRect(); - if (rect.width > this.MIN_WINDOW_PX || rect.height > this.MIN_WINDOW_PX) { - return true; - } - } - return false; - }; - - IframeSearcher.prototype._getAdSizeMeta = function () { - const adSizeMeta = this.doc.querySelectorAll('meta[name="ad.size"]'); - if (adSizeMeta.length > 0) { - return adSizeMeta[0].content; - } - return null; - }; - - function getFirst(arr, testFn) { - let i, - el; - for (i = 0; i < arr.length; i++) { - el = arr[i]; - if (testFn(el)) { - return el; - } - } - return null; - } - - function isStandardImage(img) { - return img.src && (img.parentNode.tagName === 'A' || img.getAttribute('onclick')); - } - - function getFriendlyIframes(win) { - let iframes = win.document.querySelectorAll('iframe'); - iframes = exports.utils.realArray(iframes); - const friendlyIframes = iframes.filter(ifr => exports.utils.isFriendlyWindow(ifr.contentWindow)); - return friendlyIframes; - } - - function findAds(win) { - let i, - iframes, - searcher, - ad, - ads = []; - - if (win === win.top) { - searcher = new TopSearcher(win); - ads = ads.concat(searcher.search()); - } else { - searcher = new IframeSearcher(win); - ad = searcher.search(); - if (ad) { - ads.push(ad); - } - } - - iframes = getFriendlyIframes(win); - for (i = 0; i < iframes.length; i++) { - ads = ads.concat(findAds(iframes[i].contentWindow)); - } - - return ads; - } - - exports.adfinder = { - getMatchedAdSize: SizeMatcher.getMatchedAdSize.bind(SizeMatcher), - findAds, - }; - }(exports)); - - (function (exports) { - const parser = { - TAGS_WITH_SRC_ATTR: { - IMG: true, - SCRIPT: true, - IFRAME: true, - EMBED: true, - }, - - MAX_ATTR_LEN: 100, - - getUrl(el, params) { - let url; - - if (this.TAGS_WITH_SRC_ATTR.hasOwnProperty(el.tagName)) { - url = el.src; - } else if (el.tagName === 'OBJECT') { - url = el.data || (params && params.movie) || null; - } else if (el.tagName === 'A') { - url = el.href; - } - - if (url && url.indexOf('http') === 0) { - return url; - } - return null; - }, - - getParams(el) { - if (el.tagName !== 'OBJECT') { - return null; - } - - let i, - child; - const params = {}; - const children = el.children; - for (i = 0; i < children.length; i++) { - child = children[i]; - if (child.tagName === 'PARAM' && child.name) { - params[child.name.toLowerCase()] = child.value; - } - } - return params; - }, - - getPosition(el) { - const rect = el.getBoundingClientRect(); - const win = exports.utils.elementWindow(el); - - return { - width: Math.round(rect.width), - height: Math.round(rect.height), - left: Math.round(rect.left + win.pageXOffset), - top: Math.round(rect.top + win.pageYOffset), - }; - }, - - getFlashvars(el, params, url) { - let flashvars; - const urlQS = url && url.split('?')[1]; - - if (el.tagName === 'EMBED') { - flashvars = el.getAttribute('flashvars') || urlQS; - } else if (el.tagName === 'OBJECT') { - flashvars = params.flashvars || el.getAttribute('flashvars') || urlQS; - } - - return (flashvars && exports.utils.parseQS(flashvars)) || null; - }, - - findClickThru(el, flashvars) { - let key; - if (el.tagName === 'IMG' && el.parentElement.tagName === 'A') { - return el.parentElement.href; - } if (flashvars) { - for (key in flashvars) { - if (flashvars.hasOwnProperty(key)) { - if (key.toLowerCase().indexOf('clicktag') === 0) { - return flashvars[key]; - } - } - } - } - return null; - }, - - getAttr(el, name) { - const val = el.getAttribute(name); - - if (val && val.slice && val.toString) { - return val.slice(0, this.MAX_ATTR_LEN).toString(); - } - return null; - }, - - putPropIfExists(obj, name, val) { - if (val) { - obj[name] = val; - } - }, - - putAttrIfExists(obj, el, name) { - const val = this.getAttr(el, name); - this.putPropIfExists(obj, name, val); - }, - - elementToJSON(el, opt_findClickThru) { - const pos = this.getPosition(el); - const params = this.getParams(el); - const url = this.getUrl(el, params); - const flashvars = this.getFlashvars(el, params, url); - const clickThru = opt_findClickThru && this.findClickThru(el, flashvars); - const json = { - tagName: el.tagName, - width: pos.width, - height: pos.height, - left: pos.left, - top: pos.top, - children: [], - }; - - if (params) { - delete params.flashvars; - } - - this.putAttrIfExists(json, el, 'id'); - this.putAttrIfExists(json, el, 'class'); - this.putAttrIfExists(json, el, 'name'); - - this.putPropIfExists(json, 'flashvars', flashvars); - this.putPropIfExists(json, 'url', url); - this.putPropIfExists(json, 'params', params); - this.putPropIfExists(json, 'clickThru', clickThru); - - return json; - }, - }; - - exports.parser = { elementToJSON: parser.elementToJSON.bind(parser) }; - }(exports)); - - (function (exports) { - const ContextManager = function (adData) { - this.adData = adData; - }; - - ContextManager.prototype = { - CONTAINER_SIZE_TOL: 0.4, - ASPECT_RATIO_FOR_LEADERBOARDS: 2, - - isValidContainer(el, opt_curWin) { - const cWidth = el.clientWidth; - const cHeight = el.clientHeight; - - const adWidth = this.adData.width; - const adHeight = this.adData.height; - - const winWidth = opt_curWin && opt_curWin.innerWidth; - const winHeight = opt_curWin && opt_curWin.innerHeight; - const similarWin = opt_curWin && this.withinTol(adWidth, winWidth) && this.withinTol(adHeight, winHeight); - - const similarSizeX = this.withinTol(adWidth, cWidth); - const similarSizeY = this.withinTol(adHeight, cHeight); - const adAspect = adWidth / adHeight; - - return similarWin || el.tagName === 'A' || (adAspect >= this.ASPECT_RATIO_FOR_LEADERBOARDS && similarSizeY) || (similarSizeX && similarSizeY); - }, - - withinTol(adlen, conlen) { - const pct = (conlen - adlen) / adlen; - - return pct <= this.CONTAINER_SIZE_TOL; - }, - - serializeElements(el) { - if (!el) { - return; - } - let i; - let ifrWin; - const adId = this.adData.adId; - let elIsAd = false; - - if (adId && el[adId] && el[adId].isAd === true) { - elIsAd = true; - } - - const json = exports.parser.elementToJSON(el, elIsAd); - let childJSON; - - if (elIsAd) { - json.adId = adId; - this.adData.element = {}; - - const keys = Object.keys(json); - for (i = 0; i < keys.length; i++) { - const key = keys[i]; - if (key !== 'children' && key !== 'contents') { - this.adData.element[key] = json[key]; - } - } - } - - const children = exports.utils.realArray(el.children).filter((el) => { - const param = el.tagName === 'PARAM'; - const inlineScript = el.tagName === 'SCRIPT' && !(el.src && el.src.indexOf('http') >= 0); - const noScript = el.tagName === 'NOSCRIPT'; - return !(param || inlineScript || noScript); - }); - - for (i = 0; i < children.length; i++) { - childJSON = this.serializeElements(children[i]); - if (childJSON) { - json.children.push(childJSON); - } - } - - if (el.tagName === 'IFRAME') { - ifrWin = el.contentWindow; - - if (adId && el[adId] && el[adId].needsWindow) { - json.contents = this.adData.serializedIframeContents; - el[adId].needsWindow = false; - delete this.adData.serializedIframeContents; - } else if (exports.utils.isFriendlyWindow(ifrWin)) { - childJSON = this.serializeElements(ifrWin.document.documentElement); - if (childJSON) { - json.contents = childJSON; - } - } - } - - if (json.children.length > 0 || json.adId || json.tagName === 'IFRAME' || json.url) { - return json; - } - return null; - }, - - captureHTML(containerEl) { - this.adData.context = this.serializeElements(containerEl); - }, - - nodeCount(el) { - return el.getElementsByTagName('*').length + 1; - }, - - highestContainer(curWin, referenceElement) { - let curContainer = referenceElement; - const docEl = curWin.document.documentElement; - let parentContainer; - - if (curWin !== curWin.top && this.isValidContainer(docEl, curWin)) { - return docEl; - } - - while (true) { - parentContainer = curContainer.parentElement; - if (parentContainer && this.isValidContainer(parentContainer)) { - curContainer = parentContainer; - } else { - return curContainer; - } - } - }, - }; - - const tagfinder = { - - setPositions(adData, opt_el, opt_winPos) { - const el = opt_el || adData.context; - const winPos = opt_winPos || { left: 0, top: 0 }; - let ifrPos; - - el.left += winPos.left; - el.top += winPos.top; - - if (el.children) { - el.children.forEach(function (child) { - this.setPositions(adData, child, winPos); - }, this); - } - - if (el.contents) { - ifrPos = { left: el.left, top: el.top }; - this.setPositions(adData, el.contents, ifrPos); - } - - if (el.adId === adData.adId) { - adData.element.left = el.left; - adData.element.top = el.top; - } - }, - - appendTags(adData, referenceElement) { - const mgr = new ContextManager(adData); - let curWin = exports.utils.elementWindow(referenceElement); - let highestContainer; - - while (true) { - highestContainer = mgr.highestContainer(curWin, referenceElement); - mgr.captureHTML(highestContainer); - if (curWin === curWin.top) { - break; - } else { - curWin.mpAdFound = true; - - mgr.adData.serializedIframeContents = mgr.adData.context; - - if (exports.utils.isFriendlyWindow(curWin.parent)) { - referenceElement = curWin.frameElement; - referenceElement[mgr.adData.adId] = { needsWindow: true }; - curWin = curWin.parent; - } else { - break; - } - } - } - return { - referenceElement, - highestContainer, - }; - }, - }; - - exports.tagfinder = tagfinder; - }(exports)); - - (function (exports) { - let _onAdFound; - const _logGen = new exports.utils.LogGenerator(); - let _pageTags; - const INIT_MS_BW_SEARCHES = 2000; - const PAGE_TAG_RE = new RegExp('gpt|oascentral'); - const POST_MSG_ID = '1532387537-31575-24732-11173-32339'; - const AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)'); - - function getPageTags(doc) { - let scripts = doc.getElementsByTagName('script'); - const pageTags = []; - scripts = exports.utils.realArray(scripts); - scripts.forEach((script) => { - if (PAGE_TAG_RE.exec(script.src)) { - pageTags.push({ tagName: 'SCRIPT', url: script.src }); - } - }); - return pageTags; - } - - function messageAllParentFrames(adData) { - adData.postMessageId = POST_MSG_ID; - - adData = JSON.stringify(adData); - - let win = window; - while (win !== win.top) { - win = win.parent; - win.postMessage(adData, win.location.href); - } - } - - function appendTagsAndSendToParent(adData, referenceElement) { - const results = exports.tagfinder.appendTags(adData, referenceElement); - if (exports.utils.SCRIPT_IN_HOSTILE_IFRAME) { - messageAllParentFrames(adData); - } else if (exports.utils.SCRIPT_IN_WINDOW_TOP) { - exports.tagfinder.setPositions(adData); - - adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height); - if (!adData.matchedSize) { - if (AD_SERVER_RE.exec(results.referenceElement.id)) { - adData.matchedSize = [adData.width, adData.height]; - adData.oddSize = true; - } else { - return; - } - } - delete adData.width; - delete adData.height; - adData.curPageUrl = exports.utils.getPageUrl(); - _pageTags = _pageTags || getPageTags(document); - const log = _logGen.log('ad', [adData], _pageTags); - - if (_onAdFound) { - _onAdFound(log, results.referenceElement); - } - } - } - - function extractAdsWrapper() { - if (exports.utils.SCRIPT_IN_WINDOW_TOP || document.readyState === 'complete') { - extractAds(); - } - setTimeout( - () => { extractAdsWrapper(); }, INIT_MS_BW_SEARCHES, - ); - } - - function extractAds() { - const ads = exports.adfinder.findAds(window); - ads.forEach((ad) => { - const startTime = new Date().getTime(); - const adId = `${startTime}-${Math.floor(Math.random() * 10e12)}`; - - const adData = { - width: Math.round(ad.offsetWidth), - height: Math.round(ad.offsetHeight), - startTime, - adId, - html5: ad.html5 || false, - }; - - if (ad.html5) { - adData.adSizeMeta = ad.adSizeMeta || null; - adData.winClickTag = ad.winClickTag || null; - } - - ad[adId] = { isAd: true }; - - appendTagsAndSendToParent(adData, ad); - }); - } - - function isChildWin(myWin, otherWin) { - let parentWin = otherWin.parent; - while (parentWin !== otherWin) { - if (parentWin === myWin) { - return true; - } - otherWin = parentWin; - parentWin = parentWin.parent; - } - return false; - } - - function iframeFromWindow(win, winToMatch) { - let i, - ifr, - ifrWin, - iframes = win.document.querySelectorAll('iframe'); - - for (i = 0; i < iframes.length; i++) { - ifr = iframes[i]; - if (ifr.contentWindow === winToMatch) { - return ifr; - } - } - - for (i = 0; i < iframes.length; i++) { - ifrWin = iframes[i].contentWindow; - if (exports.utils.isFriendlyWindow(ifrWin)) { - ifr = iframeFromWindow(ifrWin, winToMatch); - if (ifr) { - return ifr; - } - } - } - } - - function onPostMessage(event) { - let adData, - ifrWin = event.source, - - myWin = window.document.defaultView, - ifrTag; - - try { - adData = JSON.parse(event.data); - } catch (e) { - return; - } - - if (!adData) return; - - if (adData.postMessageId === POST_MSG_ID) { - delete adData.postMessageId; - - event.stopImmediatePropagation(); - - if (isChildWin(myWin, ifrWin)) { - if (exports.utils.isFriendlyWindow(ifrWin)) { - ifrTag = ifrWin.frameElement; - } else { - ifrTag = iframeFromWindow(myWin, ifrWin); - } - - if (ifrTag) { - ifrTag[adData.adId] = { needsWindow: true }; - appendTagsAndSendToParent(adData, ifrTag); - } - } - } - } - - function onVideoMessage(msg, sender, callback) { - let log; - if (msg.event === 'new-video-ad') { - msg.assets.forEach((asset) => { - - }); - log = _logGen.log('video', msg.assets); - } else { - log = _logGen.log('invalid-video', msg.assets); - } - - msg.assets.forEach((a) => { delete a.isVideo; }); - log.displayAdFound = msg.displayAdFound; - log.requests = msg.requests; - log.data = msg.event_data; - - log.doc.finalPageUrl = log.doc.url; - log.doc.url = exports.utils.normalizeUrl(msg.origUrl); - - _onAdFound(log); - } - - function addBackgroundListener(event, callback) { - if (typeof browser !== 'undefined') { - browser.runtime.onMessage.addListener((msg) => { - if (msg.event === event) { - callback(msg); - } - }); - } else if (typeof chrome !== 'undefined') { - chrome.runtime.onMessage.addListener((msg) => { - if (msg.event === event) { - callback(msg); - } - }); - } else if (window.self.port) { - window.self.port.on(event, callback); - } - } - - exports.coordinator = { - addPostMessageListener() { - if (!exports.utils.SCRIPT_IN_FRIENDLY_IFRAME) { - window.addEventListener('message', onPostMessage, false); - } - }, - - blockedRobotsMsgGen(sendFcn, origUrl) { - if (origUrl.indexOf('google.com/_/chrome/newtab') === -1) { - const onBlockedRobotsMessage = function () { - let log; - log = _logGen.log('invalid-robotstxt', []); - log.doc.finalPageUrl = log.doc.url; - log.doc.url = exports.utils.normalizeUrl(origUrl); - - sendFcn(log); - }; - return onBlockedRobotsMessage; - } - return function () {}; - }, - - init(onAdFound) { - if (exports.utils.SCRIPT_IN_FRIENDLY_IFRAME) { - return false; - } - - _onAdFound = onAdFound; - if (exports.utils.SCRIPT_IN_WINDOW_TOP) { - const log = _logGen.log('page'); - onAdFound(log); - - window.addEventListener('beforeunload', (event) => { - const log = _logGen.log('unload'); - log.timing = window.performance.timing; - onAdFound(log); - }); - - addBackgroundListener('new-video-ad', onVideoMessage); - addBackgroundListener('new-invalid-video-ad', onVideoMessage); - } - - exports.utils.onDocLoaded(document, extractAdsWrapper); - }, - }; - }(exports)); - - if (exports.utils.SCRIPT_IN_WINDOW_TOP) { - window.adparser = { - init: exports.coordinator.init, - addPostMessageListener: exports.coordinator.addPostMessageListener, - askIfTrackingEnabled: exports.utils.askIfTrackingEnabled, - blockedRobotsMsgGen: exports.coordinator.blockedRobotsMsgGen, - inWindowTop: exports.utils.SCRIPT_IN_WINDOW_TOP, - sendToBackground: exports.utils.sendToBackground, - }; - } else { - exports.coordinator.addPostMessageListener(); - exports.utils.askIfTrackingEnabled( - () => { - exports.coordinator.init(() => {}); - }, - () => {}, - ); - } -}(window)); -(function (adparser, pageUrl) { - function onAdFound(log) { - adparser.sendToBackground({ id: 'ad_log', subject: log }, 'ad_log', '', () => {}); - } - - if (adparser && adparser.inWindowTop) { - adparser.addPostMessageListener(); - adparser.askIfTrackingEnabled( - () => { - adparser.init(onAdFound); - }, - adparser.blockedRobotsMsgGen(onAdFound, pageUrl), - ); - } -}(window.adparser, window.location.href)); +var exports = {}; + +(function(exports) { + + var port = browser.runtime.connect({name:"adparser"}); + var utils = { + normalizeUrl: function(url) { + + return this.hashUrl(url) || null; + + }, + + getReferrer: function() { + return this.normalizeUrl(document.referrer); + }, + + getPageUrl: function() { + return this.normalizeUrl(window.location.href); + }, + hashUrl: function(url) { + var a, + result; + + if ( !url || url.indexOf('http') !== 0 ) { + return null; + } + + a = document.createElement('a'); + a.href = url; + + result = a.protocol + '//' + a.hostname + '/'; + + if ( a.pathname && a.pathname !== '/' ) { + result += this.hashCode(a.pathname); + } + + if ( a.search ) { + result += '?' + this.hashCode(a.search); + } + + if ( a.hash ) { + result += '#' + this.hashCode(a.hash); + } + + return result; + }, + + hashCode: function(str) { + var hash = 0, + kar, + i; + + if ( str.length === 0 ) { + return hash; + } + + for ( i = 0; i < str.length; i++ ) { + kar = str.charCodeAt(i); + hash = ((hash << 5) - hash) + kar; + hash = hash & hash; + } + + return hash + Math.pow(2, 32); + }, + + realArray: function(a) { + return Array.prototype.slice.apply(a); + }, + + onDocLoaded: function(doc, callback) { + if ( doc.readyState === 'loading' ) { + doc.addEventListener('DOMContentLoaded', callback); + } else { + callback(); + } + }, + + SCRIPT_IN_WINDOW_TOP: window === window.top, + + isFriendlyWindow: function(win) { + + var href; + try { + href = win.location.href; + } catch(e) { + return false; + } + return true; + }, + + elementWindow: function(el) { + return el.ownerDocument.defaultView; + }, + + viewport: function(win) { + return {width: win.innerWidth, height: win.innerHeight}; + }, + + parseQS: function(qs) { + if ( qs.indexOf('http') === 0 ) { + qs = qs.split('?')[1]; + } + var i, kvs, key, val; + var dict = {}; + qs = qs.split('&'); + for ( i = 0; i < qs.length; i++ ) { + kvs = qs[i].split('='); + key = kvs[0]; + val = kvs.slice(1).join('='); + try { + dict[key] = window.decodeURIComponent(val); + } catch (e) { + + continue; + } + } + return dict; + }, + sendToBackground: function(message, event, responseMessage) { + if ( typeof browser !== 'undefined' || typeof chrome !== 'undefined' ) { + port.postMessage(message); + } else if ( window.self.port ) { + window.self.port.on(responseMessage, onResponse); + window.self.port.emit(event, message); + } + }, + + askIfTrackingEnabled: function(callback, elseCallback) { + utilCallback = callback; + utilElseCallback = elseCallback; + + this.sendToBackground( + 'is_tracking_enabled', + '', + 'tracking_enabled_response' + ); + + } + }; + + port.onMessage.addListener((message) => { + if ( message && message.tracking_enabled ) { + + utilCallback(); + } else { + + utilElseCallback(); + } + }); + + utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent); + utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME; + + function LogGenerator() { + this.msgNum = 0; + this.pageMeta = { + 'url': utils.getPageUrl(), + 'isHP': window.location.pathname === '/', + 'referrer': utils.getReferrer(), + 'rand': Math.floor(Math.random() * 10e12), + 'startTime': new Date().getTime() + }; + } + + LogGenerator.prototype = { + log: function(event, opt_assets, opt_pageTags) { + var opt_video_assets; + if ( event === 'video' || event === 'invalid-video' ) { + opt_video_assets = opt_assets || []; + opt_assets = []; + } else { + opt_video_assets = []; + opt_assets = opt_assets || []; + } + var result = { + doc: this.pageMeta, + event: event, + video_assets: opt_video_assets, + assets: opt_assets, + version: '3', + mrev: 'dfd96a9-d', + msgNum: this.msgNum, + timestamp: new Date().getTime(), + pageVis: document.visibilityState, + pageFoc: document.hasFocus(), + pageTags: opt_pageTags || [] + }; + this.msgNum++; + return result; + } + }; + + utils.LogGenerator = LogGenerator; + + let utilCallback, utilElseCallback; + + exports.utils = utils; +})(exports); + +(function(exports) { + + var SizeMatcher = { + VALID_AD_SIZES: [ + [300, 50], + [320, 50], + [160, 600], + [300, 250], + [300, 600], + [300, 1050], + [336, 280], + [336, 850], + [468, 60], + [728, 90], + [728, 250], + [728, 270], + [970, 66], + [970, 90], + [970, 125], + [970, 250], + [970, 400], + [970, 415], + [1280, 100] + ], + + PX_SIZE_TOL: 10, + + getMatchedAdSize: function(width, height) { + + if ( !this.set ) { + this.set = this._makeSizeSet(); + } + + return this.set[Math.round(width) + 'x' + Math.round(height)]; + }, + + elementIsAdShaped: function(el) { + return !!this.getMatchedAdSizeForElement(el); + }, + + getMatchedAdSizeForElement: function(el) { + var rect = el.getBoundingClientRect(); + return this.getMatchedAdSize(rect.width, rect.height); + }, + + _makeSizeSet: function() { + var set = {}; + var i; + var xfuz; + var yfuz; + var size; + var width; + var height; + + for ( i = 0; i < this.VALID_AD_SIZES.length; i++ ) { + for ( xfuz = -this.PX_SIZE_TOL; xfuz <= this.PX_SIZE_TOL; xfuz++ ) { + for ( yfuz = -this.PX_SIZE_TOL; yfuz <= this.PX_SIZE_TOL; yfuz++ ) { + size = this.VALID_AD_SIZES[i]; + width = size[0] + xfuz; + height = size[1] + yfuz; + set[width + 'x' + height] = size; + } + } + } + return set; + } + }; + + var Throttler = { + MAX_SEARCHES_PER_WINDOW: 10, + MAX_SEARCHES_PER_ELEMENT: 2, + + countSearch: function(el) { + if ( typeof el.searches !== 'number' ) { + el.searches = 0; + } + + el.searches += 1; + }, + + throttle: function(el, max) { + if ( typeof el.searches === 'number' && el.searches >= max ) { + return true; + } + return false; + }, + + throttleElement: function(el) { + return this.throttle(el, this.MAX_SEARCHES_PER_ELEMENT); + }, + + throttleWin: function(win) { + return this.throttle(win, this.MAX_SEARCHES_PER_WINDOW); + }, + + getCount: function(el) { + return el.searches || 0; + } + }; + + function TopSearcher(win) { + this.win = win; + this.doc = win.document; + } + + TopSearcher.prototype.search = function() { + var candidates = exports.utils.realArray(this.doc.querySelectorAll('img, object, embed')), + html5Ad, + ads = []; + + ads = ads.concat(candidates.filter(function(el) { + if ( !el.mpAdFound && !Throttler.throttleElement(el) ) { + Throttler.countSearch(el); + if ( (el.tagName !== 'IMG' || isStandardImage(el)) && SizeMatcher.elementIsAdShaped(el) ) { + el.mpAdFound = true; + return true; + } + } + return false; + })); + + html5Ad = this._mainGetHTMLAd(); + if ( html5Ad ) { + html5Ad.html5 = true; + html5Ad.mpAdFound = true; + ads.push(html5Ad); + } + + return ads; + }; + + TopSearcher.prototype._mainGetHTMLAd = function() { + var styles = this.doc.querySelectorAll('div > style, div > link[rel="stylesheet"]'), + i, div; + for ( i = 0; i < styles.length; i++ ) { + div = styles[i].parentNode; + if ( !div.mpAdFound && SizeMatcher.elementIsAdShaped(div) && this._jumpedOut(div) ) { + return div; + } + } + }; + + TopSearcher.prototype._jumpedOut = function(el) { + var siblings, ifrs; + siblings = exports.utils.realArray(el.parentNode.children); + ifrs = siblings.filter(function(el) { + return el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0; + }); + return ifrs.length > 0; + }; + + function IframeSearcher(win) { + this.MIN_AD_AREA = 14000; + this.MIN_WINDOW_PX = 10; + + this.win = win; + this.doc = win.document; + this.body = win.document.body; + this.winClickTag = win.clickTag; + this.adSizeMeta = this._getAdSizeMeta(); + this.numElementsInBody = (this.body && this.body.querySelectorAll('*').length) || 0; + + this.shouldSearchWindow = false; + if ( !this.win.mpAdFound && this.body && !Throttler.throttleWin(this.win) ) { + this.winWidth = this.win.innerWidth; + this.winHeight = this.win.innerHeight; + if ( this._meetsMinAdSize(this.winWidth, this.winHeight) && !this._containsLargeIframes() ) { + this.shouldSearchWindow = true; + } + } + + } + + IframeSearcher.prototype.search = function() { + var ad; + + if ( this.shouldSearchWindow ) { + ad = this._search(); + if ( ad ) { + ad.mpAdFound = true; + win.mpAdFound = true; + return ad; + } + Throttler.countSearch(this.win); + } + + return null; + }; + + IframeSearcher.prototype._search = function() { + var _this = this, + stdCandidates, + html5Candidates, + stdEl, + html5El; + + stdCandidates = this.body.querySelectorAll('img, object, embed'); + + stdEl = getFirst(stdCandidates, function(el) { + if ( !el.mpAdFound && + !Throttler.throttleElement(el) && + (el.tagName !== 'IMG' || isStandardImage(el)) && + _this._elementIsAtLeastAsBigAsWindow(el)) + { + return true; + } + Throttler.countSearch(el); + return false; + }); + + if ( stdEl ) { + return stdEl; + } + + if ( this._isHTML5Iframe() ) { + html5Candidates = this.doc.querySelectorAll('body, canvas, button, video, svg, div'); + html5El = getFirst(html5Candidates, function(el) { + + if ( _this._elementIsAtLeastAsBigAsWindow(el) ) { + return true; + } + Throttler.countSearch(el); + return false; + }); + } + + if ( html5El ) { + html5El.html5 = true; + html5El.winClickTag = this.winClickTag; + html5El.adSizeMeta = this.adSizeMeta; + return html5El; + } + + return null; + }; + + IframeSearcher.prototype._isHTML5Iframe = function() { + if ( this.winClickTag || this.adSizeMeta ) { + return true; + } + + if ( this.doc.querySelectorAll('canvas', 'button', 'video', 'svg').length > 0 ) { + return true; + } + + if ( this.numElementsInBody >= 5 && Throttler.getCount(this.win) > 0 && this.doc.querySelectorAll('div').length > 0 ) { + return true; + } + + return false; + }; + + IframeSearcher.prototype._elementIsAtLeastAsBigAsWindow = function(el) { + var rect = el.getBoundingClientRect(), + tol = 0.95; + + return rect.width >= (tol * this.winWidth) && rect.height >= (tol * this.winHeight); + }; + + IframeSearcher.prototype._meetsMinAdSize = function(width, height) { + return (width * height) >= this.MIN_AD_AREA; + }; + + IframeSearcher.prototype._containsLargeIframes = function() { + var iframes = this.doc.querySelectorAll('iframe'); + var rect; + var i; + for ( i = 0; i < iframes.length; i++ ) { + rect = iframes[i].getBoundingClientRect(); + if ( rect.width > this.MIN_WINDOW_PX || rect.height > this.MIN_WINDOW_PX ) { + return true; + } + } + return false; + }; + + IframeSearcher.prototype._getAdSizeMeta = function() { + var adSizeMeta = this.doc.querySelectorAll('meta[name="ad.size"]'); + if ( adSizeMeta.length > 0 ) { + return adSizeMeta[0].content; + } else { + return null; + } + }; + + function getFirst(arr, testFn) { + var i, el; + for ( i = 0; i < arr.length; i++ ) { + el = arr[i]; + if ( testFn(el) ) { + return el; + } + } + return null; + } + + function isStandardImage(img) { + + return img.src && (img.parentNode.tagName === 'A' || img.getAttribute('onclick')); + } + + function getFriendlyIframes(win) { + var iframes = win.document.querySelectorAll('iframe'); + iframes = exports.utils.realArray(iframes); + var friendlyIframes = iframes.filter(function(ifr) { + return exports.utils.isFriendlyWindow(ifr.contentWindow); + }); + return friendlyIframes; + } + + function findAds(win) { + var i, + iframes, + searcher, + ad, + ads = []; + + if ( win === win.top ) { + searcher = new TopSearcher(win); + ads = ads.concat(searcher.search()); + } else { + searcher = new IframeSearcher(win); + ad = searcher.search(); + if ( ad ) { + ads.push(ad); + } + } + + iframes = getFriendlyIframes(win); + for ( i = 0; i < iframes.length; i++ ) { + ads = ads.concat(findAds(iframes[i].contentWindow)); + } + + return ads; + } + + exports.adfinder = { + getMatchedAdSize: SizeMatcher.getMatchedAdSize.bind(SizeMatcher), + findAds: findAds + }; +})(exports); + +(function(exports) { + + var parser = { + TAGS_WITH_SRC_ATTR: { + 'IMG': true, + 'SCRIPT': true, + 'IFRAME': true, + 'EMBED': true + }, + + MAX_ATTR_LEN: 100, + + getUrl: function(el, params) { + var url; + + if ( this.TAGS_WITH_SRC_ATTR.hasOwnProperty(el.tagName) ) { + url = el.src; + + } else if ( el.tagName === 'OBJECT' ) { + url = el.data || (params && params.movie) || null; + + } else if ( el.tagName === 'A' ) { + url = el.href; + } + + if ( url && url.indexOf('http') === 0 ) { + return url; + } else { + return null; + } + }, + + getParams: function(el) { + if ( el.tagName !== 'OBJECT' ) { + return null; + } + + var i, child; + var params = {}; + var children = el.children; + for ( i = 0; i < children.length; i++ ) { + child = children[i]; + if ( child.tagName === 'PARAM' && child.name ) { + + params[child.name.toLowerCase()] = child.value; + } + } + return params; + }, + + getPosition: function(el) { + var rect = el.getBoundingClientRect(); + var win = exports.utils.elementWindow(el); + + return { + width: Math.round(rect.width), + height: Math.round(rect.height), + left: Math.round(rect.left + win.pageXOffset), + top: Math.round(rect.top + win.pageYOffset) + }; + }, + + getFlashvars: function(el, params, url) { + var flashvars; + var urlQS = url && url.split('?')[1]; + + if ( el.tagName === 'EMBED' ) { + flashvars = el.getAttribute('flashvars') || urlQS; + + } else if ( el.tagName === 'OBJECT' ) { + flashvars = params.flashvars || el.getAttribute('flashvars') || urlQS; + } + + return (flashvars && exports.utils.parseQS(flashvars)) || null; + }, + + findClickThru: function(el, flashvars) { + var key; + if ( el.tagName === 'IMG' && el.parentElement.tagName === 'A' ) { + return el.parentElement.href; + } else if ( flashvars ) { + for ( key in flashvars ) { + if ( flashvars.hasOwnProperty(key) ) { + + if ( key.toLowerCase().indexOf('clicktag') === 0 ) { + return flashvars[key]; + } + } + } + } + return null; + }, + + getAttr: function(el, name) { + var val = el.getAttribute(name); + + if ( val && val.slice && val.toString ) { + + return val.slice(0, this.MAX_ATTR_LEN).toString(); + } else { + return null; + } + }, + + putPropIfExists: function(obj, name, val) { + if ( val ) { + obj[name] = val; + } + }, + + putAttrIfExists: function(obj, el, name) { + var val = this.getAttr(el, name); + this.putPropIfExists(obj, name, val); + }, + + elementToJSON: function(el, opt_findClickThru) { + var pos = this.getPosition(el); + var params = this.getParams(el); + var url = this.getUrl(el, params); + var flashvars = this.getFlashvars(el, params, url); + var clickThru = opt_findClickThru && this.findClickThru(el, flashvars); + var json = { + tagName: el.tagName, + width: pos.width, + height: pos.height, + left: pos.left, + top: pos.top, + children: [] + }; + + if ( params ) { + + delete params.flashvars; + } + + this.putAttrIfExists(json, el, 'id'); + this.putAttrIfExists(json, el, 'class'); + this.putAttrIfExists(json, el, 'name'); + + this.putPropIfExists(json, 'flashvars', flashvars); + this.putPropIfExists(json, 'url', url); + this.putPropIfExists(json, 'params', params); + this.putPropIfExists(json, 'clickThru', clickThru); + + return json; + } + }; + + exports.parser = { elementToJSON: parser.elementToJSON.bind(parser) }; +})(exports); + +(function(exports) { + + var ContextManager = function(adData) { + this.adData = adData; + }; + + ContextManager.prototype = { + CONTAINER_SIZE_TOL: 0.4, + ASPECT_RATIO_FOR_LEADERBOARDS: 2, + + isValidContainer: function(el, opt_curWin) { + + var cWidth = el.clientWidth; + var cHeight = el.clientHeight; + + var adWidth = this.adData.width; + var adHeight = this.adData.height; + + var winWidth = opt_curWin && opt_curWin.innerWidth; + var winHeight = opt_curWin && opt_curWin.innerHeight; + var similarWin = opt_curWin && this.withinTol(adWidth, winWidth) && this.withinTol(adHeight, winHeight); + + var similarSizeX = this.withinTol(adWidth, cWidth); + var similarSizeY = this.withinTol(adHeight, cHeight); + var adAspect = adWidth / adHeight; + + return similarWin || el.tagName === 'A' || (adAspect >= this.ASPECT_RATIO_FOR_LEADERBOARDS && similarSizeY) || (similarSizeX && similarSizeY); + }, + + withinTol: function(adlen, conlen) { + var pct = (conlen - adlen) / adlen; + + return pct <= this.CONTAINER_SIZE_TOL; + }, + + serializeElements: function(el) { + if ( !el ) { + return; + } + var i; + var ifrWin; + var adId = this.adData.adId; + var elIsAd = false; + + if ( adId && el[adId] && el[adId].isAd === true ) { + elIsAd = true; + } + + var json = exports.parser.elementToJSON(el, elIsAd); + var childJSON; + + if ( elIsAd ) { + json.adId = adId; + this.adData.element = {}; + + var keys = Object.keys(json); + for ( i = 0; i < keys.length; i++ ) { + var key = keys[i]; + if ( key !== 'children' && key !== 'contents' ) { + this.adData.element[key] = json[key]; + } + } + } + + var children = exports.utils.realArray(el.children).filter(function(el) { + var param = el.tagName === 'PARAM'; + var inlineScript = el.tagName === 'SCRIPT' && !(el.src && el.src.indexOf('http') >= 0); + var noScript = el.tagName === 'NOSCRIPT'; + return !(param || inlineScript || noScript); + }); + + for ( i = 0; i < children.length; i++ ) { + childJSON = this.serializeElements(children[i]); + if ( childJSON ) { + json.children.push(childJSON); + } + } + + if ( el.tagName === 'IFRAME' ) { + ifrWin = el.contentWindow; + + if ( adId && el[adId] && el[adId].needsWindow ) { + + json.contents = this.adData.serializedIframeContents; + el[adId].needsWindow = false; + delete this.adData.serializedIframeContents; + + } else if ( exports.utils.isFriendlyWindow(ifrWin) ) { + + childJSON = this.serializeElements(ifrWin.document.documentElement); + if ( childJSON ) { + json.contents = childJSON; + } + } + } + + if ( json.children.length > 0 || json.adId || json.tagName === 'IFRAME' || json.url ) { + return json; + } else { + return null; + } + }, + + captureHTML: function(containerEl) { + this.adData.context = this.serializeElements(containerEl); + }, + + nodeCount: function(el) { + return el.getElementsByTagName('*').length + 1; + }, + + highestContainer: function(curWin, referenceElement) { + var curContainer = referenceElement; + var docEl = curWin.document.documentElement; + var parentContainer; + + if ( curWin !== curWin.top && this.isValidContainer(docEl, curWin) ) { + return docEl; + } + + while ( true ) { + parentContainer = curContainer.parentElement; + if ( parentContainer && this.isValidContainer(parentContainer) ) { + curContainer = parentContainer; + } else { + return curContainer; + } + } + } + }; + + var tagfinder = { + + setPositions: function(adData, opt_el, opt_winPos) { + var el = opt_el || adData.context; + var winPos = opt_winPos || {left: 0, top: 0}; + var ifrPos; + + el.left += winPos.left; + el.top += winPos.top; + + if ( el.children ) { + el.children.forEach(function(child) { + this.setPositions(adData, child, winPos); + }, this); + } + + if ( el.contents ) { + ifrPos = {left: el.left, top: el.top}; + this.setPositions(adData, el.contents, ifrPos); + } + + if ( el.adId === adData.adId ) { + adData.element.left = el.left; + adData.element.top = el.top; + } + }, + + appendTags: function(adData, referenceElement) { + var mgr = new ContextManager(adData); + var curWin = exports.utils.elementWindow(referenceElement); + var highestContainer; + + while ( true ) { + highestContainer = mgr.highestContainer(curWin, referenceElement); + mgr.captureHTML(highestContainer); + if ( curWin === curWin.top ) { + break; + } else { + + curWin.mpAdFound = true; + + mgr.adData.serializedIframeContents = mgr.adData.context; + + if ( exports.utils.isFriendlyWindow(curWin.parent) ) { + referenceElement = curWin.frameElement; + referenceElement[mgr.adData.adId] = {needsWindow: true}; + curWin = curWin.parent; + } else { + break; + } + } + } + return { + referenceElement:referenceElement, + highestContainer: highestContainer + }; + } + }; + + exports.tagfinder = tagfinder; +})(exports); + +(function(exports) { + var _onAdFound; + var _logGen = new exports.utils.LogGenerator(); + var _pageTags; + var INIT_MS_BW_SEARCHES = 2000; + var PAGE_TAG_RE = new RegExp('gpt|oascentral'); + var POST_MSG_ID = '1553148212-11522-32076-13408-5488'; + var AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)'); + + function getPageTags(doc) { + var scripts = doc.getElementsByTagName('script'); + var pageTags = []; + scripts = exports.utils.realArray(scripts); + scripts.forEach(function(script) { + if ( PAGE_TAG_RE.exec(script.src) ) { + pageTags.push({'tagName': 'SCRIPT', 'url': script.src}); + } + }); + return pageTags; + } + + function messageAllParentFrames(adData) { + + adData.postMessageId = POST_MSG_ID; + + adData = JSON.stringify(adData); + + var win = window; + while ( win !== win.top ) { + win = win.parent; + win.postMessage(adData, '*'); + } + } + + function appendTagsAndSendToParent(adData, referenceElement) { + var results = exports.tagfinder.appendTags(adData, referenceElement); + if ( exports.utils.SCRIPT_IN_HOSTILE_IFRAME ) { + messageAllParentFrames(adData); + + } else if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { + + exports.tagfinder.setPositions(adData); + + adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height); + if ( !adData.matchedSize ) { + + if ( AD_SERVER_RE.exec(results.referenceElement.id) ) { + adData.matchedSize = [adData.width, adData.height]; + adData.oddSize = true; + } else { + + return; + } + } + delete adData.width; + delete adData.height; + adData.curPageUrl = exports.utils.getPageUrl(); + _pageTags = _pageTags || getPageTags(document); + var log = _logGen.log('ad', [adData], _pageTags); + + if ( _onAdFound ) { + + _onAdFound(log, results.referenceElement); + + } + } + } + + function extractAdsWrapper() { + if ( exports.utils.SCRIPT_IN_WINDOW_TOP || document.readyState === 'complete' ) { + extractAds(); + } + setTimeout( + function() { extractAdsWrapper(); }, INIT_MS_BW_SEARCHES + ); + } + + function extractAds() { + var ads = exports.adfinder.findAds(window); + ads.forEach(function(ad) { + + var startTime = new Date().getTime(); + var adId = startTime + '-' + Math.floor(Math.random() * 10e12); + + var adData = { + width: Math.round(ad.offsetWidth), + height: Math.round(ad.offsetHeight), + startTime: startTime, + adId: adId, + html5: ad.html5 || false + }; + + if ( ad.html5 ) { + adData.adSizeMeta = ad.adSizeMeta || null; + adData.winClickTag = ad.winClickTag || null; + } + + ad[adId] = { isAd: true }; + + appendTagsAndSendToParent(adData, ad); + }); + } + + function isChildWin(myWin, otherWin) { + var parentWin = otherWin.parent; + while ( parentWin !== otherWin ) { + if ( parentWin === myWin ) { + return true; + } + otherWin = parentWin; + parentWin = parentWin.parent; + } + return false; + } + + function iframeFromWindow(win, winToMatch) { + var i, ifr, ifrWin, + iframes = win.document.querySelectorAll('iframe'); + + for ( i = 0; i < iframes.length; i++ ) { + ifr = iframes[i]; + if ( ifr.contentWindow === winToMatch ) { + return ifr; + } + } + + for ( i = 0; i < iframes.length; i++ ) { + ifrWin = iframes[i].contentWindow; + if ( exports.utils.isFriendlyWindow(ifrWin) ) { + ifr = iframeFromWindow(ifrWin, winToMatch); + if ( ifr ) { + return ifr; + } + } + } + } + + function onPostMessage(event) { + var adData, + ifrWin = event.source, + + myWin = window.document.defaultView, + ifrTag; + + try { + + adData = JSON.parse(event.data); + } catch(e) { + + return; + } + + if ( adData.postMessageId === POST_MSG_ID ) { + + delete adData.postMessageId; + + event.stopImmediatePropagation(); + + if ( isChildWin(myWin, ifrWin) ) { + if ( exports.utils.isFriendlyWindow(ifrWin) ) { + ifrTag = ifrWin.frameElement; + } else { + ifrTag = iframeFromWindow(myWin, ifrWin); + } + + if ( ifrTag ) { + ifrTag[adData.adId] = {needsWindow: true}; + appendTagsAndSendToParent(adData, ifrTag); + } + } + } + } + + function onVideoMessage(msg, sender, callback) { + var log; + if ( msg.event === 'new-video-ad' ) { + msg.assets.forEach(function(asset) { + + }); + log = _logGen.log('video', msg.assets); + } else { + log = _logGen.log('invalid-video', msg.assets); + } + + msg.assets.forEach(function(a) {delete a.isVideo;}); + log.displayAdFound = msg.displayAdFound; + log.requests = msg.requests; + log.data = msg.event_data; + + log.doc.finalPageUrl = log.doc.url; + log.doc.url = exports.utils.normalizeUrl(msg.origUrl); + + _onAdFound(log); + } + + function addBackgroundListener(event, callback) { + if ( typeof browser !== 'undefined' || typeof chrome !== 'undefined' ) { + browser.runtime.onMessage.addListener(function(msg) { + if ( msg.event === event ) { + callback(msg); + } + }); + } + else if (window.self.port) { + window.self.port.on(event, callback); + } + } + + exports.coordinator = { + addPostMessageListener: function() { + if ( !exports.utils.SCRIPT_IN_FRIENDLY_IFRAME ) { + window.addEventListener('message', onPostMessage, false); + } + }, + + blockedRobotsMsgGen: function(sendFcn, origUrl) { + + if ( origUrl.indexOf('google.com/_/chrome/newtab') === -1 ) { + var onBlockedRobotsMessage = function() { + var log; + log = _logGen.log('invalid-robotstxt', []); + log.doc.finalPageUrl = log.doc.url; + log.doc.url = exports.utils.normalizeUrl(origUrl); + + sendFcn(log); + }; + return onBlockedRobotsMessage; + } else { + return function() {}; + } + }, + + init: function(onAdFound) { + + if ( exports.utils.SCRIPT_IN_FRIENDLY_IFRAME ) { + return false; + } + + _onAdFound = onAdFound; + if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { + var log = _logGen.log('page'); + onAdFound(log); + + window.addEventListener('beforeunload', function(event) { + var log = _logGen.log('unload'); + log.timing = window.performance.timing; + onAdFound(log); + }); + + addBackgroundListener('new-video-ad', onVideoMessage); + addBackgroundListener('new-invalid-video-ad', onVideoMessage); + + } + + exports.utils.onDocLoaded(document, extractAdsWrapper); + } + }; + +})(exports); + +if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { + window.adparser = { + init: exports.coordinator.init, + addPostMessageListener: exports.coordinator.addPostMessageListener, + askIfTrackingEnabled: exports.utils.askIfTrackingEnabled, + blockedRobotsMsgGen: exports.coordinator.blockedRobotsMsgGen, + inWindowTop: exports.utils.SCRIPT_IN_WINDOW_TOP, + sendToBackground: exports.utils.sendToBackground + }; +} else { + exports.coordinator.addPostMessageListener(); + exports.utils.askIfTrackingEnabled( + function() { + exports.coordinator.init(function() {}); + }, + function() {} + ); +} +})(window); +(function(adparser, pageUrl) { + function onAdFound(log) { + adparser.sendToBackground({ id: 'ad_log', subject: log }, 'ad_log', '', function(){}); + } + + if ( adparser && adparser.inWindowTop ) { + adparser.addPostMessageListener(); + adparser.askIfTrackingEnabled( + function() { + adparser.init(onAdFound); + }, + adparser.blockedRobotsMsgGen(onAdFound, pageUrl) + ) + } +})(window.adparser, window.location.href); diff --git a/src/drivers/webextension/js/lib/network.js b/src/drivers/webextension/js/lib/network.js index fc0a22d16..7c46fcf9f 100644 --- a/src/drivers/webextension/js/lib/network.js +++ b/src/drivers/webextension/js/lib/network.js @@ -1,794 +1,822 @@ -(function () { - function isChrome() { - return (typeof chrome !== 'undefined' - && window.navigator.userAgent.match(/Chrom(?:e|ium)\/([0-9\.]+)/)); - } - - let browserProxy; - if (isChrome()) { - browserProxy = chrome; - } else { - browserProxy = browser; - } - - const MIN_FF_MAJOR_VERSION = 51; - - const requiredBrowserApis = [ - browserProxy.webNavigation, - browserProxy.tabs, - browserProxy.webRequest, - browserProxy.runtime, - ]; - let areListenersRegistered = false; - const secBefore = 2000; - const secAfter = 5000; - const secBetweenDupAssets = 10e3; - const minVidSize = 500e3; - const maxVidSize = 25e6; - const maxContentRange = 25e6; - const videoExtensions = [ - 'af', '3gp', 'asf', 'avchd', 'avi', 'cam', 'dsh', 'flv', 'm1v', 'm2v', - 'fla', 'flr', 'sol', 'm4v', 'mkv', 'wrap', 'mng', 'mov', 'mpeg', 'mpg', - 'mpe', 'mp4', 'mxf', 'nsv', 'ogg', 'rm', 'svi', 'smi', 'wmv', 'webm', - ]; - const extensionsReg = new RegExp(`\\.${videoExtensions.join('$|\\.')}$`); - const videoContentTypesPrefixes = ['binary/octet-stream', 'video/', 'flv-application/', 'media']; - - const bannedContentTypes = ['video/mp2t', 'video/f4m', 'video/f4f']; - const bannedFiletypes = ['ts']; - const bannedFiletypesReg = new RegExp(`\\.${bannedFiletypes.join('$|\\.')}$`); - const whitelistReqTypes = ['object', 'xmlhttprequest', 'other']; - - const topVideoAssetDomains = [ - '2mdn.net', - 'adap.tv', - 'adnxs.com', - 'adsrvr.org', - 'btrll.com', - 'celtra.com', - 'flashtalking.com', - 'flite.com', - 'innovid.com', - 'jivox.com', - 'mixpo.com', - 'nytimes.com', - 'playwire.com', - 'selectmedia.asia', - 'serving-sys.com', - 'solvemedia.com', - 'spotible.com', - 'teads.tv', - 'tribalfusion.com', - 'tubemogul.com', - 'videologygroup.com', - 'washingtonpost.com', - ]; - - const robotsTxtAllows = wappalyzer.robotsTxtAllows.bind(wappalyzer); - if (!String.prototype.endsWith) { - String.prototype.endsWith = function (searchString, position) { - const subjectString = this.toString(); - if (typeof position !== 'number' || !isFinite(position) - || Math.floor(position) !== position || position > subjectString.length) { - position = subjectString.length; - } - position -= searchString.length; - const lastIndex = subjectString.indexOf(searchString, position); - return lastIndex !== -1 && lastIndex === position; - }; - } - - function getFrame(getFrameDetails, callback) { - if (isChrome()) { - chrome.webNavigation.getFrame(getFrameDetails, callback); - } else if (typeof browser !== 'undefined') { - const gettingFrame = browser.webNavigation.getFrame(getFrameDetails); - gettingFrame.then(callback); - } - } - - function ifBrowserValid(callback, elseCallback) { - if (isChrome()) { - callback(); - } else if (typeof browser !== 'undefined') { - try { - const gettingInfo = browser.runtime.getBrowserInfo(); - gettingInfo.then((browserInfo) => { - const browserVersion = parseInt(browserInfo.version.split('.')[0]); - - if (browserInfo.name === 'Firefox' - && browserVersion >= MIN_FF_MAJOR_VERSION) { - callback(); - } else { - elseCallback(); - } - }); - } catch (err) { - elseCallback(); - } - } else { - elseCallback(); - } - } - - function ifTrackingEnabled(details, ifCallback, elseCallback) { - const fullIfCallback = function () { - allowedByRobotsTxt(details, ifCallback, elseCallback); - }; - - browser.storage.local.get('tracking').then((item) => { - if (item.hasOwnProperty('tracking')) { - if (item.tracking) { - fullIfCallback(); - } else { - elseCallback(); - } - } else { - fullIfCallback(); - } - }); - } - - function allowedByRobotsTxt(details, ifCallback, elseCallback) { - if (details.url && !details.url.startsWith('chrome://')) { - robotsTxtAllows(details.url).then(ifCallback, elseCallback); - } else { - elseCallback(); - } - } - - function isPixelRequest(request) { - return (request.type === 'image' || request.responseStatus === 204) - && request.size <= 1000; - } - - function isVpaidOrVastRequest(request) { - const lowerCaseUrl = request.url.toLowerCase(); - return lowerCaseUrl.indexOf('vpaid') !== -1 || lowerCaseUrl.indexOf('vast') !== -1; - } - - function hasValidRequestType(request) { - return whitelistReqTypes.indexOf(request.type) >= 0; - } - - function stripQueryParams(url) { - return url.split('?', 1)[0]; - } - - function parseHostnameFromUrl(url) { - const parser = document.createElement('a'); - parser.href = url; - return parser.hostname; - } - - function hasDomain(url, domain) { - return parseHostnameFromUrl(url).endsWith(domain); - } - - function findHeader(headers, key) { - let header; - for (let i = 0; i < headers.length; i += 1) { - header = headers[i]; - if (header.name.toLowerCase() === key) { - return header; - } - } - return null; - } - - function validVideoType(vtype) { - const goodType = videoContentTypesPrefixes.some(prefix => vtype.indexOf(prefix) === 0); - return goodType; - } - - function assetMsgKey(assetReq) { - const url = stripQueryParams(assetReq.url); - const key = `${assetReq.frameId}-${url}`; - return key; - } - - const PageNetworkTrafficCollector = function (tabId) { - this.tabId = tabId; - this.displayAdFound = false; - this.requests = {}; - this.msgsBeingSent = {}; - this.assetsSeen = {}; - this.allRedirects = {}; - }; - - var globalPageContainer = { - collectors: {}, - dyingCollectors: {}, - - cleanupCollector(tabId) { - if (tabId in this.collectors) { - delete globalPageContainer.collectors[tabId]; - } - }, - - onNewNavigation(details) { - const tabId = details.tabId; - this.cleanupCollector(tabId); - - ifTrackingEnabled( - details, - () => { - if (!areListenersRegistered) { - registerListeners(); - } - this.collectors[tabId] = new PageNetworkTrafficCollector(tabId); - }, - () => { - if (areListenersRegistered) { - unregisterListeners(); - } - }, - ); - }, - - onNavigationCommitted(details) { - - }, - - onNavigationCompleted(details) { - - }, - - onTabClose(tabId, closeInfo) { - this.cleanupCollector(tabId); - delete this.collectors[tabId]; - }, - - onDisplayAdFound(tabId) { - this.collectors[tabId].displayAdFound = true; - }, - - getRandId() { - return String(Math.floor(Math.random() * 1e9)); - }, - - getCollector(tabId) { - if (this.collectors.hasOwnProperty(tabId)) { - return this.collectors[tabId]; - } - return null; - }, - - forwardCall(details, collectorMemberFunction) { - const collector = this.getCollector(details.tabId); - if (collector !== null) { - collectorMemberFunction.apply(collector, [details]); - } - }, - }; - - PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function () { - const logMessage = Array.from(arguments).join(' '); - const message = { message: logMessage, event: 'console-log-message' }; - browserProxy.tabs.sendMessage(this.tabId, message); - }; - - PageNetworkTrafficCollector.prototype.sendToTab = function (assetReq, reqs, curPageUrl, adTrackingEvent) { - const msg = {}; - msg.assets = []; - msg.requests = []; - msg.event_data = {}; - msg.event = adTrackingEvent; - if (adTrackingEvent === 'new-video-ad') { - msg.requests = reqs; - msg.requests.sort((reqA, reqB) => reqA.requestTimestamp - reqB.requestTimestamp); - if (assetReq) { - msg.assets = [assetReq]; - } - } else if (adTrackingEvent === 'new-invalid-video-ad') { - msg.requests = reqs.map(request => parseHostnameFromUrl(request.url)); - msg.assets = [{ - - url: parseHostnameFromUrl(assetReq.url), - - contentType: assetReq.contentType, - size: assetReq.size, - }]; - } - msg.origUrl = curPageUrl; - msg.displayAdFound = this.displayAdFound; - - browserProxy.tabs.sendMessage(this.tabId, msg); - }; - - PageNetworkTrafficCollector.prototype.getRedirKey = function (url, frameId) { - return `${url}:${frameId}`; - }; - - PageNetworkTrafficCollector.prototype.seenBefore = function (request) { - const oldTime = this.assetsSeen[assetMsgKey(request)]; - if (oldTime && (request.requestTimestamp - oldTime < secBetweenDupAssets)) { - return true; - } - return false; - }; - - PageNetworkTrafficCollector.prototype.recordSeenAsset = function (request) { - this.assetsSeen[assetMsgKey(request)] = request.requestTimestamp; - }; - - PageNetworkTrafficCollector.prototype.onBeforeRequest = function (details) { - const req = { - url: details.url, - type: details.type, - httpMethod: details.method, - frameId: details.frameId, - parentFrameId: details.parentFrameId, - requestTimestamp: details.timeStamp, - }; - this.requests[details.requestId] = req; - }; - - PageNetworkTrafficCollector.prototype.onSendHeaders = function (details) { - let request, - header; - request = this.requests[details.requestId]; - header = request && findHeader(details.requestHeaders, 'x-requested-with'); - if (header && header.value.toLowerCase().indexOf('flash') > -1) { - request.from_flash = true; - } - }; - - PageNetworkTrafficCollector.prototype.onHeadersReceived = function (details) { - const getFrameDetails = { - tabId: details.tabId, - processId: null, - frameId: details.frameId, - }; - const pageNetworkTrafficController = this; - getFrame(getFrameDetails, (frameDetails) => { - if (frameDetails && frameDetails.url) { - pageNetworkTrafficController._onHeadersReceived(details, frameDetails); - } - }); - }; - - PageNetworkTrafficCollector.prototype._onHeadersReceived = function (details, frameDetails) { - let contentSize, - contentRange; - - const request = this.requests[details.requestId]; - if (request) { - const redirParent = this.allRedirects[this.getRedirKey(details.url, details.frameId)]; - let header = request && findHeader(details.responseHeaders, 'content-type'); - const contentType = header && header.value.toLowerCase(); - - if (contentType) { - request.contentType = contentType; - } - header = request && findHeader(details.responseHeaders, 'content-length'); - contentSize = header && header.value; - if (contentSize) { - request.size = request.size || 0; - request.size += parseInt(contentSize); - } - header = request && findHeader(details.responseHeaders, 'content-range'); - contentRange = header && header.value; - if (contentRange) { - request.contentRange = parseInt(contentRange.split('/')[1]); - } - - let frameUrl = null; - if (frameDetails && frameDetails.url) { - frameUrl = frameDetails.url; - } - if (!this.bannedRequest(request) - && (this.isVideoReq(frameUrl, request) || (redirParent && redirParent.isVideo))) { - request.isVideo = true; - } - } - }; - - PageNetworkTrafficCollector.prototype.onBeforeRedirect = function (details) { - const request = this.requests[details.requestId]; - if (request) { - if (request.redirects) { - request.redirects.push(details.redirectUrl); - } else { - request.redirects = [details.redirectUrl]; - } - this.allRedirects[this.getRedirKey(details.redirectUrl, details.frameId)] = request; - } - }; - - PageNetworkTrafficCollector.prototype.isYoutubeMastheadRequest = function (url) { - const re = /video_masthead/; - return this.hasYoutubeDomain(url) && re.test(url); - }; - PageNetworkTrafficCollector.prototype.isYoutubeVideoRequest = function (srcUrl, destUrl) { - if (!this.hasYoutubeDomain(srcUrl)) { - return false; - } - - const re = /https?:\/\/r.*?\.googlevideo\.com\/videoplayback\?/; - return re.test(destUrl); - }; - PageNetworkTrafficCollector.prototype.processResponse = function (requestDetails, frameDetails) { - let request; - if (requestDetails) { - request = this.requests[requestDetails.requestId]; - if (request) { - request.responseStatus = requestDetails.statusCode; - request.responseTimestamp = requestDetails.timeStamp; - - let frameUrl = null; - if (frameDetails && frameDetails.url) { - frameUrl = frameDetails.url; - } - - let requestUrl = null; - if (request.url) { - requestUrl = request.url; - } - - if (this.isYoutubeAdReq(frameUrl, requestUrl)) { - const destVideoId = this.parseYoutubeVideoIdFromUrl(requestUrl); - const srcVideoId = this.parseYoutubeVideoIdFromUrl(frameUrl); - if (srcVideoId && destVideoId) { - request.isYoutubeAd = true; - request.isVideo = true; - request.rawSrcUrl = frameUrl; - request.rawDestUrl = requestUrl; - request.url = `https://www.youtube.com/watch?v=${this.parseYoutubeVideoIdFromUrl(requestUrl)}`; - } - } else if (!this.bannedRequest(request) - && (this.isVideo || this.isVideoReq(frameUrl, request))) { - request.isVideo = true; - } - - if (request.isVideo) { - const msgKey = assetMsgKey(request); - this.msgsBeingSent[msgKey] = request; - if (!this.seenBefore(request)) { - this.sendMsgWhenQuiet(msgKey); - } - this.recordSeenAsset(request); - } - } - } - }; - - PageNetworkTrafficCollector.prototype.onResponseStarted = function (responseDetails) { - if (responseDetails.frameId < 0) { - responseDetails.frameId = 99999; - } - const getFrameDetails = { - tabId: responseDetails.tabId, - processId: null, - frameId: responseDetails.frameId, - }; - const pageNetworkTrafficController = this; - getFrame(getFrameDetails, (frameDetails) => { - if (frameDetails && frameDetails.url) { - pageNetworkTrafficController.processResponse(responseDetails, frameDetails); - } - }); - }; - - PageNetworkTrafficCollector.prototype.hasBannedFiletype = function (request) { - const url = stripQueryParams(request.url); - if (bannedFiletypesReg.exec(url)) { - return true; - } - return false; - }; - - PageNetworkTrafficCollector.prototype.checkContentHeaders = function (request) { - if (request.contentType && validVideoType(request.contentType)) { - return true; - } - return false; - }; - - PageNetworkTrafficCollector.prototype.checkUrlExtension = function (request) { - const url = stripQueryParams(request.url); - if (extensionsReg.exec(url)) { - return true; - } - return false; - }; - - PageNetworkTrafficCollector.prototype.isVideoReq = function (srcUrl, request) { - if (this.isYoutubeVideoRequest(srcUrl, request.url)) { - return false; - } - return this.checkUrlExtension(request) || this.checkContentHeaders(request); - }; - PageNetworkTrafficCollector.prototype.hasYoutubeDomain = function (url) { - const hostname = parseHostnameFromUrl(url); - if (hostname === 'www.youtube.com') { - return true; - } - return false; - }; - PageNetworkTrafficCollector.prototype.parseYoutubeVideoIdFromUrl = function (url) { - let re = /^https?:\/\/www\.youtube\.com\/get_video_info.*(?:\?|&)video_id=(.*?)(?:$|&)/; - let match = re.exec(url); - if (match && match.length > 1) { - return match[1]; - } - - re = /^https?:\/\/www\.youtube\.com\/embed\/(.*?)(?:$|\?)/; - match = re.exec(url); - if (match && match.length > 1) { - return match[1]; - } - - re = /^https?:\/\/www\.youtube\.com\/watch.*(\?|&)v=([^&]*)/; - match = re.exec(url); - if (match && match.length > 1) { - return match[1]; - } - return null; - }; - - PageNetworkTrafficCollector.prototype.isYoutubeGetVideoInfoReq = function (url) { - const re = /^https?:\/\/www\.youtube\.com\/get_video_info\?/; - return re.test(url); - }; - PageNetworkTrafficCollector.prototype.isYoutubeAdReq = function (srcUrl, destUrl) { - if (!this.hasYoutubeDomain(srcUrl) - || !this.isYoutubeGetVideoInfoReq(destUrl)) { - return false; - } - if (this.parseYoutubeVideoIdFromUrl(srcUrl) - === this.parseYoutubeVideoIdFromUrl(destUrl) - && !this.isYoutubeMastheadRequest(destUrl)) { - return false; - } - return true; - }; - - PageNetworkTrafficCollector.prototype.bannedRequest = function (request) { - return this.bannedVideoType(request) || this.hasBannedFiletype(request) || this.bannedVideoSize(request); - }; - - PageNetworkTrafficCollector.prototype.bannedVideoType = function (request) { - let badType = false; - if (request.contentType) { - badType = bannedContentTypes.some(prefix => request.contentType.indexOf(prefix) >= 0); - } - return badType; - }; - - PageNetworkTrafficCollector.prototype.bannedVideoSize = function (request) { - if (request.size !== null) { - if (request.size < minVidSize || request.size > maxVidSize || request.contentRange > maxContentRange) { - return true; - } - } - return false; - }; - - PageNetworkTrafficCollector.prototype.grabTagReqs = function (tabRequests, assetRequest) { - let minTimestamp, - maxTimestamp; - minTimestamp = assetRequest.requestTimestamp - secBefore; - maxTimestamp = assetRequest.requestTimestamp + secAfter; - - const filteredRequests = tabRequests.filter(request => (request.requestTimestamp > minTimestamp - && request.requestTimestamp < maxTimestamp - && request.frameId === assetRequest.frameId - && request.url !== assetRequest.url - && (hasValidRequestType(request) - || isPixelRequest(request)))); - - return filteredRequests; - }; - - PageNetworkTrafficCollector.prototype.isValidVideoAd = function (assetRequest, tagRequests) { - const hasVpaidOrVastRequest = tagRequests.some(tagRequest => isVpaidOrVastRequest(tagRequest)); - - if (assetRequest.isYoutubeAd) { - return true; - } - if (hasVpaidOrVastRequest) { - return true; - } - const hasTopVideoAssetDomain = topVideoAssetDomains.some(assetDomain => hasDomain(assetRequest.url, assetDomain)); - - return hasTopVideoAssetDomain; - }; - - PageNetworkTrafficCollector.prototype.sendMsgWhenQuiet = function (msgKey) { - let _this = this, - origPageUrl, - msgAssetReq; - msgAssetReq = this.msgsBeingSent[msgKey]; - browserProxy.tabs.get(this.tabId, (tab) => { origPageUrl = tab.url; }); - - setTimeout(() => { - const rawRequests = []; - if (globalPageContainer.collectors[_this.tabId] === _this) { - for (const reqId in _this.requests) { - rawRequests.push(_this.requests[reqId]); - } - const tagReqs = _this.grabTagReqs(rawRequests, msgAssetReq); - - if (_this.isValidVideoAd(msgAssetReq, tagReqs)) { - _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, 'new-video-ad'); - } else { - _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, 'new-invalid-video-ad'); - } - } else { - - } - delete _this.msgsBeingSent[msgKey]; - }, secAfter + secBefore); - }; - - PageNetworkTrafficCollector.prototype.existingMessage = function (candidateRequest) { - const frameMsg = this.msgsBeingSent[candidateRequest.frameId]; - if (frameMsg) { - return frameMsg; - } - return null; - }; - - function onBeforeRequestListener(details) { - globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRequest); - } - - function onSendHeadersListener(details) { - globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onSendHeaders); - } - - function onHeadersReceivedListener(details) { - globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onHeadersReceived); - } - - function onBeforeRedirectListener(details) { - globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRedirect); - } - - function onResponseStartedListener(details) { - globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onResponseStarted); - } - - function onCommittedListener(details) { - if (details.frameId === 0) { - globalPageContainer.onNavigationCommitted(details); - } - } - - function onCompletedListener(details) { - if (details.frameId === 0) { - globalPageContainer.onNavigationCompleted(details); - } - } - - function onRemovedListener(tabId, closeInfo) { - globalPageContainer.onTabClose(tabId, closeInfo); - } - - function onMessageListener(message, sender, sendResponse) { - if (message.event === 'new-ad' && message.data.event === 'ad') { - const tabId = sender.tab.id; - if (tabId) { - globalPageContainer.onDisplayAdFound(tabId); - } - } - } - - function registerListeners() { - browserProxy.webRequest.onBeforeRequest.addListener( - onBeforeRequestListener, - { urls: ['http://*/*', 'https://*/*'] }, - [], - ); - - browserProxy.webRequest.onSendHeaders.addListener( - onSendHeadersListener, - { urls: ['http://*/*', 'https://*/*'] }, - ['requestHeaders'], - ); - - browserProxy.webRequest.onHeadersReceived.addListener( - onHeadersReceivedListener, - { urls: ['http://*/*', 'https://*/*'] }, - ['responseHeaders'], - ); - - browserProxy.webRequest.onBeforeRedirect.addListener( - onBeforeRedirectListener, - { urls: ['http://*/*', 'https://*/*'] }, - [], - ); - - browserProxy.webRequest.onResponseStarted.addListener( - onResponseStartedListener, - { urls: ['http://*/*', 'https://*/*'] }, - ['responseHeaders'], - ); - - browserProxy.webNavigation.onCommitted.addListener(onCommittedListener); - browserProxy.webNavigation.onCompleted.addListener(onCompletedListener); - browserProxy.tabs.onRemoved.addListener(onRemovedListener); - browserProxy.runtime.onMessage.addListener(onMessageListener); - - areListenersRegistered = true; - } - - function unregisterListeners() { - browserProxy.webRequest.onBeforeRequest.removeListener( - onBeforeRequestListener, - ); - - browserProxy.webRequest.onSendHeaders.removeListener( - onSendHeadersListener, - ); - - browserProxy.webRequest.onHeadersReceived.removeListener( - onHeadersReceivedListener, - ); - - browserProxy.webRequest.onBeforeRedirect.removeListener( - onBeforeRedirectListener, - ); - - browserProxy.webRequest.onResponseStarted.removeListener( - onResponseStartedListener, - ); - - browserProxy.webNavigation.onCommitted.removeListener(onCommittedListener); - browserProxy.webNavigation.onCompleted.removeListener(onCompletedListener); - browserProxy.tabs.onRemoved.removeListener(onRemovedListener); - browserProxy.runtime.onMessage.removeListener(onMessageListener); - - areListenersRegistered = false; - } - - function areRequiredBrowserApisAvailable() { - return requiredBrowserApis.every(api => typeof api !== 'undefined'); - } - - if (areRequiredBrowserApisAvailable()) { - ifBrowserValid( - () => { - browserProxy.webNavigation.onBeforeNavigate.addListener( - (details) => { - if (details.frameId === 0) { - globalPageContainer.onNewNavigation(details); - } - }, - { - url: [{ urlMatches: 'http://*/*' }, { urlMatches: 'https://*/*' }], - }, - ); - }, () => { - - }, - ); - } - - browserProxy.runtime.onMessage.addListener((request, sender, sendResponse) => { - if (request === 'is_browser_valid') { - ifBrowserValid( - sendResponse({ browser_valid: true }), - sendResponse({ browser_valid: false }), - ); - } - }); - - browserProxy.runtime.onMessage.addListener((request, sender, sendResponse) => { - if (request === 'is_tracking_enabled') { - ifTrackingEnabled( - sender.tab, - () => { - console.log('enabled'); - try { sendResponse({ tracking_enabled: true }); } catch (err) {} - }, - () => { - console.log('not enabled'); - try { sendResponse({ tracking_enabled: false }); } catch (err) {} - }, - ); - } - return true; - }); -}()); +(function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i subjectString.length) { + position = subjectString.length; + } + position -= searchString.length; + var lastIndex = subjectString.indexOf(searchString, position); + return lastIndex !== -1 && lastIndex === position; + }; + } + + function getFrame(getFrameDetails, callback) { + var gettingFrame = browser.webNavigation.getFrame(getFrameDetails); + gettingFrame.then(callback); + } + + function ifBrowserValid(callback, elseCallback) { + if ( isChrome() ) { + + callback(); + } else if ( typeof browser !== 'undefined' ) { + try { + var gettingInfo = browser.runtime.getBrowserInfo(); + gettingInfo.then(function(browserInfo) { + var browserVersion = parseInt(browserInfo.version.split('.')[0]); + + if ( browserInfo.name === 'Firefox' && + browserVersion >= MIN_FF_MAJOR_VERSION) { + callback(); + } else { + elseCallback(); + } + }); + } catch (err) { + + elseCallback(); + } + } else { + elseCallback(); + } + } + + function ifTrackingEnabled(details, ifCallback, elseCallback) { + + var fullIfCallback = function() { + allowedByRobotsTxt(details, ifCallback, elseCallback); + }; + + browser.storage.local.get('tracking').then(function(item) { + + if ( item.hasOwnProperty('tracking') ) { + if ( item.tracking ) { + fullIfCallback(); + } else { + elseCallback(); + } + } else { + fullIfCallback(); + } + }); + + } + + function allowedByRobotsTxt(details, ifCallback, elseCallback) { + if ( details.url && !details.url.startsWith('chrome://') ) { + robotsTxtAllows(details.url).then(ifCallback, elseCallback); + } else { + elseCallback(); + } + } + + function isPixelRequest(request) { + return (request.type === 'image' || request.responseStatus === 204) && + request.size <= 1000; + } + + function isVpaidOrVastRequest(request) { + var lowerCaseUrl = request.url.toLowerCase(); + return lowerCaseUrl.indexOf('vpaid') !== -1 || lowerCaseUrl.indexOf('vast') !== -1; + } + + function hasValidRequestType(request) { + return whitelistReqTypes.indexOf(request.type) >= 0; + } + + function stripQueryParams(url) { + return url.split('?', 1)[0]; + } + + function parseHostnameFromUrl(url) { + var parser = document.createElement('a'); + parser.href = url; + return parser.hostname; + } + + function hasDomain(url, domain) { + return parseHostnameFromUrl(url).endsWith(domain); + } + + function findHeader(headers, key) { + var header; + for ( var i = 0; i < headers.length; i += 1 ) { + header = headers[i]; + if ( header.name.toLowerCase() === key ) { + return header; + } + } + return null; + } + + function validVideoType(vtype) { + var goodType = videoContentTypesPrefixes.some(function(prefix) { + return vtype.indexOf(prefix) === 0; + }); + return goodType; + } + + function assetMsgKey(assetReq) { + var url = stripQueryParams(assetReq.url); + var key = assetReq.frameId + '-' + url; + return key; + } + + var PageNetworkTrafficCollector = function(tabId) { + this.tabId = tabId; + this.displayAdFound = false; + this.requests = {}; + this.msgsBeingSent = {}; + this.assetsSeen = {}; + this.allRedirects = {}; + }; + + var globalPageContainer = { + collectors: {}, + dyingCollectors: {}, + + cleanupCollector: function(tabId) { + if ( tabId in this.collectors ) { + delete globalPageContainer.collectors[tabId]; + } + }, + + onNewNavigation: function(details) { + var tabId = details.tabId; + this.cleanupCollector(tabId); + + ifTrackingEnabled( + details, + function() { + if ( !areListenersRegistered ) { + + registerListeners(); + } + this.collectors[tabId] = new PageNetworkTrafficCollector(tabId); + }.bind(this), + function() { + if ( areListenersRegistered ) { + + unregisterListeners(); + } + } + ); + }, + + onNavigationCommitted: function(details) { + + }, + + onNavigationCompleted: function(details) { + + }, + + onTabClose: function(tabId, closeInfo) { + + this.cleanupCollector(tabId); + delete this.collectors[tabId]; + }, + + onDisplayAdFound: function(tabId) { + this.collectors[tabId].displayAdFound = true; + }, + + getRandId: function() { + return String(Math.floor(Math.random() * 1e9)); + }, + + getCollector: function(tabId) { + if ( this.collectors.hasOwnProperty(tabId) ) { + return this.collectors[tabId]; + } + return null; + }, + + forwardCall: function(details, collectorMemberFunction) { + var collector = this.getCollector(details.tabId); + if ( collector !== null ) { + collectorMemberFunction.apply(collector, [details]); + } + } + }; + + PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() { + var logMessage = Array.from(arguments).join(' '); + var message = {message: logMessage, event: 'console-log-message'}; + browser.tabs.sendMessage(this.tabId, message); + }; + + PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, adTrackingEvent) { + var msg = {}; + msg.assets = []; + msg.requests = []; + msg.event_data = {}; + msg.event = adTrackingEvent; + if ( adTrackingEvent === 'new-video-ad' ) { + msg.requests = reqs; + msg.requests.sort(function(reqA, reqB) {return reqA.requestTimestamp - reqB.requestTimestamp;}); + if ( assetReq ) { + msg.assets = [assetReq]; + } + } else if ( adTrackingEvent === 'new-invalid-video-ad' ) { + msg.requests = reqs.map(function(request) { + return parseHostnameFromUrl(request.url); + }); + msg.assets = [{ + + url: parseHostnameFromUrl(assetReq.url), + + contentType: assetReq.contentType, + size: assetReq.size + }]; + } + msg.origUrl = curPageUrl; + msg.displayAdFound = this.displayAdFound; + + browser.tabs.sendMessage(this.tabId, msg); + }; + + PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) { + return url + ':' + frameId; + }; + + PageNetworkTrafficCollector.prototype.seenBefore = function(request) { + var oldTime = this.assetsSeen[assetMsgKey(request)]; + if ( oldTime && (request.requestTimestamp-oldTime < secBetweenDupAssets)){ + + return true; + } + return false; + }; + + PageNetworkTrafficCollector.prototype.recordSeenAsset = function(request) { + this.assetsSeen[assetMsgKey(request)] = request.requestTimestamp; + }; + + PageNetworkTrafficCollector.prototype.onBeforeRequest = function(details) { + var req = { + url: details.url, + type: details.type, + httpMethod: details.method, + frameId: details.frameId, + parentFrameId: details.parentFrameId, + requestTimestamp: details.timeStamp, + }; + this.requests[details.requestId] = req; + }; + + PageNetworkTrafficCollector.prototype.onSendHeaders = function(details) { + var request, header; + request = this.requests[details.requestId]; + header = request && findHeader(details.requestHeaders, 'x-requested-with'); + if ( header && header.value.toLowerCase().indexOf('flash') > -1 ) { + request.from_flash = true; + } + }; + + PageNetworkTrafficCollector.prototype.onHeadersReceived = function(details) { + var getFrameDetails = { + tabId: details.tabId, + processId: null, + frameId: details.frameId + }; + var pageNetworkTrafficController = this; + getFrame(getFrameDetails, function(frameDetails) { + if ( frameDetails && frameDetails.url ) { + pageNetworkTrafficController._onHeadersReceived(details, frameDetails); + } + }); + }; + + PageNetworkTrafficCollector.prototype._onHeadersReceived = function(details, frameDetails) { + var contentSize, contentRange; + + var request = this.requests[details.requestId]; + if ( request ) { + var redirParent = this.allRedirects[this.getRedirKey(details.url, details.frameId)]; + var header = request && findHeader(details.responseHeaders, 'content-type'); + var contentType = header && header.value.toLowerCase(); + + if ( contentType){ + request.contentType = contentType; + } + header = request && findHeader(details.responseHeaders, 'content-length'); + contentSize = header && header.value; + if ( contentSize ) { + request.size = request.size || 0; + request.size += parseInt(contentSize); + } + header = request && findHeader(details.responseHeaders, 'content-range'); + contentRange = header && header.value; + if ( contentRange ) { + request.contentRange = parseInt(contentRange.split('/')[1]); + } + + var frameUrl = null; + if ( frameDetails && frameDetails.url ) { + frameUrl = frameDetails.url; + } + if ( !this.bannedRequest(request) && + (this.isVideoReq(frameUrl, request) || (redirParent && redirParent.isVideo))) { + request.isVideo = true; + } + } + }; + + PageNetworkTrafficCollector.prototype.onBeforeRedirect = function(details) { + var request = this.requests[details.requestId]; + if ( request ) { + if ( request.redirects ) { + request.redirects.push(details.redirectUrl); + } else { + request.redirects = [details.redirectUrl]; + } + this.allRedirects[this.getRedirKey(details.redirectUrl, details.frameId)] = request; + } + }; + + PageNetworkTrafficCollector.prototype.isYoutubeMastheadRequest = function(url) { + var re = /video_masthead/; + return this.hasYoutubeDomain(url) && re.test(url); + }; + PageNetworkTrafficCollector.prototype.isYoutubeVideoRequest = function(srcUrl, destUrl) { + if ( !this.hasYoutubeDomain(srcUrl) ) { + return false; + } + + var re = /https?:\/\/r.*?\.googlevideo\.com\/videoplayback\?/; + return re.test(destUrl); + }; + PageNetworkTrafficCollector.prototype.processResponse = function(requestDetails, frameDetails) { + var request; + if ( requestDetails ) { + request = this.requests[requestDetails.requestId]; + if ( request ) { + request.responseStatus = requestDetails.statusCode; + request.responseTimestamp = requestDetails.timeStamp; + + var frameUrl = null; + if ( frameDetails && frameDetails.url ) { + frameUrl = frameDetails.url; + } + + var requestUrl = null; + if ( request.url ) { + requestUrl = request.url; + } + + if ( this.isYoutubeAdReq(frameUrl, requestUrl) ) { + var destVideoId = this.parseYoutubeVideoIdFromUrl(requestUrl); + var srcVideoId = this.parseYoutubeVideoIdFromUrl(frameUrl); + if ( srcVideoId && destVideoId ) { + request.isYoutubeAd = true; + request.isVideo = true; + request.rawSrcUrl = frameUrl; + request.rawDestUrl = requestUrl; + request.url = 'https://www.youtube.com/watch?v=' + this.parseYoutubeVideoIdFromUrl(requestUrl); + } + } else if ( !this.bannedRequest(request) && + (this.isVideo || this.isVideoReq(frameUrl, request))) { + request.isVideo = true; + } + + if ( request.isVideo ) { + + var msgKey = assetMsgKey(request); + this.msgsBeingSent[msgKey] = request; + if ( !this.seenBefore(request) ) { + this.sendMsgWhenQuiet(msgKey); + } + this.recordSeenAsset(request); + } + } + } + }; + + PageNetworkTrafficCollector.prototype.onResponseStarted = function(responseDetails) { + if ( responseDetails.frameId < 0 ) { + responseDetails.frameId = 99999; + + } + var getFrameDetails = { + tabId: responseDetails.tabId, + processId: null, + frameId: responseDetails.frameId + }; + var pageNetworkTrafficController = this; + getFrame(getFrameDetails, function(frameDetails) { + if ( frameDetails && frameDetails.url ) { + pageNetworkTrafficController.processResponse(responseDetails, frameDetails); + } + }); + }; + + PageNetworkTrafficCollector.prototype.hasBannedFiletype = function(request) { + var url = stripQueryParams(request.url); + if ( bannedFiletypesReg.exec(url) ) { + return true; + } else { + return false; + } + }; + + PageNetworkTrafficCollector.prototype.checkContentHeaders = function(request) { + if ( request.contentType && validVideoType(request.contentType) ) { + return true; + } + return false; + }; + + PageNetworkTrafficCollector.prototype.checkUrlExtension = function(request) { + var url = stripQueryParams(request.url); + if ( extensionsReg.exec(url) ) { + return true; + } else { + return false; + } + }; + + PageNetworkTrafficCollector.prototype.isVideoReq = function(srcUrl, request) { + if ( this.isYoutubeVideoRequest(srcUrl, request.url) ) { + return false; + } + return this.checkUrlExtension(request) || this.checkContentHeaders(request); + }; + PageNetworkTrafficCollector.prototype.hasYoutubeDomain = function(url) { + var hostname = parseHostnameFromUrl(url) ; + if ( hostname === 'www.youtube.com' ) { + return true; + } + return false; + }; + PageNetworkTrafficCollector.prototype.parseYoutubeVideoIdFromUrl = function(url) { + var re = /^https?:\/\/www\.youtube\.com\/get_video_info.*(?:\?|&)video_id=(.*?)(?:$|&)/; + var match = re.exec(url); + if ( match && match.length > 1 ) { + return match[1]; + } + + re = /^https?:\/\/www\.youtube\.com\/embed\/(.*?)(?:$|\?)/; + match = re.exec(url); + if ( match && match.length > 1 ) { + return match[1]; + } + + re = /^https?:\/\/www\.youtube\.com\/watch.*(\?|&)v=([^&]*)/; + match = re.exec(url); + if ( match && match.length > 1 ) { + return match[1]; + } + return null; + }; + + PageNetworkTrafficCollector.prototype.isYoutubeGetVideoInfoReq = function(url) { + var re = /^https?:\/\/www\.youtube\.com\/get_video_info\?/; + return re.test(url); + }; + PageNetworkTrafficCollector.prototype.isYoutubeAdReq = function(srcUrl, destUrl) { + + if ( !this.hasYoutubeDomain(srcUrl) || + !this.isYoutubeGetVideoInfoReq(destUrl)) { + return false; + } + if ( this.parseYoutubeVideoIdFromUrl(srcUrl) === + this.parseYoutubeVideoIdFromUrl(destUrl) && + !this.isYoutubeMastheadRequest(destUrl)) { + return false; + } + return true; + }; + + PageNetworkTrafficCollector.prototype.bannedRequest = function(request) { + return this.bannedVideoType(request) || this.hasBannedFiletype(request) || this.bannedVideoSize(request); + }; + + PageNetworkTrafficCollector.prototype.bannedVideoType = function(request) { + var badType = false; + if ( request.contentType ) { + badType = bannedContentTypes.some(function(prefix) { + return request.contentType.indexOf(prefix) >= 0; + }); + } + return badType; + }; + + PageNetworkTrafficCollector.prototype.bannedVideoSize = function(request) { + if ( request.size !== null ) { + if ( request.size < minVidSize || request.size > maxVidSize || request.contentRange > maxContentRange ) { + return true; + } + } + return false; + }; + + PageNetworkTrafficCollector.prototype.grabTagReqs = function(tabRequests, assetRequest) { + var minTimestamp, maxTimestamp; + minTimestamp = assetRequest.requestTimestamp - secBefore; + maxTimestamp = assetRequest.requestTimestamp + secAfter; + + var filteredRequests = tabRequests.filter(function(request) { + return (request.requestTimestamp > minTimestamp && + request.requestTimestamp < maxTimestamp && + request.frameId === assetRequest.frameId && + request.url !== assetRequest.url && + (hasValidRequestType(request) || + isPixelRequest(request))); + }); + + return filteredRequests; + }; + + PageNetworkTrafficCollector.prototype.isValidVideoAd = function(assetRequest, tagRequests) { + var hasVpaidOrVastRequest = tagRequests.some(function(tagRequest) { + return isVpaidOrVastRequest(tagRequest); + }); + + if ( assetRequest.isYoutubeAd ) { + return true; + } + if ( hasVpaidOrVastRequest ) { + return true; + } + var hasTopVideoAssetDomain = topVideoAssetDomains.some(function(assetDomain) { + return hasDomain(assetRequest.url, assetDomain); + }); + + return hasTopVideoAssetDomain; + }; + + PageNetworkTrafficCollector.prototype.sendMsgWhenQuiet = function(msgKey) { + var _this = this, + origPageUrl, msgAssetReq; + msgAssetReq = this.msgsBeingSent[msgKey]; + browser.tabs.get(this.tabId).then(function(tab) { + origPageUrl = tab.url; + }); + + setTimeout(function() { + var rawRequests = []; + if ( globalPageContainer.collectors[_this.tabId] === _this ) { + for ( var reqId in _this.requests ) { + rawRequests.push(_this.requests[reqId]); + } + var tagReqs = _this.grabTagReqs(rawRequests, msgAssetReq); + + if ( _this.isValidVideoAd(msgAssetReq, tagReqs) ) { + _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, 'new-video-ad'); + } else { + + _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, 'new-invalid-video-ad'); + } + + } else { + + } + delete _this.msgsBeingSent[msgKey]; + }, secAfter+secBefore); + }; + + PageNetworkTrafficCollector.prototype.existingMessage = function(candidateRequest) { + var frameMsg = this.msgsBeingSent[candidateRequest.frameId]; + if ( frameMsg ) { + return frameMsg; + } else { + return null; + } + }; + + function onBeforeRequestListener(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRequest); + } + + function onSendHeadersListener(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onSendHeaders); + } + + function onHeadersReceivedListener(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onHeadersReceived); + } + + function onBeforeRedirectListener(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRedirect); + } + + function onResponseStartedListener(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onResponseStarted); + } + + function onCommittedListener(details) { + if ( details.frameId === 0 ) { + globalPageContainer.onNavigationCommitted(details); + } + } + + function onCompletedListener(details) { + if ( details.frameId === 0 ) { + globalPageContainer.onNavigationCompleted(details); + } + } + + function onRemovedListener(tabId, closeInfo) { + globalPageContainer.onTabClose(tabId, closeInfo); + } + + function onMessageListener(message, sender, sendResponse) { + if ( message.event === 'new-ad' && message.data.event === 'ad' ) { + var tabId = sender.tab.id; + if ( tabId ) { + globalPageContainer.onDisplayAdFound(tabId); + } + } + } + + function registerListeners() { + + browser.webRequest.onBeforeRequest.addListener( + onBeforeRequestListener, + {urls: ['http://*/*', 'https://*/*']}, + [] + ); + + browser.webRequest.onSendHeaders.addListener( + onSendHeadersListener, + {urls: ['http://*/*', 'https://*/*']}, + ['requestHeaders'] + ); + + browser.webRequest.onHeadersReceived.addListener( + onHeadersReceivedListener, + {urls: ['http://*/*', 'https://*/*']}, + ['responseHeaders'] + ); + + browser.webRequest.onBeforeRedirect.addListener( + onBeforeRedirectListener, + {urls: ['http://*/*', 'https://*/*']}, + [] + ); + + browser.webRequest.onResponseStarted.addListener( + onResponseStartedListener, + {urls: ['http://*/*', 'https://*/*']}, + ['responseHeaders'] + ); + + browser.webNavigation.onCommitted.addListener(onCommittedListener); + browser.webNavigation.onCompleted.addListener(onCompletedListener); + browser.tabs.onRemoved.addListener(onRemovedListener); + browser.runtime.onMessage.addListener(onMessageListener); + + areListenersRegistered = true; + } + + function unregisterListeners() { + + browser.webRequest.onBeforeRequest.removeListener( + onBeforeRequestListener + ); + + browser.webRequest.onSendHeaders.removeListener( + onSendHeadersListener + ); + + browser.webRequest.onHeadersReceived.removeListener( + onHeadersReceivedListener + ); + + browser.webRequest.onBeforeRedirect.removeListener( + onBeforeRedirectListener + ); + + browser.webRequest.onResponseStarted.removeListener( + onResponseStartedListener + ); + + browser.webNavigation.onCommitted.removeListener(onCommittedListener); + browser.webNavigation.onCompleted.removeListener(onCompletedListener); + browser.tabs.onRemoved.removeListener(onRemovedListener); + browser.runtime.onMessage.removeListener(onMessageListener); + areListenersRegistered = false; + } + + function areRequiredBrowserApisAvailable() { + return requiredBrowserApis.every(function(api) { + return typeof api !== 'undefined'; + }); + } + + if ( areRequiredBrowserApisAvailable() ) { + ifBrowserValid( + function() { + browser.webNavigation.onBeforeNavigate.addListener( + function(details) { + if ( details.frameId === 0 ) { + globalPageContainer.onNewNavigation(details); + } + }, + { + url: [{urlMatches: 'http://*/*'}, {urlMatches: 'https://*/*'}] + } + ); + }, function() { + + } + ); + } + + browser.runtime.onConnect.addListener((port) => { + port.onMessage.addListener((message) => { + if ( message === 'is_browser_valid' ) { + ifBrowserValid( + port.postMessage({'browser_valid': true}), + port.postMessage({'browser_valid': false}) + ); + } + }); + }); + + browser.runtime.onConnect.addListener((port) => { + port.onMessage.addListener((message) => { + if ( message === 'is_tracking_enabled' ) { + ifTrackingEnabled( + port.sender.tab, + function() { + try {port.postMessage({'tracking_enabled': true});} + catch(err) {} }, + function() { + try {port.postMessage({'tracking_enabled': false});} + catch(err) {} } + ); + } + return true; + }); + }); + +})(); + +},{}]},{},[1]); diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json index c9ae4e1a0..1017f422e 100644 --- a/src/drivers/webextension/manifest.json +++ b/src/drivers/webextension/manifest.json @@ -4,7 +4,7 @@ "author": "Elbert Alias", "homepage_url": "https://www.wappalyzer.com", "description": "Identify web technologies", - "version": "5.7.3", + "version": "5.7.4", "default_locale": "en", "manifest_version": 2, "icons": { @@ -51,6 +51,7 @@ "https://www.alphaecommerce.gr/*" ], "js": [ + "node_modules/webextension-polyfill/dist/browser-polyfill.js", "js/lib/iframe.js" ], "run_at": "document_start",