diff --git a/bin/wappalyzer-links b/bin/wappalyzer-links index dbb52a483..251cebbf2 100755 --- a/bin/wappalyzer-links +++ b/bin/wappalyzer-links @@ -50,7 +50,6 @@ ln -f $path/wappalyzer.js $path/drivers/webextension/js ln -f $path/apps.json $path/drivers/webextension ln -f $path/icons/*.png $path/drivers/webextension/images/icons ln -f $path/icons/*.svg $path/drivers/webextension/images/icons -ln -f $path/utils/*.js $path/drivers/webextension/js if [ "$(compgen -G "$path/icons/converted/*.png" | head -n1)" ]; then ln -f $path/icons/converted/*.png $path/drivers/webextension/images/icons/converted diff --git a/src/drivers/webextension/.gitignore b/src/drivers/webextension/.gitignore index 602df1590..d0671fb4f 100644 --- a/src/drivers/webextension/.gitignore +++ b/src/drivers/webextension/.gitignore @@ -3,5 +3,3 @@ images/icons/converted/*.png images/icons/*.png images/icons/*.svg js/wappalyzer.js -js/iframe.js -js/network.js diff --git a/src/drivers/webextension/js/iframe.js b/src/drivers/webextension/js/iframe.js new file mode 100644 index 000000000..70f6fc408 --- /dev/null +++ b/src/drivers/webextension/js/iframe.js @@ -0,0 +1,1117 @@ +'use strict'; + +(function(win) { + +var exports = {}; + +(function(exports) { + + var utils = { + normalizeUrl: function(url) { + + return this.hashUrl(url) || null; + + }, + + getReferrer: function() { + return this.normalizeUrl(document.referrer); + }, + + getPageUrl: function() { + return this.normalizeUrl(window.location.href); + }, + hashUrl: function(url) { + var a, + result; + + if ( !url || url.indexOf('http') !== 0 ) { + return null; + } + + a = document.createElement('a'); + a.href = url; + + result = a.protocol + '//' + a.hostname + '/'; + + if ( a.pathname && a.pathname !== '/' ) { + result += this.hashCode(a.pathname); + } + + if ( a.search ) { + result += '?' + this.hashCode(a.search); + } + + if ( a.hash ) { + result += '#' + this.hashCode(a.hash); + } + + return result; + }, + + hashCode: function(str) { + var hash = 0, + kar, + i; + + if ( str.length === 0 ) { + return hash; + } + + for ( i = 0; i < str.length; i++ ) { + kar = str.charCodeAt(i); + hash = ((hash << 5) - hash) + kar; + hash = hash & hash; + } + + return hash + Math.pow(2, 32); + }, + + realArray: function(a) { + return Array.prototype.slice.apply(a); + }, + onDocLoaded: function(doc, callback) { + if ( doc.readyState === 'loading' ) { + doc.addEventListener('DOMContentLoaded', callback); + } else { + callback(); + } + }, + + SCRIPT_IN_WINDOW_TOP: window === window.top, + + isFriendlyWindow: function(win) { + + var href; + try { + href = win.location.href; + } catch(e) { + return false; + } + return true; + }, + + elementWindow: function(el) { + return el.ownerDocument.defaultView; + }, + + viewport: function(win) { + return {width: win.innerWidth, height: win.innerHeight}; + }, + + parseQS: function(qs) { + if ( qs.indexOf('http') === 0 ) { + qs = qs.split('?')[1]; + } + var i, kvs, key, val; + var dict = {}; + qs = qs.split('&'); + for ( i = 0; i < qs.length; i++ ) { + kvs = qs[i].split('='); + key = kvs[0]; + val = kvs.slice(1).join('='); + try { + dict[key] = window.decodeURIComponent(val); + } catch (e) { + + continue; + } + } + return dict; + }, + }; + + utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent); + utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME; + + function LogGenerator() { + this.msgNum = 0; + this.pageMeta = { + 'url': utils.getPageUrl(), + 'isHP': window.location.pathname === '/', + 'referrer': utils.getReferrer(), + 'rand': Math.floor(Math.random() * 10e12), + 'startTime': new Date().getTime() + }; + } + + LogGenerator.prototype = { + log: function(event, opt_assets, opt_pageTags) { + var opt_video_assets; + if ( event === 'video' ) { + opt_video_assets = opt_assets || []; + opt_assets = []; + } else { + opt_video_assets = []; + opt_assets = opt_assets || []; + } + var result = { + doc: this.pageMeta, + event: event, + video_assets: opt_video_assets, + assets: opt_assets, + version: '3', + mrev: '5dacb94-c', + msgNum: this.msgNum, + timestamp: new Date().getTime(), + pageVis: document.visibilityState, + pageFoc: document.hasFocus(), + pageTags: opt_pageTags || [] + }; + this.msgNum++; + return result; + } + }; + + utils.LogGenerator = LogGenerator; + + exports.utils = utils; +})(exports); + +(function(exports) { + + var SizeMatcher = { + VALID_AD_SIZES: [ + [300, 50], + [320, 50], + [160, 600], + [300, 250], + [300, 600], + [300, 1050], + [336, 280], + [336, 850], + [468, 60], + [728, 90], + [728, 250], + [728, 270], + [970, 66], + [970, 90], + [970, 125], + [970, 250], + [970, 400], + [970, 415], + [1280, 100] + ], + + PX_SIZE_TOL: 10, + + getMatchedAdSize: function(width, height) { + + if ( !this.set ) { + this.set = this._makeSizeSet(); + } + + return this.set[Math.round(width) + 'x' + Math.round(height)]; + }, + + elementIsAdShaped: function(el) { + return !!this.getMatchedAdSizeForElement(el); + }, + + getMatchedAdSizeForElement: function(el) { + var rect = el.getBoundingClientRect(); + return this.getMatchedAdSize(rect.width, rect.height); + }, + + _makeSizeSet: function() { + var set = {}; + var i; + var xfuz; + var yfuz; + var size; + var width; + var height; + + for ( i = 0; i < this.VALID_AD_SIZES.length; i++ ) { + for ( xfuz = -this.PX_SIZE_TOL; xfuz <= this.PX_SIZE_TOL; xfuz++ ) { + for ( yfuz = -this.PX_SIZE_TOL; yfuz <= this.PX_SIZE_TOL; yfuz++ ) { + size = this.VALID_AD_SIZES[i]; + width = size[0] + xfuz; + height = size[1] + yfuz; + set[width + 'x' + height] = size; + } + } + } + return set; + } + }; + + var Throttler = { + MAX_SEARCHES_PER_WINDOW: 10, + MAX_SEARCHES_PER_ELEMENT: 2, + + countSearch: function(el) { + if ( typeof el.searches !== 'number' ) { + el.searches = 0; + } + + el.searches += 1; + }, + + throttle: function(el, max) { + if ( typeof el.searches === 'number' && el.searches >= max ) { + return true; + } + return false; + }, + + throttleElement: function(el) { + return this.throttle(el, this.MAX_SEARCHES_PER_ELEMENT); + }, + + throttleWin: function(win) { + return this.throttle(win, this.MAX_SEARCHES_PER_WINDOW); + }, + + getCount: function(el) { + return el.searches || 0; + } + }; + + function TopSearcher(win) { + this.win = win; + this.doc = win.document; + } + + TopSearcher.prototype.search = function() { + var candidates = exports.utils.realArray(this.doc.querySelectorAll('img, object, embed')), + html5Ad, + ads = []; + + ads = ads.concat(candidates.filter(function(el) { + if ( !el.mpAdFound && !Throttler.throttleElement(el) ) { + Throttler.countSearch(el); + if ( (el.tagName !== 'IMG' || isStandardImage(el)) && SizeMatcher.elementIsAdShaped(el) ) { + el.mpAdFound = true; + return true; + } + } + return false; + })); + + html5Ad = this._mainGetHTMLAd(); + if ( html5Ad ) { + html5Ad.html5 = true; + html5Ad.mpAdFound = true; + ads.push(html5Ad); + } + + return ads; + }; + + TopSearcher.prototype._mainGetHTMLAd = function() { + var styles = this.doc.querySelectorAll('div > style, div > link[rel="stylesheet"]'), + i, div; + for ( i = 0; i < styles.length; i++ ) { + div = styles[i].parentNode; + if ( !div.mpAdFound && SizeMatcher.elementIsAdShaped(div) && this._jumpedOut(div) ) { + return div; + } + } + }; + + TopSearcher.prototype._jumpedOut = function(el) { + var siblings, ifrs; + siblings = exports.utils.realArray(el.parentNode.children); + ifrs = siblings.filter(function(el) { + return el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0; + }); + return ifrs.length > 0; + }; + + function IframeSearcher(win) { + this.MIN_AD_AREA = 14000; + this.MIN_WINDOW_PX = 10; + + this.win = win; + this.doc = win.document; + this.body = win.document.body; + this.winClickTag = win.clickTag; + this.adSizeMeta = this._getAdSizeMeta(); + this.numElementsInBody = (this.body && this.body.querySelectorAll('*').length) || 0; + + this.shouldSearchWindow = false; + if ( !this.win.mpAdFound && this.body && !Throttler.throttleWin(this.win) ) { + this.winWidth = this.win.innerWidth; + this.winHeight = this.win.innerHeight; + if ( this._meetsMinAdSize(this.winWidth, this.winHeight) && !this._containsLargeIframes() ) { + this.shouldSearchWindow = true; + } + } + + } + + IframeSearcher.prototype.search = function() { + var ad; + + if ( this.shouldSearchWindow ) { + ad = this._search(); + if ( ad ) { + ad.mpAdFound = true; + win.mpAdFound = true; + return ad; + } + Throttler.countSearch(this.win); + } + + return null; + }; + + IframeSearcher.prototype._search = function() { + var _this = this, + stdCandidates, + html5Candidates, + stdEl, + html5El; + + stdCandidates = this.body.querySelectorAll('img, object, embed'); + + stdEl = getFirst(stdCandidates, function(el) { + if ( !el.mpAdFound && + !Throttler.throttleElement(el) && + (el.tagName !== 'IMG' || isStandardImage(el)) && + _this._elementIsAtLeastAsBigAsWindow(el)) + { + return true; + } + Throttler.countSearch(el); + return false; + }); + + if ( stdEl ) { + return stdEl; + } + + if ( this._isHTML5Iframe() ) { + html5Candidates = this.doc.querySelectorAll('body, canvas, button, video, svg, div'); + html5El = getFirst(html5Candidates, function(el) { + + if ( _this._elementIsAtLeastAsBigAsWindow(el) ) { + return true; + } + Throttler.countSearch(el); + return false; + }); + } + + if ( html5El ) { + html5El.html5 = true; + html5El.winClickTag = this.winClickTag; + html5El.adSizeMeta = this.adSizeMeta; + return html5El; + } + + return null; + }; + + IframeSearcher.prototype._isHTML5Iframe = function() { + if ( this.winClickTag || this.adSizeMeta ) { + return true; + } + + if ( this.doc.querySelectorAll('canvas', 'button', 'video', 'svg').length > 0 ) { + return true; + } + + if ( this.numElementsInBody >= 5 && Throttler.getCount(this.win) > 0 && this.doc.querySelectorAll('div').length > 0 ) { + return true; + } + + return false; + }; + + IframeSearcher.prototype._elementIsAtLeastAsBigAsWindow = function(el) { + var rect = el.getBoundingClientRect(), + tol = 0.95; + + return rect.width >= (tol * this.winWidth) && rect.height >= (tol * this.winHeight); + }; + + IframeSearcher.prototype._meetsMinAdSize = function(width, height) { + return (width * height) >= this.MIN_AD_AREA; + }; + + IframeSearcher.prototype._containsLargeIframes = function() { + var iframes = this.doc.querySelectorAll('iframe'); + var rect; + var i; + for ( i = 0; i < iframes.length; i++ ) { + rect = iframes[i].getBoundingClientRect(); + if ( rect.width > this.MIN_WINDOW_PX || rect.height > this.MIN_WINDOW_PX ) { + return true; + } + } + return false; + }; + + IframeSearcher.prototype._getAdSizeMeta = function() { + var adSizeMeta = this.doc.querySelectorAll('meta[name="ad.size"]'); + if ( adSizeMeta.length > 0 ) { + return adSizeMeta[0].content; + } else { + return null; + } + }; + + function getFirst(arr, testFn) { + var i, el; + for ( i = 0; i < arr.length; i++ ) { + el = arr[i]; + if ( testFn(el) ) { + return el; + } + } + return null; + } + + function isStandardImage(img) { + + return img.src && (img.parentNode.tagName === 'A' || img.getAttribute('onclick')); + } + + function getFriendlyIframes(win) { + var iframes = win.document.querySelectorAll('iframe'); + iframes = exports.utils.realArray(iframes); + var friendlyIframes = iframes.filter(function(ifr) { + return exports.utils.isFriendlyWindow(ifr.contentWindow); + }); + return friendlyIframes; + } + + function findAds(win) { + var i, + iframes, + searcher, + ad, + ads = []; + + if ( win === win.top ) { + searcher = new TopSearcher(win); + ads = ads.concat(searcher.search()); + } else { + searcher = new IframeSearcher(win); + ad = searcher.search(); + if ( ad ) { + ads.push(ad); + } + } + + iframes = getFriendlyIframes(win); + for ( i = 0; i < iframes.length; i++ ) { + ads = ads.concat(findAds(iframes[i].contentWindow)); + } + + return ads; + } + + exports.adfinder = { + getMatchedAdSize: SizeMatcher.getMatchedAdSize.bind(SizeMatcher), + findAds: findAds + }; +})(exports); + +(function(exports) { + + var parser = { + TAGS_WITH_SRC_ATTR: { + 'IMG': true, + 'SCRIPT': true, + 'IFRAME': true, + 'EMBED': true + }, + + MAX_ATTR_LEN: 100, + + getUrl: function(el, params) { + var url; + + if ( this.TAGS_WITH_SRC_ATTR.hasOwnProperty(el.tagName) ) { + url = el.src; + + } else if ( el.tagName === 'OBJECT' ) { + url = el.data || (params && params.movie) || null; + + } else if ( el.tagName === 'A' ) { + url = el.href; + } + + if ( url && url.indexOf('http') === 0 ) { + return url; + } else { + return null; + } + }, + + getParams: function(el) { + if ( el.tagName !== 'OBJECT' ) { + return null; + } + + var i, child; + var params = {}; + var children = el.children; + for ( i = 0; i < children.length; i++ ) { + child = children[i]; + if ( child.tagName === 'PARAM' && child.name ) { + + params[child.name.toLowerCase()] = child.value; + } + } + return params; + }, + + getPosition: function(el) { + var rect = el.getBoundingClientRect(); + var win = exports.utils.elementWindow(el); + + return { + width: Math.round(rect.width), + height: Math.round(rect.height), + left: Math.round(rect.left + win.pageXOffset), + top: Math.round(rect.top + win.pageYOffset) + }; + }, + + getFlashvars: function(el, params, url) { + var flashvars; + var urlQS = url && url.split('?')[1]; + + if ( el.tagName === 'EMBED' ) { + flashvars = el.getAttribute('flashvars') || urlQS; + + } else if ( el.tagName === 'OBJECT' ) { + flashvars = params.flashvars || el.getAttribute('flashvars') || urlQS; + } + + return (flashvars && exports.utils.parseQS(flashvars)) || null; + }, + + findClickThru: function(el, flashvars) { + var key; + if ( el.tagName === 'IMG' && el.parentElement.tagName === 'A' ) { + return el.parentElement.href; + } else if ( flashvars ) { + for ( key in flashvars ) { + if ( flashvars.hasOwnProperty(key) ) { + + if ( key.toLowerCase().indexOf('clicktag') === 0 ) { + return flashvars[key]; + } + } + } + } + return null; + }, + + getAttr: function(el, name) { + var val = el.getAttribute(name); + + if ( val && val.slice && val.toString ) { + + return val.slice(0, this.MAX_ATTR_LEN).toString(); + } else { + return null; + } + }, + + putPropIfExists: function(obj, name, val) { + if ( val ) { + obj[name] = val; + } + }, + + putAttrIfExists: function(obj, el, name) { + var val = this.getAttr(el, name); + this.putPropIfExists(obj, name, val); + }, + + elementToJSON: function(el, opt_findClickThru) { + var pos = this.getPosition(el); + var params = this.getParams(el); + var url = this.getUrl(el, params); + var flashvars = this.getFlashvars(el, params, url); + var clickThru = opt_findClickThru && this.findClickThru(el, flashvars); + var json = { + tagName: el.tagName, + width: pos.width, + height: pos.height, + left: pos.left, + top: pos.top, + children: [] + }; + + if ( params ) { + + delete params.flashvars; + } + + this.putAttrIfExists(json, el, 'id'); + this.putAttrIfExists(json, el, 'class'); + this.putAttrIfExists(json, el, 'name'); + + this.putPropIfExists(json, 'flashvars', flashvars); + this.putPropIfExists(json, 'url', url); + this.putPropIfExists(json, 'params', params); + this.putPropIfExists(json, 'clickThru', clickThru); + + return json; + } + }; + + exports.parser = { elementToJSON: parser.elementToJSON.bind(parser) }; +})(exports); + +(function(exports) { + + var ContextManager = function(adData) { + this.adData = adData; + }; + + ContextManager.prototype = { + CONTAINER_SIZE_TOL: 0.4, + ASPECT_RATIO_FOR_LEADERBOARDS: 2, + + isValidContainer: function(el, opt_curWin) { + + var cWidth = el.clientWidth; + var cHeight = el.clientHeight; + + var adWidth = this.adData.width; + var adHeight = this.adData.height; + + var winWidth = opt_curWin && opt_curWin.innerWidth; + var winHeight = opt_curWin && opt_curWin.innerHeight; + var similarWin = opt_curWin && this.withinTol(adWidth, winWidth) && this.withinTol(adHeight, winHeight); + + var similarSizeX = this.withinTol(adWidth, cWidth); + var similarSizeY = this.withinTol(adHeight, cHeight); + var adAspect = adWidth / adHeight; + + return similarWin || el.tagName === 'A' || (adAspect >= this.ASPECT_RATIO_FOR_LEADERBOARDS && similarSizeY) || (similarSizeX && similarSizeY); + }, + + withinTol: function(adlen, conlen) { + var pct = (conlen - adlen) / adlen; + + return pct <= this.CONTAINER_SIZE_TOL; + }, + + serializeElements: function(el) { + if ( !el ) { + return; + } + var i; + var ifrWin; + var adId = this.adData.adId; + var elIsAd = false; + + if ( adId && el[adId] && el[adId].isAd === true ) { + elIsAd = true; + } + + var json = exports.parser.elementToJSON(el, elIsAd); + var childJSON; + + if ( elIsAd ) { + json.adId = adId; + this.adData.element = {}; + + var keys = Object.keys(json); + for ( i = 0; i < keys.length; i++ ) { + var key = keys[i]; + if ( key !== 'children' && key !== 'contents' ) { + this.adData.element[key] = json[key]; + } + } + } + + var children = exports.utils.realArray(el.children).filter(function(el) { + var param = el.tagName === 'PARAM'; + var inlineScript = el.tagName === 'SCRIPT' && !(el.src && el.src.indexOf('http') >= 0); + var noScript = el.tagName === 'NOSCRIPT'; + return !(param || inlineScript || noScript); + }); + + for ( i = 0; i < children.length; i++ ) { + childJSON = this.serializeElements(children[i]); + if ( childJSON ) { + json.children.push(childJSON); + } + } + + if ( el.tagName === 'IFRAME' ) { + ifrWin = el.contentWindow; + + if ( adId && el[adId] && el[adId].needsWindow ) { + + json.contents = this.adData.serializedIframeContents; + el[adId].needsWindow = false; + delete this.adData.serializedIframeContents; + + } else if ( exports.utils.isFriendlyWindow(ifrWin) ) { + + childJSON = this.serializeElements(ifrWin.document.documentElement); + if ( childJSON ) { + json.contents = childJSON; + } + } + } + + if ( json.children.length > 0 || json.adId || json.tagName === 'IFRAME' || json.url ) { + return json; + } else { + return null; + } + }, + + captureHTML: function(containerEl) { + this.adData.context = this.serializeElements(containerEl); + }, + + nodeCount: function(el) { + return el.getElementsByTagName('*').length + 1; + }, + + highestContainer: function(curWin, referenceElement) { + var curContainer = referenceElement; + var docEl = curWin.document.documentElement; + var parentContainer; + + if ( curWin !== curWin.top && this.isValidContainer(docEl, curWin) ) { + return docEl; + } + + while ( true ) { + parentContainer = curContainer.parentElement; + if ( parentContainer && this.isValidContainer(parentContainer) ) { + curContainer = parentContainer; + } else { + return curContainer; + } + } + } + }; + + var tagfinder = { + + setPositions: function(adData, opt_el, opt_winPos) { + var el = opt_el || adData.context; + var winPos = opt_winPos || {left: 0, top: 0}; + var ifrPos; + + el.left += winPos.left; + el.top += winPos.top; + + if ( el.children ) { + el.children.forEach(function(child) { + this.setPositions(adData, child, winPos); + }, this); + } + + if ( el.contents ) { + ifrPos = {left: el.left, top: el.top}; + this.setPositions(adData, el.contents, ifrPos); + } + + if ( el.adId === adData.adId ) { + adData.element.left = el.left; + adData.element.top = el.top; + } + }, + + appendTags: function(adData, referenceElement) { + var mgr = new ContextManager(adData); + var curWin = exports.utils.elementWindow(referenceElement); + var highestContainer; + + while ( true ) { + highestContainer = mgr.highestContainer(curWin, referenceElement); + mgr.captureHTML(highestContainer); + if ( curWin === curWin.top ) { + break; + } else { + + curWin.mpAdFound = true; + + mgr.adData.serializedIframeContents = mgr.adData.context; + + if ( exports.utils.isFriendlyWindow(curWin.parent) ) { + referenceElement = curWin.frameElement; + referenceElement[mgr.adData.adId] = {needsWindow: true}; + curWin = curWin.parent; + } else { + break; + } + } + } + return { + referenceElement:referenceElement, + highestContainer: highestContainer + }; + } + }; + + exports.tagfinder = tagfinder; +})(exports); + +(function(exports) { + var _onAdFound; + var _logGen = new exports.utils.LogGenerator(); + var _pageTags; + var INIT_MS_BW_SEARCHES = 2000; + var PAGE_TAG_RE = new RegExp('gpt|oascentral'); + var POST_MSG_ID = '1484952787-11883-24071-12354-1549'; + var AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)'); + + function getPageTags(doc) { + var scripts = doc.getElementsByTagName('script'); + var pageTags = []; + scripts = exports.utils.realArray(scripts); + scripts.forEach(function(script) { + if ( PAGE_TAG_RE.exec(script.src) ) { + pageTags.push({'tagName': 'SCRIPT', 'url': script.src}); + } + }); + return pageTags; + } + + function messageAllParentFrames(adData) { + + adData.postMessageId = POST_MSG_ID; + + adData = JSON.stringify(adData); + + var win = window; + while ( win !== win.top ) { + win = win.parent; + win.postMessage(adData, '*'); + } + } + + function appendTagsAndSendToParent(adData, referenceElement) { + var results = exports.tagfinder.appendTags(adData, referenceElement); + if ( exports.utils.SCRIPT_IN_HOSTILE_IFRAME ) { + messageAllParentFrames(adData); + + } else if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { + + exports.tagfinder.setPositions(adData); + + adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height); + if ( !adData.matchedSize ) { + + if ( AD_SERVER_RE.exec(results.referenceElement.id) ) { + adData.matchedSize = [adData.width, adData.height]; + adData.oddSize = true; + } else { + + return; + } + } + delete adData.width; + delete adData.height; + adData.curPageUrl = exports.utils.getPageUrl(); + _pageTags = _pageTags || getPageTags(document); + var log = _logGen.log('ad', [adData], _pageTags); + + if ( _onAdFound ) { + + _onAdFound(log, results.referenceElement); + + } + } + } + + function extractAdsWrapper() { + if ( exports.utils.SCRIPT_IN_WINDOW_TOP || document.readyState === 'complete' ) { + extractAds(); + } + setTimeout(function() { + extractAdsWrapper(); + }, INIT_MS_BW_SEARCHES); + } + + function extractAds() { + var ads = exports.adfinder.findAds(window); + ads.forEach(function(ad) { + + var startTime = new Date().getTime(); + var adId = startTime + '-' + Math.floor(Math.random() * 10e12); + + var adData = { + width: Math.round(ad.offsetWidth), + height: Math.round(ad.offsetHeight), + startTime: startTime, + adId: adId, + html5: ad.html5 || false + }; + + if ( ad.html5 ) { + adData.adSizeMeta = ad.adSizeMeta || null; + adData.winClickTag = ad.winClickTag || null; + } + + ad[adId] = { isAd: true }; + + appendTagsAndSendToParent(adData, ad); + }); + } + + function isChildWin(myWin, otherWin) { + var parentWin = otherWin.parent; + while ( parentWin !== otherWin ) { + if ( parentWin === myWin ) { + return true; + } + otherWin = parentWin; + parentWin = parentWin.parent; + } + return false; + } + + function iframeFromWindow(win, winToMatch) { + var i, ifr, ifrWin, + iframes = win.document.querySelectorAll('iframe'); + + for ( i = 0; i < iframes.length; i++ ) { + ifr = iframes[i]; + if ( ifr.contentWindow === winToMatch ) { + return ifr; + } + } + + for ( i = 0; i < iframes.length; i++ ) { + ifrWin = iframes[i].contentWindow; + if ( exports.utils.isFriendlyWindow(ifrWin) ) { + ifr = iframeFromWindow(ifrWin, winToMatch); + if ( ifr ) { + return ifr; + } + } + } + } + + function onPostMessage(event) { + var adData, + ifrWin = event.source, + + myWin = window.document.defaultView, + ifrTag; + + try { + + adData = JSON.parse(event.data); + } catch(e) { + + return; + } + + if ( adData.postMessageId === POST_MSG_ID ) { + + delete adData.postMessageId; + + if ( isChildWin(myWin, ifrWin) ) { + if ( exports.utils.isFriendlyWindow(ifrWin) ) { + ifrTag = ifrWin.frameElement; + } else { + ifrTag = iframeFromWindow(myWin, ifrWin); + } + + if ( ifrTag ) { + ifrTag[adData.adId] = {needsWindow: true}; + appendTagsAndSendToParent(adData, ifrTag); + } + } + } + } + + function onVideoMessage(msg, sender, callback) { + var log; + if ( msg.event === 'new-video-ad' ) { + msg.assets.forEach(function(asset) { + + }); + log = _logGen.log('video', msg.assets); + } else { + log = _logGen.log('invalid-video', msg.assets); + } + + msg.assets.forEach(function(a) {delete a.isVideo;}); + log.displayAdFound = msg.displayAdFound; + log.requests = msg.requests; + log.data = msg.event_data; + + log.doc.finalPageUrl = log.doc.url; + log.doc.url = exports.utils.normalizeUrl(msg.origUrl); + + _onAdFound(log); + } + + function addBackgroundListener(event, callback) { + if ( typeof browser !== 'undefined' ) { + browser.runtime.onMessage.addListener(function(msg) { + if ( msg.event === event ) { + callback(msg); + } + }); + } else if ( window.self.port ) { + window.self.port.on(event, callback); + } + } + + exports.coordinator = { + init: function(onAdFound) { + + if ( exports.utils.SCRIPT_IN_FRIENDLY_IFRAME ) { + return false; + } + + _onAdFound = onAdFound; + window.addEventListener('message', onPostMessage, false); + + if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { + var log = _logGen.log('page'); + onAdFound(log); + + window.addEventListener('beforeunload', function(event) { + var log = _logGen.log('unload'); + log.timing = window.performance.timing; + onAdFound(log); + }); + + addBackgroundListener('new-video-ad', onVideoMessage); + addBackgroundListener('new-invalid-video-ad', onVideoMessage); + + } + + exports.utils.onDocLoaded(document, extractAdsWrapper); + } + }; + +})(exports); + +if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { + window.adparser = { + init: exports.coordinator.init, + }; +} else { + exports.coordinator.init(function() {}); +} + +})(window); +(function(adparser) { + function sendToBackground(event, message) { + if ( window.self.port ) { + window.self.port.emit(event, message); + } else if ( typeof browser !== 'undefined' ) { + browser.runtime.sendMessage(message); + } + } + + function onAdFound(log) { + sendToBackground('ad_log', { id: 'ad_log', subject: log }); + } + + if ( window === window.top ) { + adparser.init(onAdFound); + } +})(window.adparser); diff --git a/src/drivers/webextension/js/network.js b/src/drivers/webextension/js/network.js new file mode 100644 index 000000000..9dc38f5a2 --- /dev/null +++ b/src/drivers/webextension/js/network.js @@ -0,0 +1,584 @@ +'use strict'; +(function() { + + var secBefore = 2000; + var secAfter = 5000; + var secBetweenDupAssets = 10e3; + var minVidSize = 500e3; + var maxVidSize = 25e6; + var maxContentRange = 25e6; + var videoExtensions = [ + 'af', '3gp', 'asf', 'avchd', 'avi', 'cam', 'dsh', 'flv', 'm1v', 'm2v', + 'fla', 'flr', 'sol', 'm4v', 'mkv', 'wrap', 'mng', 'mov', 'mpeg', 'mpg', + 'mpe', 'mp4', 'mxf', 'nsv', 'ogg', 'rm', 'svi', 'smi', 'wmv', 'webm' + ]; + var extensionsReg = new RegExp('\\.' + videoExtensions.join('$|\\.') + '$'); + var videoContentTypesPrefixes = ['binary/octet-stream', 'video/', 'flv-application/', 'media']; + + var bannedContentTypes = ['video/mp2t','video/f4m','video/f4f']; + var bannedFiletypes = ['ts']; + var bannedFiletypesReg = new RegExp('\\.' + bannedFiletypes.join('$|\\.') + '$'); + var whitelistReqTypes = ['object', 'xmlhttprequest', 'other']; + + var topVideoAssetDomains = [ + '2mdn.net', + 'innovid.com', + 'serving-sys.com', + 'btrll.com', + 'teads.tv', + 'tubemogul.com' + ]; + + if ( !String.prototype.endsWith ) { + String.prototype.endsWith = function(searchString, position) { + var subjectString = this.toString(); + if ( typeof position !== 'number' || !isFinite(position) || + Math.floor(position) !== position || position > subjectString.length) { + position = subjectString.length; + } + position -= searchString.length; + var lastIndex = subjectString.indexOf(searchString, position); + return lastIndex !== -1 && lastIndex === position; + }; + } + + function isPixelRequest(request) { + return (request.type === 'image' || request.responseStatus === 204) && + request.size <= 1000; + } + + function isVpaidOrVastRequest(request) { + var lowerCaseUrl = request.url.toLowerCase(); + return lowerCaseUrl.indexOf('vpaid') !== -1 || lowerCaseUrl.indexOf('vast') !== -1; + } + + function hasValidRequestType(request) { + return whitelistReqTypes.indexOf(request.type) >= 0; + } + + function stripQueryParams(url) { + return url.split('?',1)[0]; + } + + function parseHostnameFromUrl(url) { + var parser = document.createElement('a'); + parser.href = url; + return parser.hostname; + } + + function hasDomain(url, domain) { + return parseHostnameFromUrl(url).endsWith(domain); + } + + function findHeader(headers, key) { + var header; + for ( var i = 0; i < headers.length; i += 1 ) { + header = headers[i]; + if ( header.name.toLowerCase() === key ) { + return header; + } + } + return null; + } + + function validVideoType(vtype) { + var goodType = videoContentTypesPrefixes.some(function(prefix) { + return vtype.indexOf(prefix) === 0; + }); + return goodType; + } + + function assetMsgKey(assetReq) { + var url = stripQueryParams(assetReq.url); + var key = assetReq.frameId + '-' + url; + return key; + } + + var PageNetworkTrafficCollector = function(tabId) { + this.tabId = tabId; + this.displayAdFound = false; + this.requests = {}; + this.msgsBeingSent = {}; + this.assetsSeen = {}; + this.allRedirects = {}; + }; + + var globalPageContainer = { + collectors: {}, + dyingCollectors: {}, + + cleanupCollector: function(tabId) { + if ( tabId in this.collectors ) { + delete globalPageContainer.collectors[tabId]; + } + }, + + onNewNavigation: function(details) { + var tabId = details.tabId; + this.cleanupCollector(tabId); + + this.collectors[tabId] = new PageNetworkTrafficCollector(tabId); + }, + + onNavigationCommitted: function(details) { + + }, + + onNavigationCompleted: function(details) { + + }, + + onTabClose: function(tabId, closeInfo) { + + this.cleanupCollector(tabId); + delete this.collectors[tabId]; + }, + + onDisplayAdFound: function(tabId) { + this.collectors[tabId].displayAdFound = true; + }, + + getRandId: function() { + return String(Math.floor(Math.random() * 1e9)); + }, + + getCollector: function(tabId) { + if ( this.collectors.hasOwnProperty(tabId) ) { + return this.collectors[tabId]; + } + return null; + }, + + forwardCall: function(details, collectorMemberFunction) { + var collector = this.getCollector(details.tabId); + if ( collector !== null ) { + collectorMemberFunction.apply(collector, [details]); + } + } + }; + + PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() { + var logMessage = Array.from(arguments).join(' '); + var message = {message: logMessage, event: 'console-log-message'}; + browser.tabs.sendMessage(this.tabId, message); + }; + + PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, isValidAd) { + var msg = {}; + msg.assets = []; + msg.event_data = {}; + if ( isValidAd ) { + msg.event = 'new-video-ad'; + msg.requests = reqs; + msg.requests.sort(function(reqA, reqB) {return reqA.requestTimestamp - reqB.requestTimestamp;}); + if ( assetReq ) { + msg.assets = [assetReq]; + } + } else { + msg.requests = reqs.map(function(request) { + return parseHostnameFromUrl(request.url); + }); + msg.assets = [{ + url: parseHostnameFromUrl(assetReq.url), + contentType: assetReq.contentType, + size: assetReq.size + }]; + msg.event = 'new-invalid-video-ad'; + } + msg.origUrl = curPageUrl; + msg.displayAdFound = this.displayAdFound; + + browser.tabs.sendMessage(this.tabId, msg); + }; + + PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) { + return url + ':' + frameId; + }; + + PageNetworkTrafficCollector.prototype.seenBefore = function(request) { + var oldTime = this.assetsSeen[assetMsgKey(request)]; + if ( oldTime && (request.requestTimestamp-oldTime < secBetweenDupAssets)){ + + return true; + } + return false; + }; + + PageNetworkTrafficCollector.prototype.recordSeenAsset = function(request) { + this.assetsSeen[assetMsgKey(request)] = request.requestTimestamp; + }; + + PageNetworkTrafficCollector.prototype.onBeforeRequest = function(details) { + var req = { + url: details.url, + type: details.type, + httpMethod: details.method, + frameId: details.frameId, + parentFrameId: details.parentFrameId, + requestTimestamp: details.timeStamp, + }; + this.requests[details.requestId] = req; + }; + + PageNetworkTrafficCollector.prototype.onSendHeaders = function(details) { + var request, header; + request = this.requests[details.requestId]; + header = request && findHeader(details.requestHeaders, 'x-requested-with'); + if ( header && header.value.toLowerCase().indexOf('flash') > -1 ) { + request.from_flash = true; + } + }; + + PageNetworkTrafficCollector.prototype.onHeadersReceived = function(details) { + var getFrameDetails = { + tabId: details.tabId, + processId: null, + frameId: details.frameId + }; + var pageNetworkTrafficController = this; + browser.webNavigation.getFrame(getFrameDetails).then(function(frameDetails) { + if ( frameDetails && frameDetails.url ) { + pageNetworkTrafficController._onHeadersReceived(details, frameDetails); + } + }); + }; + + PageNetworkTrafficCollector.prototype._onHeadersReceived = function(details, frameDetails) { + var contentSize, contentRange; + + var request = this.requests[details.requestId]; + if ( request ) { + var redirParent = this.allRedirects[this.getRedirKey(details.url, details.frameId)]; + var header = request && findHeader(details.responseHeaders, 'content-type'); + var contentType = header && header.value.toLowerCase(); + + if ( contentType){ + request.contentType = contentType; + } + header = request && findHeader(details.responseHeaders, 'content-length'); + contentSize = header && header.value; + if ( contentSize ) { + request.size = request.size || 0; + request.size += parseInt(contentSize); + } + header = request && findHeader(details.responseHeaders, 'content-range'); + contentRange = header && header.value; + if ( contentRange ) { + request.contentRange = parseInt(contentRange.split('/')[1]); + } + + var frameUrl = null; + if ( frameDetails && frameDetails.url ) { + frameUrl = frameDetails.url; + } + if ( !this.bannedRequest(request) && + (this.isVideoReq(frameUrl, request) || (redirParent && redirParent.isVideo))) { + request.isVideo = true; + } + } + }; + + PageNetworkTrafficCollector.prototype.onBeforeRedirect = function(details) { + var request = this.requests[details.requestId]; + if ( request ) { + if ( request.redirects ) { + request.redirects.push(details.redirectUrl); + } else { + request.redirects = [details.redirectUrl]; + } + this.allRedirects[this.getRedirKey(details.redirectUrl, details.frameId)] = request; + } + }; + + PageNetworkTrafficCollector.prototype.isYoutubeMastheadRequest = function(url) { + var re = /video_masthead/; + return this.hasYoutubeDomain(url) && re.test(url); + }; + PageNetworkTrafficCollector.prototype.isYoutubeVideoRequest = function(srcUrl, destUrl) { + if ( !this.hasYoutubeDomain(srcUrl) ) { + return false; + } + + var re = /https?:\/\/r.*?\.googlevideo\.com\/videoplayback\?/; + return re.test(destUrl); + }; + PageNetworkTrafficCollector.prototype.processResponse = function(requestDetails, frameDetails) { + var request; + if ( requestDetails ) { + request = this.requests[requestDetails.requestId]; + if ( request ) { + request.responseStatus = requestDetails.statusCode; + request.responseTimestamp = requestDetails.timeStamp; + + var frameUrl = null; + if ( frameDetails && frameDetails.url ) { + frameUrl = frameDetails.url; + } + + var requestUrl = null; + if ( request.url ) { + requestUrl = request.url; + } + + if ( this.isYoutubeAdReq(frameUrl, requestUrl) ) { + var videoId = this.parseYoutubeVideoIdFromUrl(requestUrl); + if ( videoId ) { + request.isYoutubeAd = true; + request.isVideo = true; + request.url = 'https://www.youtube.com/watch?v=' + this.parseYoutubeVideoIdFromUrl(requestUrl); + } + } else if ( !this.bannedRequest(request) && + (this.isVideo || this.isVideoReq(frameUrl, request))) { + request.isVideo = true; + } + + if ( request.isVideo ) { + + var msgKey = assetMsgKey(request); + this.msgsBeingSent[msgKey] = request; + if ( !this.seenBefore(request) ) { + this.sendMsgWhenQuiet(msgKey); + } + this.recordSeenAsset(request); + } + } + } + }; + + PageNetworkTrafficCollector.prototype.onResponseStarted = function(responseDetails) { + if ( responseDetails.frameId < 0 ) { + responseDetails.frameId = 99999; + + } + var getFrameDetails = { + tabId: responseDetails.tabId, + processId: null, + frameId: responseDetails.frameId + }; + var pageNetworkTrafficController = this; + browser.webNavigation.getFrame(getFrameDetails).then(function(frameDetails) { + if ( frameDetails && frameDetails.url ) { + pageNetworkTrafficController.processResponse(responseDetails, frameDetails); + } + }); + }; + + PageNetworkTrafficCollector.prototype.hasBannedFiletype = function(request) { + var url = stripQueryParams(request.url); + if ( bannedFiletypesReg.exec(url) ) { + return true; + } else { + return false; + } + }; + + PageNetworkTrafficCollector.prototype.checkContentHeaders = function(request) { + if ( request.contentType && validVideoType(request.contentType) ) { + return true; + } + return false; + }; + + PageNetworkTrafficCollector.prototype.checkUrlExtension = function(request) { + var url = stripQueryParams(request.url); + if ( extensionsReg.exec(url) ) { + return true; + } else { + return false; + } + }; + + PageNetworkTrafficCollector.prototype.isVideoReq = function(srcUrl, request) { + if ( this.isYoutubeVideoRequest(srcUrl, request.url) ) { + return false; + } + return this.checkUrlExtension(request) || this.checkContentHeaders(request); + }; + PageNetworkTrafficCollector.prototype.hasYoutubeDomain = function(url) { + var hostname = parseHostnameFromUrl(url) ; + if ( hostname === 'www.youtube.com' ) { + return true; + } + return false; + }; + PageNetworkTrafficCollector.prototype.parseYoutubeVideoIdFromUrl = function(url) { + var re = /^https?:\/\/www\.youtube\.com\/get_video_info.*(?:\?|&)video_id=(.*?)(?:$|&)/; + var match = re.exec(url); + if ( match && match.length > 1 ) { + return match[1]; + } + + re = /^https?:\/\/www\.youtube\.com\/embed\/(.*?)(?:$|\?)/; + match = re.exec(url); + if ( match && match.length > 1 ) { + return match[1]; + } + + re = /^https?:\/\/www\.youtube\.com\/watch\?v=(.*$)/; + match = re.exec(url); + if ( match && match.length > 1 ) { + return match[1]; + } + return null; + }; + + PageNetworkTrafficCollector.prototype.isYoutubeGetVideoInfoReq = function(url) { + var re = /^https?:\/\/www\.youtube\.com\/get_video_info\?/; + return re.test(url); + }; + PageNetworkTrafficCollector.prototype.isYoutubeAdReq = function(srcUrl, destUrl) { + + if ( !this.hasYoutubeDomain(srcUrl) || + !this.isYoutubeGetVideoInfoReq(destUrl)) { + return false; + } + if ( this.parseYoutubeVideoIdFromUrl(srcUrl) === + this.parseYoutubeVideoIdFromUrl(destUrl) && + !this.isYoutubeMastheadRequest(destUrl)) { + return false; + } + return true; + }; + + PageNetworkTrafficCollector.prototype.bannedRequest = function(request) { + return this.bannedVideoType(request) || this.hasBannedFiletype(request) || this.bannedVideoSize(request); + }; + + PageNetworkTrafficCollector.prototype.bannedVideoType = function(request) { + var badType = false; + if ( request.contentType ) { + badType = bannedContentTypes.some(function(prefix) { + return request.contentType.indexOf(prefix) >= 0; + }); + } + return badType; + }; + + PageNetworkTrafficCollector.prototype.bannedVideoSize = function(request) { + if ( request.size !== null ) { + if ( request.size < minVidSize || request.size > maxVidSize || request.contentRange > maxContentRange ) { + return true; + } + } + return false; + }; + + PageNetworkTrafficCollector.prototype.grabTagReqs = function(tabRequests, assetRequest) { + var minTimestamp, maxTimestamp; + minTimestamp = assetRequest.requestTimestamp - secBefore; + maxTimestamp = assetRequest.requestTimestamp + secAfter; + + var filteredRequests = tabRequests.filter(function(request) { + return (request.requestTimestamp > minTimestamp && + request.requestTimestamp < maxTimestamp && + request.frameId === assetRequest.frameId && + request.url !== assetRequest.url && + (hasValidRequestType(request) || + isPixelRequest(request))); + }); + + return filteredRequests; + }; + + PageNetworkTrafficCollector.prototype.isValidVideoAd = function(assetRequest, tagRequests) { + var hasVpaidOrVastRequest = tagRequests.some(function(tagRequest) { + return isVpaidOrVastRequest(tagRequest); + }); + if ( hasVpaidOrVastRequest ) { + return true; + } + var hasTopVideoAssetDomain = topVideoAssetDomains.some(function(assetDomain) { + return hasDomain(assetRequest.url, assetDomain); + }); + + return hasTopVideoAssetDomain; + }; + + PageNetworkTrafficCollector.prototype.sendMsgWhenQuiet = function(msgKey) { + var _this = this, + origPageUrl, msgAssetReq; + msgAssetReq = this.msgsBeingSent[msgKey]; + browser.tabs.get(this.tabId, function(tab) {origPageUrl = tab.url;}); + + setTimeout(function() { + var rawRequests = []; + if ( globalPageContainer.collectors[_this.tabId] === _this ) { + for ( var reqId in _this.requests ) { + rawRequests.push(_this.requests[reqId]); + } + var tagReqs = _this.grabTagReqs(rawRequests, msgAssetReq); + + if ( _this.isValidVideoAd(msgAssetReq, tagReqs) ) { + _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, true); + } else { + + _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, false); + } + + } else { + + } + delete _this.msgsBeingSent[msgKey]; + }, secAfter+secBefore); + }; + + PageNetworkTrafficCollector.prototype.existingMessage = function(candidateRequest) { + var frameMsg = this.msgsBeingSent[candidateRequest.frameId]; + if ( frameMsg ) { + return frameMsg; + } else { + return null; + } + }; + + browser.webRequest.onBeforeRequest.addListener(function(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRequest); + }, {urls: ['http://*/*', 'https://*/*']}, []); + + browser.webRequest.onSendHeaders.addListener(function(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onSendHeaders); + }, {urls: ['http://*/*', 'https://*/*']}, ['requestHeaders']); + + browser.webRequest.onHeadersReceived.addListener(function(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onHeadersReceived); + }, {urls: ['http://*/*', 'https://*/*']}, ['responseHeaders']); + + browser.webRequest.onBeforeRedirect.addListener(function(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRedirect); + }, {urls: ['http://*/*', 'https://*/*']}, []); + + browser.webRequest.onResponseStarted.addListener(function(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onResponseStarted); + }, {urls: ['http://*/*', 'https://*/*']}, ['responseHeaders']); + + browser.webNavigation.onBeforeNavigate.addListener(function(details) { + if ( details.frameId === 0 ) { + globalPageContainer.onNewNavigation(details); + } + }, {}); + + browser.webNavigation.onCommitted.addListener(function(details) { + if ( details.frameId === 0 ) { + globalPageContainer.onNavigationCommitted(details); + } + }); + + browser.webNavigation.onCompleted.addListener(function(details) { + if ( details.frameId === 0 ) { + globalPageContainer.onNavigationCompleted(details); + } + }); + + browser.tabs.onRemoved.addListener(function(tabId, closeInfo) { + globalPageContainer.onTabClose(tabId, closeInfo); + }); + + browser.runtime.onMessage.addListener(function(message, sender, sendResponse) { + if ( message.event === 'new-ad' && message.data.event === 'ad' ) { + var tabId = sender.tab.id; + if ( tabId ) { + globalPageContainer.onDisplayAdFound(tabId); + } + } + }); +})();