diff --git a/.gitignore b/.gitignore index 6ee2ee009..19de018bb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ src/icons/converted/* node_modules npm-debug.log +tags + !.gitkeep diff --git a/bin/wappalyzer-links b/bin/wappalyzer-links index 78425cd9d..3a665b7c3 100755 --- a/bin/wappalyzer-links +++ b/bin/wappalyzer-links @@ -30,11 +30,11 @@ echo "Creating hard links..." ln -f $path/wappalyzer.js $path/drivers/phantomjs ln -f $path/apps.json $path/drivers/phantomjs -ln -f $path/wappalyzer.js $path/drivers/firefox/lib -ln -f $path/apps.json $path/drivers/firefox/data -ln -f $path/icons/*.png $path/drivers/firefox/data/images/icons -ln -f $path/icons/*.svg $path/drivers/firefox/data/images/icons -ln -f $path/utils/*.js $path/drivers/firefox/data/js +ln -f $path/wappalyzer.js $path/drivers/firefox/lib +ln -f $path/apps.json $path/drivers/firefox/data +ln -f $path/icons/*.png $path/drivers/firefox/data/images/icons +ln -f $path/icons/*.svg $path/drivers/firefox/data/images/icons +ln -f $path/utils/iframe.js $path/drivers/firefox/data/js ln -f $path/wappalyzer.js $path/drivers/chrome/js ln -f $path/apps.json $path/drivers/chrome diff --git a/src/drivers/chrome/.gitignore b/src/drivers/chrome/.gitignore index 72da3a6ba..602df1590 100644 --- a/src/drivers/chrome/.gitignore +++ b/src/drivers/chrome/.gitignore @@ -4,3 +4,4 @@ images/icons/*.png images/icons/*.svg js/wappalyzer.js js/iframe.js +js/network.js diff --git a/src/drivers/chrome/background.html b/src/drivers/chrome/background.html index 3abbe7968..3aaa6249f 100644 --- a/src/drivers/chrome/background.html +++ b/src/drivers/chrome/background.html @@ -8,6 +8,7 @@ + diff --git a/src/drivers/chrome/manifest.json b/src/drivers/chrome/manifest.json index 7ef03d0a2..7e973b7b3 100644 --- a/src/drivers/chrome/manifest.json +++ b/src/drivers/chrome/manifest.json @@ -29,6 +29,6 @@ "js/inject.js" ], "options_page": "options.html", - "permissions": [ "tabs", "webRequest", "http://*/*", "https://*/*" ], + "permissions": [ "tabs", "webRequest", "webNavigation", "http://*/*", "https://*/*" ], "content_security_policy": "script-src 'self' https://ssl.google-analytics.com; object-src 'self'" } diff --git a/src/utils/iframe.js b/src/utils/iframe.js index 6292dadb1..2322b509f 100644 --- a/src/utils/iframe.js +++ b/src/utils/iframe.js @@ -7,18 +7,19 @@ var exports = {}; (function(exports) { var utils = { - getReferrer: function() { + normalizeUrl: function(url) { - return this.hashUrl(document.referrer) || null; + return this.hashUrl(url) || null; }, - getPageUrl: function() { - - return this.hashUrl(window.location.href) || null; - + getReferrer: function() { + return this.normalizeUrl(document.referrer); }, + getPageUrl: function() { + return this.normalizeUrl(window.location.href); + }, hashUrl: function(url) { var a, result; @@ -121,6 +122,7 @@ var exports = {}; utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent); utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME; + function LogGenerator() { this.msgNum = 0; this.pageMeta = { @@ -134,11 +136,21 @@ var exports = {}; LogGenerator.prototype = { log: function(event, opt_assets, opt_pageTags) { + var opt_video_assets; + if ( event === 'video' ) { + opt_video_assets = opt_assets || []; + opt_assets = []; + } else { + opt_video_assets = []; + opt_assets = opt_assets || []; + } var result = { doc: this.pageMeta, event: event, - assets: opt_assets || [], + video_assets: opt_video_assets, + assets: opt_assets, version: '3', + mrev: '5dacb94-c', msgNum: this.msgNum, timestamp: new Date().getTime(), pageVis: document.visibilityState, @@ -157,255 +169,342 @@ var exports = {}; (function(exports) { - var VALID_AD_SIZES = [ - [160, 600], - - [300, 250], - [300, 600], - [300, 1050], - - [336, 280], - [336, 850], - [468, 60], - [728, 90], - [728, 270], - [970, 66], - [970, 90], - [970, 125], - [970, 250], - [970, 400], - [970, 415], - [1280, 100] - ]; - - var PX_SIZE_TOL = 10; - var MIN_WINDOW_PX = 10; - var MAX_SEARCHES_PER_WINDOW = 10; - var MAX_SEARCHES_PER_ELEMENT = 2; - - function makeSizeSet(validAdSizes, sizeTol) { - var set = {}; - var i; - var xfuz; - var yfuz; - var size; - var width; - var height; - - for ( i = 0; i < validAdSizes.length; i++ ) { - for ( xfuz = -sizeTol; xfuz <= sizeTol; xfuz++ ) { - for ( yfuz = -sizeTol; yfuz <= sizeTol; yfuz++ ) { - size = validAdSizes[i]; - width = size[0] + xfuz; - height = size[1] + yfuz; - set[width + 'x' + height] = size; - } + var SizeMatcher = { + VALID_AD_SIZES: [ + [300, 50], + [320, 50], + [160, 600], + [300, 250], + [300, 600], + [300, 1050], + [336, 280], + [336, 850], + [468, 60], + [728, 90], + [728, 250], + [728, 270], + [970, 66], + [970, 90], + [970, 125], + [970, 250], + [970, 400], + [970, 415], + [1280, 100] + ], + + PX_SIZE_TOL: 10, + + getMatchedAdSize: function(width, height) { + + if ( !this.set ) { + this.set = this._makeSizeSet(); } - } - return set; - } - - var SIZE_SET = makeSizeSet(VALID_AD_SIZES, PX_SIZE_TOL); - - function elementIsAd(el) { - if ( typeof el.searches !== 'number' ) { - el.searches = 0; - } - - if ( el.searches >= MAX_SEARCHES_PER_ELEMENT ) { - return false; - } - el.searches += 1; - - var isImgWithoutSrc = el.tagName === 'IMG' && !el.src; - var isImgWithoutAnchor = el.tagName === 'IMG' && !(el.parentNode.tagName === 'A' || el.getAttribute('onclick')); + return this.set[Math.round(width) + 'x' + Math.round(height)]; + }, - return elementIsAdShaped(el) && !isImgWithoutSrc && !isImgWithoutAnchor; - } + elementIsAdShaped: function(el) { + return !!this.getMatchedAdSizeForElement(el); + }, - function isNewAd(el, win) { - return !el.mp_adFound && (win === win.top || !win.mp_adFound); - } + getMatchedAdSizeForElement: function(el) { + var rect = el.getBoundingClientRect(); + return this.getMatchedAdSize(rect.width, rect.height); + }, - function getFriendlyIframes(win) { - var iframes = win.document.querySelectorAll('iframe'); - iframes = exports.utils.realArray(iframes); - var friendlyIframes = iframes.filter(function(ifr) { - return exports.utils.isFriendlyWindow(ifr.contentWindow); - }); - return friendlyIframes; - } + _makeSizeSet: function() { + var set = {}; + var i; + var xfuz; + var yfuz; + var size; + var width; + var height; + + for ( i = 0; i < this.VALID_AD_SIZES.length; i++ ) { + for ( xfuz = -this.PX_SIZE_TOL; xfuz <= this.PX_SIZE_TOL; xfuz++ ) { + for ( yfuz = -this.PX_SIZE_TOL; yfuz <= this.PX_SIZE_TOL; yfuz++ ) { + size = this.VALID_AD_SIZES[i]; + width = size[0] + xfuz; + height = size[1] + yfuz; + set[width + 'x' + height] = size; + } + } + } + return set; + } + }; - function getMatchedAdSize(width, height) { - return SIZE_SET[width + 'x' + height]; - } + var Throttler = { + MAX_SEARCHES_PER_WINDOW: 10, + MAX_SEARCHES_PER_ELEMENT: 2, - function elementIsAdShaped(el) { - return !!getMatchedAdSizeForElement(el); - } + countSearch: function(el) { + if ( typeof el.searches !== 'number' ) { + el.searches = 0; + } - function getMatchedAdSizeForElement(el) { - var rect = el.getBoundingClientRect(); - return getMatchedAdSize(rect.width, rect.height); - } + el.searches += 1; + }, - function containsLargeIframes(win) { - var iframes = win.document.querySelectorAll('iframe'); - var rect; - var i; - for ( i = 0; i < iframes.length; i++ ) { - rect = iframes[i].getBoundingClientRect(); - if ( rect.width > 10 || rect.height > 10 ) { + throttle: function(el, max) { + if ( typeof el.searches === 'number' && el.searches >= max ) { return true; } - } - return false; - } - - function isValidHTML5Div(div, winSize) { - var elSize = getMatchedAdSizeForElement(div); - - if ( typeof div.checks !== 'number' ) { - div.checks = 1; - } else { - div.checks += 1; - } - - return (elSize && - elSize[0] === winSize[0] && elSize[1] === winSize[1] && - div.checks > 1); - } + return false; + }, - var HTML5_SIGNAL_ELEMENTS = 'canvas, button, video, svg, img'; - function iframeGetHTMLAd(win) { - var body = win.document.body, - elements, i, el, divs, div, numElements, - winSize, elSize; + throttleElement: function(el) { + return this.throttle(el, this.MAX_SEARCHES_PER_ELEMENT); + }, - if ( !body ) { - return null; - } - winSize = getMatchedAdSize(win.innerWidth, win.innerHeight); + throttleWin: function(win) { + return this.throttle(win, this.MAX_SEARCHES_PER_WINDOW); + }, - if ( !winSize ) { - return null; + getCount: function(el) { + return el.searches || 0; } + }; - elements = body.querySelectorAll(HTML5_SIGNAL_ELEMENTS); + function TopSearcher(win) { + this.win = win; + this.doc = win.document; + } - for ( i = 0; i < elements.length; i++ ) { - el = elements[i]; - elSize = getMatchedAdSizeForElement(el); - if ( elSize && elSize[0] === winSize[0] && elSize[1] === winSize[1] ) { - return el; + TopSearcher.prototype.search = function() { + var candidates = exports.utils.realArray(this.doc.querySelectorAll('img, object, embed')), + html5Ad, + ads = []; + + ads = ads.concat(candidates.filter(function(el) { + if ( !el.mpAdFound && !Throttler.throttleElement(el) ) { + Throttler.countSearch(el); + if ( (el.tagName !== 'IMG' || isStandardImage(el)) && SizeMatcher.elementIsAdShaped(el) ) { + el.mpAdFound = true; + return true; + } } - } + return false; + })); - numElements = body.querySelectorAll('*').length; - if ( numElements < 5 ) { - return null; + html5Ad = this._mainGetHTMLAd(); + if ( html5Ad ) { + html5Ad.html5 = true; + html5Ad.mpAdFound = true; + ads.push(html5Ad); } - divs = body.querySelectorAll('div'); + return ads; + }; - for ( i = 0; i < divs.length; i++ ) { - div = divs[i]; - if ( isValidHTML5Div(div, winSize) ) { + TopSearcher.prototype._mainGetHTMLAd = function() { + var styles = this.doc.querySelectorAll('div > style, div > link[rel="stylesheet"]'), + i, div; + for ( i = 0; i < styles.length; i++ ) { + div = styles[i].parentNode; + if ( !div.mpAdFound && SizeMatcher.elementIsAdShaped(div) && this._jumpedOut(div) ) { return div; } } + }; - return null; - } - - function jumpedOut(el) { + TopSearcher.prototype._jumpedOut = function(el) { var siblings, ifrs; siblings = exports.utils.realArray(el.parentNode.children); ifrs = siblings.filter(function(el) { return el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0; }); return ifrs.length > 0; - } + }; - function mainGetHTMLAd(win) { - var styles = win.document.querySelectorAll('div > style, div > link[rel="stylesheet"]'), - i, div; - for ( i = 0; i < styles.length; i++ ) { - div = styles[i].parentNode; - if ( elementIsAdShaped(div) && jumpedOut(div) ) { - return div; + function IframeSearcher(win) { + this.MIN_AD_AREA = 14000; + this.MIN_WINDOW_PX = 10; + + this.win = win; + this.doc = win.document; + this.body = win.document.body; + this.winClickTag = win.clickTag; + this.adSizeMeta = this._getAdSizeMeta(); + this.numElementsInBody = (this.body && this.body.querySelectorAll('*').length) || 0; + + this.shouldSearchWindow = false; + if ( !this.win.mpAdFound && this.body && !Throttler.throttleWin(this.win) ) { + this.winWidth = this.win.innerWidth; + this.winHeight = this.win.innerHeight; + if ( this._meetsMinAdSize(this.winWidth, this.winHeight) && !this._containsLargeIframes() ) { + this.shouldSearchWindow = true; } } + } - function findAds(win, opt_ads) { + IframeSearcher.prototype.search = function() { + var ad; - if ( typeof win.searches !== 'number' ) { - win.searches = 0; + if ( this.shouldSearchWindow ) { + ad = this._search(); + if ( ad ) { + ad.mpAdFound = true; + win.mpAdFound = true; + return ad; + } + Throttler.countSearch(this.win); } - var ads = opt_ads || []; - var adsFound = 0; + return null; + }; + + IframeSearcher.prototype._search = function() { + var _this = this, + stdCandidates, + html5Candidates, + stdEl, + html5El; + + stdCandidates = this.body.querySelectorAll('img, object, embed'); + + stdEl = getFirst(stdCandidates, function(el) { + if ( !el.mpAdFound && + !Throttler.throttleElement(el) && + (el.tagName !== 'IMG' || isStandardImage(el)) && + _this._elementIsAtLeastAsBigAsWindow(el)) + { + return true; + } + Throttler.countSearch(el); + return false; + }); - if ( win.innerWidth <= MIN_WINDOW_PX || win.innerHeight <= MIN_WINDOW_PX ) { - win.searches++; - return ads; + if ( stdEl ) { + return stdEl; } - if ( exports.utils.SCRIPT_IN_WINDOW_TOP || win.searches < MAX_SEARCHES_PER_WINDOW ) { - var adCandidates = win.document.querySelectorAll('img, object, embed'); - adCandidates = exports.utils.realArray(adCandidates); + if ( this._isHTML5Iframe() ) { + html5Candidates = this.doc.querySelectorAll('body, canvas, button, video, svg, div'); + html5El = getFirst(html5Candidates, function(el) { - adCandidates.forEach(function(el) { - if ( elementIsAd(el) && isNewAd(el, win) ) { - el.mp_adFound = true; - el.inIframe = win !== win.top; - win.mp_adFound = true; - ads.push(el); - adsFound += 1; + if ( _this._elementIsAtLeastAsBigAsWindow(el) ) { + return true; } + Throttler.countSearch(el); + return false; }); + } - var htmlAd, adSizeMeta; - if ( win === win.top ) { - htmlAd = mainGetHTMLAd(win); - } else { - if ( adsFound === 0 && !containsLargeIframes(win) ) { - htmlAd = iframeGetHTMLAd(win); - } - } + if ( html5El ) { + html5El.html5 = true; + html5El.winClickTag = this.winClickTag; + html5El.adSizeMeta = this.adSizeMeta; + return html5El; + } - if ( htmlAd && isNewAd(htmlAd, win) ) { - htmlAd.html5 = true; - htmlAd.inIframe = win !== win.top; - if ( htmlAd.inIframe ) { - adSizeMeta = win.document.querySelectorAll('meta[name="ad.size"]'); - if ( adSizeMeta.length > 0 ) { - htmlAd.adSizeMeta = adSizeMeta[0].content; - } - if ( win.clickTag ) { - htmlAd.winClickTag = win.clickTag; - } - } - htmlAd.mp_adFound = true; - win.mp_adFound = true; - ads.push(htmlAd); + return null; + }; + + IframeSearcher.prototype._isHTML5Iframe = function() { + if ( this.winClickTag || this.adSizeMeta ) { + return true; + } + + if ( this.doc.querySelectorAll('canvas', 'button', 'video', 'svg').length > 0 ) { + return true; + } + + if ( this.numElementsInBody >= 5 && Throttler.getCount(this.win) > 0 && this.doc.querySelectorAll('div').length > 0 ) { + return true; + } + + return false; + }; + + IframeSearcher.prototype._elementIsAtLeastAsBigAsWindow = function(el) { + var rect = el.getBoundingClientRect(), + tol = 0.95; + + return rect.width >= (tol * this.winWidth) && rect.height >= (tol * this.winHeight); + }; + + IframeSearcher.prototype._meetsMinAdSize = function(width, height) { + return (width * height) >= this.MIN_AD_AREA; + }; + + IframeSearcher.prototype._containsLargeIframes = function() { + var iframes = this.doc.querySelectorAll('iframe'); + var rect; + var i; + for ( i = 0; i < iframes.length; i++ ) { + rect = iframes[i].getBoundingClientRect(); + if ( rect.width > this.MIN_WINDOW_PX || rect.height > this.MIN_WINDOW_PX ) { + return true; } + } + return false; + }; - win.searches += 1; + IframeSearcher.prototype._getAdSizeMeta = function() { + var adSizeMeta = this.doc.querySelectorAll('meta[name="ad.size"]'); + if ( adSizeMeta.length > 0 ) { + return adSizeMeta[0].content; + } else { + return null; } + }; + + function getFirst(arr, testFn) { + var i, el; + for ( i = 0; i < arr.length; i++ ) { + el = arr[i]; + if ( testFn(el) ) { + return el; + } + } + return null; + } + + function isStandardImage(img) { + + return img.src && (img.parentNode.tagName === 'A' || img.getAttribute('onclick')); + } - var iframes = getFriendlyIframes(win); - iframes.forEach(function(ifr) { - findAds(ifr.contentWindow, ads); + function getFriendlyIframes(win) { + var iframes = win.document.querySelectorAll('iframe'); + iframes = exports.utils.realArray(iframes); + var friendlyIframes = iframes.filter(function(ifr) { + return exports.utils.isFriendlyWindow(ifr.contentWindow); }); + return friendlyIframes; + } + + function findAds(win) { + var i, + iframes, + searcher, + ad, + ads = []; + + if ( win === win.top ) { + searcher = new TopSearcher(win); + ads = ads.concat(searcher.search()); + } else { + searcher = new IframeSearcher(win); + ad = searcher.search(); + if ( ad ) { + ads.push(ad); + } + } + + iframes = getFriendlyIframes(win); + for ( i = 0; i < iframes.length; i++ ) { + ads = ads.concat(findAds(iframes[i].contentWindow)); + } return ads; } exports.adfinder = { - getMatchedAdSize: getMatchedAdSize, + getMatchedAdSize: SizeMatcher.getMatchedAdSize.bind(SizeMatcher), findAds: findAds }; })(exports); @@ -463,11 +562,12 @@ var exports = {}; getPosition: function(el) { var rect = el.getBoundingClientRect(); var win = exports.utils.elementWindow(el); + return { - width: rect.width, - height: rect.height, - left: rect.left + win.pageXOffset, - top: rect.top + win.pageYOffset + width: Math.round(rect.width), + height: Math.round(rect.height), + left: Math.round(rect.left + win.pageXOffset), + top: Math.round(rect.top + win.pageYOffset) }; }, @@ -682,7 +782,7 @@ var exports = {}; while ( true ) { parentContainer = curContainer.parentElement; - if ( this.isValidContainer(parentContainer) ) { + if ( parentContainer && this.isValidContainer(parentContainer) ) { curContainer = parentContainer; } else { return curContainer; @@ -693,12 +793,6 @@ var exports = {}; var tagfinder = { - prepToSend: function(adData) { - adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height); - delete adData.width; - delete adData.height; - }, - setPositions: function(adData, opt_el, opt_winPos) { var el = opt_el || adData.context; var winPos = opt_winPos || {left: 0, top: 0}; @@ -732,10 +826,12 @@ var exports = {}; while ( true ) { highestContainer = mgr.highestContainer(curWin, referenceElement); mgr.captureHTML(highestContainer); - if ( curWin === curWin.top ) { break; } else { + + curWin.mpAdFound = true; + mgr.adData.serializedIframeContents = mgr.adData.context; if ( exports.utils.isFriendlyWindow(curWin.parent) ) { @@ -763,6 +859,8 @@ var exports = {}; var _pageTags; var INIT_MS_BW_SEARCHES = 2000; var PAGE_TAG_RE = new RegExp('gpt|oascentral'); + var POST_MSG_ID = '1484952787-11883-24071-12354-1549'; + var AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)'); function getPageTags(doc) { var scripts = doc.getElementsByTagName('script'); @@ -778,7 +876,7 @@ var exports = {}; function messageAllParentFrames(adData) { - adData.dummyId = true; + adData.postMessageId = POST_MSG_ID; adData = JSON.stringify(adData); @@ -795,16 +893,31 @@ var exports = {}; messageAllParentFrames(adData); } else if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { + exports.tagfinder.setPositions(adData); - exports.tagfinder.prepToSend(adData); + adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height); + if ( !adData.matchedSize ) { + + if ( AD_SERVER_RE.exec(results.referenceElement.id) ) { + adData.matchedSize = [adData.width, adData.height]; + adData.oddSize = true; + } else { + + return; + } + } + delete adData.width; + delete adData.height; adData.curPageUrl = exports.utils.getPageUrl(); _pageTags = _pageTags || getPageTags(document); var log = _logGen.log('ad', [adData], _pageTags); + if ( _onAdFound ) { - _onAdFound(log, results.referenceElement); - } + _onAdFound(log, results.referenceElement); + + } } } @@ -819,26 +932,20 @@ var exports = {}; function extractAds() { var ads = exports.adfinder.findAds(window); - - if ( !ads ) { - return; - } - ads.forEach(function(ad) { var startTime = new Date().getTime(); var adId = startTime + '-' + Math.floor(Math.random() * 10e12); var adData = { - width: ad.offsetWidth, - height: ad.offsetHeight, + width: Math.round(ad.offsetWidth), + height: Math.round(ad.offsetHeight), startTime: startTime, adId: adId, - html5: ad.html5 || false, - inIframe: ad.inIframe + html5: ad.html5 || false }; - if ( ad.html5 && ad.inIframe ) { + if ( ad.html5 ) { adData.adSizeMeta = ad.adSizeMeta || null; adData.winClickTag = ad.winClickTag || null; } @@ -898,9 +1005,9 @@ var exports = {}; return; } - if ( adData.dummyId ) { + if ( adData.postMessageId === POST_MSG_ID ) { - delete adData.dummyId; + delete adData.postMessageId; if ( isChildWin(myWin, ifrWin) ) { if ( exports.utils.isFriendlyWindow(ifrWin) ) { @@ -917,6 +1024,40 @@ var exports = {}; } } + function onVideoMessage(msg, sender, callback) { + var log; + if ( msg.event === 'new-video-ad' ) { + msg.assets.forEach(function(asset) { + + }); + log = _logGen.log('video', msg.assets); + } else { + log = _logGen.log('invalid-video', msg.assets); + } + + msg.assets.forEach(function(a) {delete a.isVideo;}); + log.displayAdFound = msg.displayAdFound; + log.requests = msg.requests; + log.data = msg.event_data; + + log.doc.finalPageUrl = log.doc.url; + log.doc.url = exports.utils.normalizeUrl(msg.origUrl); + + _onAdFound(log); + } + + function addBackgroundListener(event, callback) { + if ( typeof chrome !== 'undefined' ) { + chrome.runtime.onMessage.addListener(function(msg) { + if ( msg.event === event ) { + callback(msg); + } + }); + } else if ( window.self.port ) { + window.self.port.on(event, callback); + } + } + exports.coordinator = { init: function(onAdFound) { @@ -925,19 +1066,21 @@ var exports = {}; } _onAdFound = onAdFound; + window.addEventListener('message', onPostMessage, false); if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { var log = _logGen.log('page'); onAdFound(log); - } - window.addEventListener('message', onPostMessage, false); - if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { window.addEventListener('beforeunload', function(event) { var log = _logGen.log('unload'); log.timing = window.performance.timing; onAdFound(log); }); + + addBackgroundListener('new-video-ad', onVideoMessage); + addBackgroundListener('new-invalid-video-ad', onVideoMessage); + } exports.utils.onDocLoaded(document, extractAdsWrapper); diff --git a/src/utils/network.js b/src/utils/network.js new file mode 100644 index 000000000..933c59fe7 --- /dev/null +++ b/src/utils/network.js @@ -0,0 +1,584 @@ +'use strict'; +(function() { + + var secBefore = 2000; + var secAfter = 5000; + var secBetweenDupAssets = 10e3; + var minVidSize = 500e3; + var maxVidSize = 25e6; + var maxContentRange = 25e6; + var videoExtensions = [ + 'af', '3gp', 'asf', 'avchd', 'avi', 'cam', 'dsh', 'flv', 'm1v', 'm2v', + 'fla', 'flr', 'sol', 'm4v', 'mkv', 'wrap', 'mng', 'mov', 'mpeg', 'mpg', + 'mpe', 'mp4', 'mxf', 'nsv', 'ogg', 'rm', 'svi', 'smi', 'wmv', 'webm' + ]; + var extensionsReg = new RegExp('\\.' + videoExtensions.join('$|\\.') + '$'); + var videoContentTypesPrefixes = ['binary/octet-stream', 'video/', 'flv-application/', 'media']; + + var bannedContentTypes = ['video/mp2t','video/f4m','video/f4f']; + var bannedFiletypes = ['ts']; + var bannedFiletypesReg = new RegExp('\\.' + bannedFiletypes.join('$|\\.') + '$'); + var whitelistReqTypes = ['object', 'xmlhttprequest', 'other']; + + var topVideoAssetDomains = [ + '2mdn.net', + 'innovid.com', + 'serving-sys.com', + 'btrll.com', + 'teads.tv', + 'tubemogul.com' + ]; + + if ( !String.prototype.endsWith ) { + String.prototype.endsWith = function(searchString, position) { + var subjectString = this.toString(); + if ( typeof position !== 'number' || !isFinite(position) || + Math.floor(position) !== position || position > subjectString.length) { + position = subjectString.length; + } + position -= searchString.length; + var lastIndex = subjectString.indexOf(searchString, position); + return lastIndex !== -1 && lastIndex === position; + }; + } + + function isPixelRequest(request) { + return (request.type === 'image' || request.responseStatus === 204) && + request.size <= 1000; + } + + function isVpaidOrVastRequest(request) { + var lowerCaseUrl = request.url.toLowerCase(); + return lowerCaseUrl.indexOf('vpaid') !== -1 || lowerCaseUrl.indexOf('vast') !== -1; + } + + function hasValidRequestType(request) { + return whitelistReqTypes.indexOf(request.type) >= 0; + } + + function stripQueryParams(url) { + return url.split('?',1)[0]; + } + + function parseHostnameFromUrl(url) { + var parser = document.createElement('a'); + parser.href = url; + return parser.hostname; + } + + function hasDomain(url, domain) { + return parseHostnameFromUrl(url).endsWith(domain); + } + + function findHeader(headers, key) { + var header; + for ( var i = 0; i < headers.length; i += 1 ) { + header = headers[i]; + if ( header.name.toLowerCase() === key ) { + return header; + } + } + return null; + } + + function validVideoType(vtype) { + var goodType = videoContentTypesPrefixes.some(function(prefix) { + return vtype.indexOf(prefix) === 0; + }); + return goodType; + } + + function assetMsgKey(assetReq) { + var url = stripQueryParams(assetReq.url); + var key = assetReq.frameId + '-' + url; + return key; + } + + var PageNetworkTrafficCollector = function(tabId) { + this.tabId = tabId; + this.displayAdFound = false; + this.requests = {}; + this.msgsBeingSent = {}; + this.assetsSeen = {}; + this.allRedirects = {}; + }; + + var globalPageContainer = { + collectors: {}, + dyingCollectors: {}, + + cleanupCollector: function(tabId) { + if ( tabId in this.collectors ) { + delete globalPageContainer.collectors[tabId]; + } + }, + + onNewNavigation: function(details) { + var tabId = details.tabId; + this.cleanupCollector(tabId); + + this.collectors[tabId] = new PageNetworkTrafficCollector(tabId); + }, + + onNavigationCommitted: function(details) { + + }, + + onNavigationCompleted: function(details) { + + }, + + onTabClose: function(tabId, closeInfo) { + + this.cleanupCollector(tabId); + delete this.collectors[tabId]; + }, + + onDisplayAdFound: function(tabId) { + this.collectors[tabId].displayAdFound = true; + }, + + getRandId: function() { + return String(Math.floor(Math.random() * 1e9)); + }, + + getCollector: function(tabId) { + if ( this.collectors.hasOwnProperty(tabId) ) { + return this.collectors[tabId]; + } + return null; + }, + + forwardCall: function(details, collectorMemberFunction) { + var collector = this.getCollector(details.tabId); + if ( collector !== null ) { + collectorMemberFunction.apply(collector, [details]); + } + } + }; + + PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() { + var logMessage = Array.from(arguments).join(' '); + var message = {message: logMessage, event: 'console-log-message'}; + chrome.tabs.sendMessage(this.tabId, message); + }; + + PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, isValidAd) { + var msg = {}; + msg.assets = []; + msg.event_data = {}; + if ( isValidAd ) { + msg.event = 'new-video-ad'; + msg.requests = reqs; + msg.requests.sort(function(reqA, reqB) {return reqA.requestTimestamp - reqB.requestTimestamp;}); + if ( assetReq ) { + msg.assets = [assetReq]; + } + } else { + msg.requests = reqs.map(function(request) { + return parseHostnameFromUrl(request.url); + }); + msg.assets = [{ + url: parseHostnameFromUrl(assetReq.url), + contentType: assetReq.contentType, + size: assetReq.size + }]; + msg.event = 'new-invalid-video-ad'; + } + msg.origUrl = curPageUrl; + msg.displayAdFound = this.displayAdFound; + + chrome.tabs.sendMessage(this.tabId, msg); + }; + + PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) { + return url + ':' + frameId; + }; + + PageNetworkTrafficCollector.prototype.seenBefore = function(request) { + var oldTime = this.assetsSeen[assetMsgKey(request)]; + if ( oldTime && (request.requestTimestamp-oldTime < secBetweenDupAssets)){ + + return true; + } + return false; + }; + + PageNetworkTrafficCollector.prototype.recordSeenAsset = function(request) { + this.assetsSeen[assetMsgKey(request)] = request.requestTimestamp; + }; + + PageNetworkTrafficCollector.prototype.onBeforeRequest = function(details) { + var req = { + url: details.url, + type: details.type, + httpMethod: details.method, + frameId: details.frameId, + parentFrameId: details.parentFrameId, + requestTimestamp: details.timeStamp, + }; + this.requests[details.requestId] = req; + }; + + PageNetworkTrafficCollector.prototype.onSendHeaders = function(details) { + var request, header; + request = this.requests[details.requestId]; + header = request && findHeader(details.requestHeaders, 'x-requested-with'); + if ( header && header.value.toLowerCase().indexOf('flash') > -1 ) { + request.from_flash = true; + } + }; + + PageNetworkTrafficCollector.prototype.onHeadersReceived = function(details) { + var getFrameDetails = { + tabId: details.tabId, + processId: null, + frameId: details.frameId + }; + var pageNetworkTrafficController = this; + chrome.webNavigation.getFrame(getFrameDetails, function(frameDetails) { + if ( frameDetails && frameDetails.url ) { + pageNetworkTrafficController._onHeadersReceived(details, frameDetails); + } + }); + }; + + PageNetworkTrafficCollector.prototype._onHeadersReceived = function(details, frameDetails) { + var contentSize, contentRange; + + var request = this.requests[details.requestId]; + if ( request ) { + var redirParent = this.allRedirects[this.getRedirKey(details.url, details.frameId)]; + var header = request && findHeader(details.responseHeaders, 'content-type'); + var contentType = header && header.value.toLowerCase(); + + if ( contentType){ + request.contentType = contentType; + } + header = request && findHeader(details.responseHeaders, 'content-length'); + contentSize = header && header.value; + if ( contentSize ) { + request.size = request.size || 0; + request.size += parseInt(contentSize); + } + header = request && findHeader(details.responseHeaders, 'content-range'); + contentRange = header && header.value; + if ( contentRange ) { + request.contentRange = parseInt(contentRange.split('/')[1]); + } + + var frameUrl = null; + if ( frameDetails && frameDetails.url ) { + frameUrl = frameDetails.url; + } + if ( !this.bannedRequest(request) && + (this.isVideoReq(frameUrl, request) || (redirParent && redirParent.isVideo))) { + request.isVideo = true; + } + } + }; + + PageNetworkTrafficCollector.prototype.onBeforeRedirect = function(details) { + var request = this.requests[details.requestId]; + if ( request ) { + if ( request.redirects ) { + request.redirects.push(details.redirectUrl); + } else { + request.redirects = [details.redirectUrl]; + } + this.allRedirects[this.getRedirKey(details.redirectUrl, details.frameId)] = request; + } + }; + + PageNetworkTrafficCollector.prototype.isYoutubeMastheadRequest = function(url) { + var re = /video_masthead/; + return this.hasYoutubeDomain(url) && re.test(url); + }; + PageNetworkTrafficCollector.prototype.isYoutubeVideoRequest = function(srcUrl, destUrl) { + if ( !this.hasYoutubeDomain(srcUrl) ) { + return false; + } + + var re = /https?:\/\/r.*?\.googlevideo\.com\/videoplayback\?/; + return re.test(destUrl); + }; + PageNetworkTrafficCollector.prototype.processResponse = function(requestDetails, frameDetails) { + var request; + if ( requestDetails ) { + request = this.requests[requestDetails.requestId]; + if ( request ) { + request.responseStatus = requestDetails.statusCode; + request.responseTimestamp = requestDetails.timeStamp; + + var frameUrl = null; + if ( frameDetails && frameDetails.url ) { + frameUrl = frameDetails.url; + } + + var requestUrl = null; + if ( request.url ) { + requestUrl = request.url; + } + + if ( this.isYoutubeAdReq(frameUrl, requestUrl) ) { + var videoId = this.parseYoutubeVideoIdFromUrl(requestUrl); + if ( videoId ) { + request.isYoutubeAd = true; + request.isVideo = true; + request.url = 'https://www.youtube.com/watch?v=' + this.parseYoutubeVideoIdFromUrl(requestUrl); + } + } else if ( !this.bannedRequest(request) && + (this.isVideo || this.isVideoReq(frameUrl, request))) { + request.isVideo = true; + } + + if ( request.isVideo ) { + + var msgKey = assetMsgKey(request); + this.msgsBeingSent[msgKey] = request; + if ( !this.seenBefore(request) ) { + this.sendMsgWhenQuiet(msgKey); + } + this.recordSeenAsset(request); + } + } + } + }; + + PageNetworkTrafficCollector.prototype.onResponseStarted = function(responseDetails) { + if ( responseDetails.frameId < 0 ) { + responseDetails.frameId = 99999; + + } + var getFrameDetails = { + tabId: responseDetails.tabId, + processId: null, + frameId: responseDetails.frameId + }; + var pageNetworkTrafficController = this; + chrome.webNavigation.getFrame(getFrameDetails, function(frameDetails) { + if ( frameDetails && frameDetails.url ) { + pageNetworkTrafficController.processResponse(responseDetails, frameDetails); + } + }); + }; + + PageNetworkTrafficCollector.prototype.hasBannedFiletype = function(request) { + var url = stripQueryParams(request.url); + if ( bannedFiletypesReg.exec(url) ) { + return true; + } else { + return false; + } + }; + + PageNetworkTrafficCollector.prototype.checkContentHeaders = function(request) { + if ( request.contentType && validVideoType(request.contentType) ) { + return true; + } + return false; + }; + + PageNetworkTrafficCollector.prototype.checkUrlExtension = function(request) { + var url = stripQueryParams(request.url); + if ( extensionsReg.exec(url) ) { + return true; + } else { + return false; + } + }; + + PageNetworkTrafficCollector.prototype.isVideoReq = function(srcUrl, request) { + if ( this.isYoutubeVideoRequest(srcUrl, request.url) ) { + return false; + } + return this.checkUrlExtension(request) || this.checkContentHeaders(request); + }; + PageNetworkTrafficCollector.prototype.hasYoutubeDomain = function(url) { + var hostname = parseHostnameFromUrl(url) ; + if ( hostname === 'www.youtube.com' ) { + return true; + } + return false; + }; + PageNetworkTrafficCollector.prototype.parseYoutubeVideoIdFromUrl = function(url) { + var re = /^https?:\/\/www\.youtube\.com\/get_video_info.*(?:\?|&)video_id=(.*?)(?:$|&)/; + var match = re.exec(url); + if ( match && match.length > 1 ) { + return match[1]; + } + + re = /^https?:\/\/www\.youtube\.com\/embed\/(.*?)(?:$|\?)/; + match = re.exec(url); + if ( match && match.length > 1 ) { + return match[1]; + } + + re = /^https?:\/\/www\.youtube\.com\/watch\?v=(.*$)/; + match = re.exec(url); + if ( match && match.length > 1 ) { + return match[1]; + } + return null; + }; + + PageNetworkTrafficCollector.prototype.isYoutubeGetVideoInfoReq = function(url) { + var re = /^https?:\/\/www\.youtube\.com\/get_video_info\?/; + return re.test(url); + }; + PageNetworkTrafficCollector.prototype.isYoutubeAdReq = function(srcUrl, destUrl) { + + if ( !this.hasYoutubeDomain(srcUrl) || + !this.isYoutubeGetVideoInfoReq(destUrl)) { + return false; + } + if ( this.parseYoutubeVideoIdFromUrl(srcUrl) === + this.parseYoutubeVideoIdFromUrl(destUrl) && + !this.isYoutubeMastheadRequest(destUrl)) { + return false; + } + return true; + }; + + PageNetworkTrafficCollector.prototype.bannedRequest = function(request) { + return this.bannedVideoType(request) || this.hasBannedFiletype(request) || this.bannedVideoSize(request); + }; + + PageNetworkTrafficCollector.prototype.bannedVideoType = function(request) { + var badType = false; + if ( request.contentType ) { + badType = bannedContentTypes.some(function(prefix) { + return request.contentType.indexOf(prefix) >= 0; + }); + } + return badType; + }; + + PageNetworkTrafficCollector.prototype.bannedVideoSize = function(request) { + if ( request.size !== null ) { + if ( request.size < minVidSize || request.size > maxVidSize || request.contentRange > maxContentRange ) { + return true; + } + } + return false; + }; + + PageNetworkTrafficCollector.prototype.grabTagReqs = function(tabRequests, assetRequest) { + var minTimestamp, maxTimestamp; + minTimestamp = assetRequest.requestTimestamp - secBefore; + maxTimestamp = assetRequest.requestTimestamp + secAfter; + + var filteredRequests = tabRequests.filter(function(request) { + return (request.requestTimestamp > minTimestamp && + request.requestTimestamp < maxTimestamp && + request.frameId === assetRequest.frameId && + request.url !== assetRequest.url && + (hasValidRequestType(request) || + isPixelRequest(request))); + }); + + return filteredRequests; + }; + + PageNetworkTrafficCollector.prototype.isValidVideoAd = function(assetRequest, tagRequests) { + var hasVpaidOrVastRequest = tagRequests.some(function(tagRequest) { + return isVpaidOrVastRequest(tagRequest); + }); + if ( hasVpaidOrVastRequest ) { + return true; + } + var hasTopVideoAssetDomain = topVideoAssetDomains.some(function(assetDomain) { + return hasDomain(assetRequest.url, assetDomain); + }); + + return hasTopVideoAssetDomain; + }; + + PageNetworkTrafficCollector.prototype.sendMsgWhenQuiet = function(msgKey) { + var _this = this, + origPageUrl, msgAssetReq; + msgAssetReq = this.msgsBeingSent[msgKey]; + chrome.tabs.get(this.tabId, function(tab) {origPageUrl = tab.url;}); + + setTimeout(function() { + var rawRequests = []; + if ( globalPageContainer.collectors[_this.tabId] === _this ) { + for ( var reqId in _this.requests ) { + rawRequests.push(_this.requests[reqId]); + } + var tagReqs = _this.grabTagReqs(rawRequests, msgAssetReq); + + if ( _this.isValidVideoAd(msgAssetReq, tagReqs) ) { + _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, true); + } else { + + _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, false); + } + + } else { + + } + delete _this.msgsBeingSent[msgKey]; + }, secAfter+secBefore); + }; + + PageNetworkTrafficCollector.prototype.existingMessage = function(candidateRequest) { + var frameMsg = this.msgsBeingSent[candidateRequest.frameId]; + if ( frameMsg ) { + return frameMsg; + } else { + return null; + } + }; + + chrome.webRequest.onBeforeRequest.addListener(function(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRequest); + }, {urls: ['http://*/*', 'https://*/*']}, []); + + chrome.webRequest.onSendHeaders.addListener(function(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onSendHeaders); + }, {urls: ['http://*/*', 'https://*/*']}, ['requestHeaders']); + + chrome.webRequest.onHeadersReceived.addListener(function(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onHeadersReceived); + }, {urls: ['http://*/*', 'https://*/*']}, ['responseHeaders']); + + chrome.webRequest.onBeforeRedirect.addListener(function(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRedirect); + }, {urls: ['http://*/*', 'https://*/*']}, []); + + chrome.webRequest.onResponseStarted.addListener(function(details) { + globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onResponseStarted); + }, {urls: ['http://*/*', 'https://*/*']}, ['responseHeaders']); + + chrome.webNavigation.onBeforeNavigate.addListener(function(details) { + if ( details.frameId === 0 ) { + globalPageContainer.onNewNavigation(details); + } + }, {}); + + chrome.webNavigation.onCommitted.addListener(function(details) { + if ( details.frameId === 0 ) { + globalPageContainer.onNavigationCommitted(details); + } + }); + + chrome.webNavigation.onCompleted.addListener(function(details) { + if ( details.frameId === 0 ) { + globalPageContainer.onNavigationCompleted(details); + } + }); + + chrome.tabs.onRemoved.addListener(function(tabId, closeInfo) { + globalPageContainer.onTabClose(tabId, closeInfo); + }); + + chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) { + if ( message.event === 'new-ad' && message.data.event === 'ad' ) { + var tabId = sender.tab.id; + if ( tabId ) { + globalPageContainer.onDisplayAdFound(tabId); + } + } + }); +})();