Ignore tags file

main
Elbert Alias 9 years ago
parent b0bc61bca7
commit 2bcb73b4ba

2
.gitignore vendored

@ -6,4 +6,6 @@ src/icons/converted/*
node_modules node_modules
npm-debug.log npm-debug.log
tags
!.gitkeep !.gitkeep

@ -30,11 +30,11 @@ echo "Creating hard links..."
ln -f $path/wappalyzer.js $path/drivers/phantomjs ln -f $path/wappalyzer.js $path/drivers/phantomjs
ln -f $path/apps.json $path/drivers/phantomjs ln -f $path/apps.json $path/drivers/phantomjs
ln -f $path/wappalyzer.js $path/drivers/firefox/lib ln -f $path/wappalyzer.js $path/drivers/firefox/lib
ln -f $path/apps.json $path/drivers/firefox/data ln -f $path/apps.json $path/drivers/firefox/data
ln -f $path/icons/*.png $path/drivers/firefox/data/images/icons ln -f $path/icons/*.png $path/drivers/firefox/data/images/icons
ln -f $path/icons/*.svg $path/drivers/firefox/data/images/icons ln -f $path/icons/*.svg $path/drivers/firefox/data/images/icons
ln -f $path/utils/*.js $path/drivers/firefox/data/js ln -f $path/utils/iframe.js $path/drivers/firefox/data/js
ln -f $path/wappalyzer.js $path/drivers/chrome/js ln -f $path/wappalyzer.js $path/drivers/chrome/js
ln -f $path/apps.json $path/drivers/chrome ln -f $path/apps.json $path/drivers/chrome

@ -4,3 +4,4 @@ images/icons/*.png
images/icons/*.svg images/icons/*.svg
js/wappalyzer.js js/wappalyzer.js
js/iframe.js js/iframe.js
js/network.js

@ -8,6 +8,7 @@
<script src="js/wappalyzer.js"></script> <script src="js/wappalyzer.js"></script>
<script src="js/defaults.js"></script> <script src="js/defaults.js"></script>
<script src="js/driver.js"></script> <script src="js/driver.js"></script>
<script src="js/network.js"></script>
</head> </head>
<body> <body>
</body> </body>

@ -29,6 +29,6 @@
"js/inject.js" "js/inject.js"
], ],
"options_page": "options.html", "options_page": "options.html",
"permissions": [ "tabs", "webRequest", "http://*/*", "https://*/*" ], "permissions": [ "tabs", "webRequest", "webNavigation", "http://*/*", "https://*/*" ],
"content_security_policy": "script-src 'self' https://ssl.google-analytics.com; object-src 'self'" "content_security_policy": "script-src 'self' https://ssl.google-analytics.com; object-src 'self'"
} }

@ -7,18 +7,19 @@ var exports = {};
(function(exports) { (function(exports) {
var utils = { var utils = {
getReferrer: function() { normalizeUrl: function(url) {
return this.hashUrl(document.referrer) || null; return this.hashUrl(url) || null;
}, },
getPageUrl: function() { getReferrer: function() {
return this.normalizeUrl(document.referrer);
return this.hashUrl(window.location.href) || null;
}, },
getPageUrl: function() {
return this.normalizeUrl(window.location.href);
},
hashUrl: function(url) { hashUrl: function(url) {
var a, var a,
result; result;
@ -121,6 +122,7 @@ var exports = {};
utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent); utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent);
utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME; utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME;
function LogGenerator() { function LogGenerator() {
this.msgNum = 0; this.msgNum = 0;
this.pageMeta = { this.pageMeta = {
@ -134,11 +136,21 @@ var exports = {};
LogGenerator.prototype = { LogGenerator.prototype = {
log: function(event, opt_assets, opt_pageTags) { log: function(event, opt_assets, opt_pageTags) {
var opt_video_assets;
if ( event === 'video' ) {
opt_video_assets = opt_assets || [];
opt_assets = [];
} else {
opt_video_assets = [];
opt_assets = opt_assets || [];
}
var result = { var result = {
doc: this.pageMeta, doc: this.pageMeta,
event: event, event: event,
assets: opt_assets || [], video_assets: opt_video_assets,
assets: opt_assets,
version: '3', version: '3',
mrev: '5dacb94-c',
msgNum: this.msgNum, msgNum: this.msgNum,
timestamp: new Date().getTime(), timestamp: new Date().getTime(),
pageVis: document.visibilityState, pageVis: document.visibilityState,
@ -157,255 +169,342 @@ var exports = {};
(function(exports) { (function(exports) {
var VALID_AD_SIZES = [ var SizeMatcher = {
[160, 600], VALID_AD_SIZES: [
[300, 50],
[300, 250], [320, 50],
[300, 600], [160, 600],
[300, 1050], [300, 250],
[300, 600],
[336, 280], [300, 1050],
[336, 850], [336, 280],
[468, 60], [336, 850],
[728, 90], [468, 60],
[728, 270], [728, 90],
[970, 66], [728, 250],
[970, 90], [728, 270],
[970, 125], [970, 66],
[970, 250], [970, 90],
[970, 400], [970, 125],
[970, 415], [970, 250],
[1280, 100] [970, 400],
]; [970, 415],
[1280, 100]
var PX_SIZE_TOL = 10; ],
var MIN_WINDOW_PX = 10;
var MAX_SEARCHES_PER_WINDOW = 10; PX_SIZE_TOL: 10,
var MAX_SEARCHES_PER_ELEMENT = 2;
getMatchedAdSize: function(width, height) {
function makeSizeSet(validAdSizes, sizeTol) {
var set = {}; if ( !this.set ) {
var i; this.set = this._makeSizeSet();
var xfuz;
var yfuz;
var size;
var width;
var height;
for ( i = 0; i < validAdSizes.length; i++ ) {
for ( xfuz = -sizeTol; xfuz <= sizeTol; xfuz++ ) {
for ( yfuz = -sizeTol; yfuz <= sizeTol; yfuz++ ) {
size = validAdSizes[i];
width = size[0] + xfuz;
height = size[1] + yfuz;
set[width + 'x' + height] = size;
}
} }
}
return set;
}
var SIZE_SET = makeSizeSet(VALID_AD_SIZES, PX_SIZE_TOL);
function elementIsAd(el) {
if ( typeof el.searches !== 'number' ) {
el.searches = 0;
}
if ( el.searches >= MAX_SEARCHES_PER_ELEMENT ) {
return false;
}
el.searches += 1; return this.set[Math.round(width) + 'x' + Math.round(height)];
},
var isImgWithoutSrc = el.tagName === 'IMG' && !el.src;
var isImgWithoutAnchor = el.tagName === 'IMG' && !(el.parentNode.tagName === 'A' || el.getAttribute('onclick'));
return elementIsAdShaped(el) && !isImgWithoutSrc && !isImgWithoutAnchor; elementIsAdShaped: function(el) {
} return !!this.getMatchedAdSizeForElement(el);
},
function isNewAd(el, win) { getMatchedAdSizeForElement: function(el) {
return !el.mp_adFound && (win === win.top || !win.mp_adFound); var rect = el.getBoundingClientRect();
} return this.getMatchedAdSize(rect.width, rect.height);
},
function getFriendlyIframes(win) { _makeSizeSet: function() {
var iframes = win.document.querySelectorAll('iframe'); var set = {};
iframes = exports.utils.realArray(iframes); var i;
var friendlyIframes = iframes.filter(function(ifr) { var xfuz;
return exports.utils.isFriendlyWindow(ifr.contentWindow); var yfuz;
}); var size;
return friendlyIframes; var width;
} var height;
for ( i = 0; i < this.VALID_AD_SIZES.length; i++ ) {
for ( xfuz = -this.PX_SIZE_TOL; xfuz <= this.PX_SIZE_TOL; xfuz++ ) {
for ( yfuz = -this.PX_SIZE_TOL; yfuz <= this.PX_SIZE_TOL; yfuz++ ) {
size = this.VALID_AD_SIZES[i];
width = size[0] + xfuz;
height = size[1] + yfuz;
set[width + 'x' + height] = size;
}
}
}
return set;
}
};
function getMatchedAdSize(width, height) { var Throttler = {
return SIZE_SET[width + 'x' + height]; MAX_SEARCHES_PER_WINDOW: 10,
} MAX_SEARCHES_PER_ELEMENT: 2,
function elementIsAdShaped(el) { countSearch: function(el) {
return !!getMatchedAdSizeForElement(el); if ( typeof el.searches !== 'number' ) {
} el.searches = 0;
}
function getMatchedAdSizeForElement(el) { el.searches += 1;
var rect = el.getBoundingClientRect(); },
return getMatchedAdSize(rect.width, rect.height);
}
function containsLargeIframes(win) { throttle: function(el, max) {
var iframes = win.document.querySelectorAll('iframe'); if ( typeof el.searches === 'number' && el.searches >= max ) {
var rect;
var i;
for ( i = 0; i < iframes.length; i++ ) {
rect = iframes[i].getBoundingClientRect();
if ( rect.width > 10 || rect.height > 10 ) {
return true; return true;
} }
} return false;
return false; },
}
function isValidHTML5Div(div, winSize) {
var elSize = getMatchedAdSizeForElement(div);
if ( typeof div.checks !== 'number' ) {
div.checks = 1;
} else {
div.checks += 1;
}
return (elSize &&
elSize[0] === winSize[0] && elSize[1] === winSize[1] &&
div.checks > 1);
}
var HTML5_SIGNAL_ELEMENTS = 'canvas, button, video, svg, img'; throttleElement: function(el) {
function iframeGetHTMLAd(win) { return this.throttle(el, this.MAX_SEARCHES_PER_ELEMENT);
var body = win.document.body, },
elements, i, el, divs, div, numElements,
winSize, elSize;
if ( !body ) { throttleWin: function(win) {
return null; return this.throttle(win, this.MAX_SEARCHES_PER_WINDOW);
} },
winSize = getMatchedAdSize(win.innerWidth, win.innerHeight);
if ( !winSize ) { getCount: function(el) {
return null; return el.searches || 0;
} }
};
elements = body.querySelectorAll(HTML5_SIGNAL_ELEMENTS); function TopSearcher(win) {
this.win = win;
this.doc = win.document;
}
for ( i = 0; i < elements.length; i++ ) { TopSearcher.prototype.search = function() {
el = elements[i]; var candidates = exports.utils.realArray(this.doc.querySelectorAll('img, object, embed')),
elSize = getMatchedAdSizeForElement(el); html5Ad,
if ( elSize && elSize[0] === winSize[0] && elSize[1] === winSize[1] ) { ads = [];
return el;
ads = ads.concat(candidates.filter(function(el) {
if ( !el.mpAdFound && !Throttler.throttleElement(el) ) {
Throttler.countSearch(el);
if ( (el.tagName !== 'IMG' || isStandardImage(el)) && SizeMatcher.elementIsAdShaped(el) ) {
el.mpAdFound = true;
return true;
}
} }
} return false;
}));
numElements = body.querySelectorAll('*').length; html5Ad = this._mainGetHTMLAd();
if ( numElements < 5 ) { if ( html5Ad ) {
return null; html5Ad.html5 = true;
html5Ad.mpAdFound = true;
ads.push(html5Ad);
} }
divs = body.querySelectorAll('div'); return ads;
};
for ( i = 0; i < divs.length; i++ ) { TopSearcher.prototype._mainGetHTMLAd = function() {
div = divs[i]; var styles = this.doc.querySelectorAll('div > style, div > link[rel="stylesheet"]'),
if ( isValidHTML5Div(div, winSize) ) { i, div;
for ( i = 0; i < styles.length; i++ ) {
div = styles[i].parentNode;
if ( !div.mpAdFound && SizeMatcher.elementIsAdShaped(div) && this._jumpedOut(div) ) {
return div; return div;
} }
} }
};
return null; TopSearcher.prototype._jumpedOut = function(el) {
}
function jumpedOut(el) {
var siblings, ifrs; var siblings, ifrs;
siblings = exports.utils.realArray(el.parentNode.children); siblings = exports.utils.realArray(el.parentNode.children);
ifrs = siblings.filter(function(el) { ifrs = siblings.filter(function(el) {
return el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0; return el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0;
}); });
return ifrs.length > 0; return ifrs.length > 0;
} };
function mainGetHTMLAd(win) { function IframeSearcher(win) {
var styles = win.document.querySelectorAll('div > style, div > link[rel="stylesheet"]'), this.MIN_AD_AREA = 14000;
i, div; this.MIN_WINDOW_PX = 10;
for ( i = 0; i < styles.length; i++ ) {
div = styles[i].parentNode; this.win = win;
if ( elementIsAdShaped(div) && jumpedOut(div) ) { this.doc = win.document;
return div; this.body = win.document.body;
this.winClickTag = win.clickTag;
this.adSizeMeta = this._getAdSizeMeta();
this.numElementsInBody = (this.body && this.body.querySelectorAll('*').length) || 0;
this.shouldSearchWindow = false;
if ( !this.win.mpAdFound && this.body && !Throttler.throttleWin(this.win) ) {
this.winWidth = this.win.innerWidth;
this.winHeight = this.win.innerHeight;
if ( this._meetsMinAdSize(this.winWidth, this.winHeight) && !this._containsLargeIframes() ) {
this.shouldSearchWindow = true;
} }
} }
} }
function findAds(win, opt_ads) { IframeSearcher.prototype.search = function() {
var ad;
if ( typeof win.searches !== 'number' ) { if ( this.shouldSearchWindow ) {
win.searches = 0; ad = this._search();
if ( ad ) {
ad.mpAdFound = true;
win.mpAdFound = true;
return ad;
}
Throttler.countSearch(this.win);
} }
var ads = opt_ads || []; return null;
var adsFound = 0; };
IframeSearcher.prototype._search = function() {
var _this = this,
stdCandidates,
html5Candidates,
stdEl,
html5El;
stdCandidates = this.body.querySelectorAll('img, object, embed');
stdEl = getFirst(stdCandidates, function(el) {
if ( !el.mpAdFound &&
!Throttler.throttleElement(el) &&
(el.tagName !== 'IMG' || isStandardImage(el)) &&
_this._elementIsAtLeastAsBigAsWindow(el))
{
return true;
}
Throttler.countSearch(el);
return false;
});
if ( win.innerWidth <= MIN_WINDOW_PX || win.innerHeight <= MIN_WINDOW_PX ) { if ( stdEl ) {
win.searches++; return stdEl;
return ads;
} }
if ( exports.utils.SCRIPT_IN_WINDOW_TOP || win.searches < MAX_SEARCHES_PER_WINDOW ) { if ( this._isHTML5Iframe() ) {
var adCandidates = win.document.querySelectorAll('img, object, embed'); html5Candidates = this.doc.querySelectorAll('body, canvas, button, video, svg, div');
adCandidates = exports.utils.realArray(adCandidates); html5El = getFirst(html5Candidates, function(el) {
adCandidates.forEach(function(el) { if ( _this._elementIsAtLeastAsBigAsWindow(el) ) {
if ( elementIsAd(el) && isNewAd(el, win) ) { return true;
el.mp_adFound = true;
el.inIframe = win !== win.top;
win.mp_adFound = true;
ads.push(el);
adsFound += 1;
} }
Throttler.countSearch(el);
return false;
}); });
}
var htmlAd, adSizeMeta; if ( html5El ) {
if ( win === win.top ) { html5El.html5 = true;
htmlAd = mainGetHTMLAd(win); html5El.winClickTag = this.winClickTag;
} else { html5El.adSizeMeta = this.adSizeMeta;
if ( adsFound === 0 && !containsLargeIframes(win) ) { return html5El;
htmlAd = iframeGetHTMLAd(win); }
}
}
if ( htmlAd && isNewAd(htmlAd, win) ) { return null;
htmlAd.html5 = true; };
htmlAd.inIframe = win !== win.top;
if ( htmlAd.inIframe ) { IframeSearcher.prototype._isHTML5Iframe = function() {
adSizeMeta = win.document.querySelectorAll('meta[name="ad.size"]'); if ( this.winClickTag || this.adSizeMeta ) {
if ( adSizeMeta.length > 0 ) { return true;
htmlAd.adSizeMeta = adSizeMeta[0].content; }
}
if ( win.clickTag ) { if ( this.doc.querySelectorAll('canvas', 'button', 'video', 'svg').length > 0 ) {
htmlAd.winClickTag = win.clickTag; return true;
} }
}
htmlAd.mp_adFound = true; if ( this.numElementsInBody >= 5 && Throttler.getCount(this.win) > 0 && this.doc.querySelectorAll('div').length > 0 ) {
win.mp_adFound = true; return true;
ads.push(htmlAd); }
return false;
};
IframeSearcher.prototype._elementIsAtLeastAsBigAsWindow = function(el) {
var rect = el.getBoundingClientRect(),
tol = 0.95;
return rect.width >= (tol * this.winWidth) && rect.height >= (tol * this.winHeight);
};
IframeSearcher.prototype._meetsMinAdSize = function(width, height) {
return (width * height) >= this.MIN_AD_AREA;
};
IframeSearcher.prototype._containsLargeIframes = function() {
var iframes = this.doc.querySelectorAll('iframe');
var rect;
var i;
for ( i = 0; i < iframes.length; i++ ) {
rect = iframes[i].getBoundingClientRect();
if ( rect.width > this.MIN_WINDOW_PX || rect.height > this.MIN_WINDOW_PX ) {
return true;
} }
}
return false;
};
win.searches += 1; IframeSearcher.prototype._getAdSizeMeta = function() {
var adSizeMeta = this.doc.querySelectorAll('meta[name="ad.size"]');
if ( adSizeMeta.length > 0 ) {
return adSizeMeta[0].content;
} else {
return null;
} }
};
function getFirst(arr, testFn) {
var i, el;
for ( i = 0; i < arr.length; i++ ) {
el = arr[i];
if ( testFn(el) ) {
return el;
}
}
return null;
}
function isStandardImage(img) {
return img.src && (img.parentNode.tagName === 'A' || img.getAttribute('onclick'));
}
var iframes = getFriendlyIframes(win); function getFriendlyIframes(win) {
iframes.forEach(function(ifr) { var iframes = win.document.querySelectorAll('iframe');
findAds(ifr.contentWindow, ads); iframes = exports.utils.realArray(iframes);
var friendlyIframes = iframes.filter(function(ifr) {
return exports.utils.isFriendlyWindow(ifr.contentWindow);
}); });
return friendlyIframes;
}
function findAds(win) {
var i,
iframes,
searcher,
ad,
ads = [];
if ( win === win.top ) {
searcher = new TopSearcher(win);
ads = ads.concat(searcher.search());
} else {
searcher = new IframeSearcher(win);
ad = searcher.search();
if ( ad ) {
ads.push(ad);
}
}
iframes = getFriendlyIframes(win);
for ( i = 0; i < iframes.length; i++ ) {
ads = ads.concat(findAds(iframes[i].contentWindow));
}
return ads; return ads;
} }
exports.adfinder = { exports.adfinder = {
getMatchedAdSize: getMatchedAdSize, getMatchedAdSize: SizeMatcher.getMatchedAdSize.bind(SizeMatcher),
findAds: findAds findAds: findAds
}; };
})(exports); })(exports);
@ -463,11 +562,12 @@ var exports = {};
getPosition: function(el) { getPosition: function(el) {
var rect = el.getBoundingClientRect(); var rect = el.getBoundingClientRect();
var win = exports.utils.elementWindow(el); var win = exports.utils.elementWindow(el);
return { return {
width: rect.width, width: Math.round(rect.width),
height: rect.height, height: Math.round(rect.height),
left: rect.left + win.pageXOffset, left: Math.round(rect.left + win.pageXOffset),
top: rect.top + win.pageYOffset top: Math.round(rect.top + win.pageYOffset)
}; };
}, },
@ -682,7 +782,7 @@ var exports = {};
while ( true ) { while ( true ) {
parentContainer = curContainer.parentElement; parentContainer = curContainer.parentElement;
if ( this.isValidContainer(parentContainer) ) { if ( parentContainer && this.isValidContainer(parentContainer) ) {
curContainer = parentContainer; curContainer = parentContainer;
} else { } else {
return curContainer; return curContainer;
@ -693,12 +793,6 @@ var exports = {};
var tagfinder = { var tagfinder = {
prepToSend: function(adData) {
adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height);
delete adData.width;
delete adData.height;
},
setPositions: function(adData, opt_el, opt_winPos) { setPositions: function(adData, opt_el, opt_winPos) {
var el = opt_el || adData.context; var el = opt_el || adData.context;
var winPos = opt_winPos || {left: 0, top: 0}; var winPos = opt_winPos || {left: 0, top: 0};
@ -732,10 +826,12 @@ var exports = {};
while ( true ) { while ( true ) {
highestContainer = mgr.highestContainer(curWin, referenceElement); highestContainer = mgr.highestContainer(curWin, referenceElement);
mgr.captureHTML(highestContainer); mgr.captureHTML(highestContainer);
if ( curWin === curWin.top ) { if ( curWin === curWin.top ) {
break; break;
} else { } else {
curWin.mpAdFound = true;
mgr.adData.serializedIframeContents = mgr.adData.context; mgr.adData.serializedIframeContents = mgr.adData.context;
if ( exports.utils.isFriendlyWindow(curWin.parent) ) { if ( exports.utils.isFriendlyWindow(curWin.parent) ) {
@ -763,6 +859,8 @@ var exports = {};
var _pageTags; var _pageTags;
var INIT_MS_BW_SEARCHES = 2000; var INIT_MS_BW_SEARCHES = 2000;
var PAGE_TAG_RE = new RegExp('gpt|oascentral'); var PAGE_TAG_RE = new RegExp('gpt|oascentral');
var POST_MSG_ID = '1484952787-11883-24071-12354-1549';
var AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)');
function getPageTags(doc) { function getPageTags(doc) {
var scripts = doc.getElementsByTagName('script'); var scripts = doc.getElementsByTagName('script');
@ -778,7 +876,7 @@ var exports = {};
function messageAllParentFrames(adData) { function messageAllParentFrames(adData) {
adData.dummyId = true; adData.postMessageId = POST_MSG_ID;
adData = JSON.stringify(adData); adData = JSON.stringify(adData);
@ -795,16 +893,31 @@ var exports = {};
messageAllParentFrames(adData); messageAllParentFrames(adData);
} else if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { } else if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
exports.tagfinder.setPositions(adData); exports.tagfinder.setPositions(adData);
exports.tagfinder.prepToSend(adData);
adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height);
if ( !adData.matchedSize ) {
if ( AD_SERVER_RE.exec(results.referenceElement.id) ) {
adData.matchedSize = [adData.width, adData.height];
adData.oddSize = true;
} else {
return;
}
}
delete adData.width;
delete adData.height;
adData.curPageUrl = exports.utils.getPageUrl(); adData.curPageUrl = exports.utils.getPageUrl();
_pageTags = _pageTags || getPageTags(document); _pageTags = _pageTags || getPageTags(document);
var log = _logGen.log('ad', [adData], _pageTags); var log = _logGen.log('ad', [adData], _pageTags);
if ( _onAdFound ) { if ( _onAdFound ) {
_onAdFound(log, results.referenceElement);
}
_onAdFound(log, results.referenceElement);
}
} }
} }
@ -819,26 +932,20 @@ var exports = {};
function extractAds() { function extractAds() {
var ads = exports.adfinder.findAds(window); var ads = exports.adfinder.findAds(window);
if ( !ads ) {
return;
}
ads.forEach(function(ad) { ads.forEach(function(ad) {
var startTime = new Date().getTime(); var startTime = new Date().getTime();
var adId = startTime + '-' + Math.floor(Math.random() * 10e12); var adId = startTime + '-' + Math.floor(Math.random() * 10e12);
var adData = { var adData = {
width: ad.offsetWidth, width: Math.round(ad.offsetWidth),
height: ad.offsetHeight, height: Math.round(ad.offsetHeight),
startTime: startTime, startTime: startTime,
adId: adId, adId: adId,
html5: ad.html5 || false, html5: ad.html5 || false
inIframe: ad.inIframe
}; };
if ( ad.html5 && ad.inIframe ) { if ( ad.html5 ) {
adData.adSizeMeta = ad.adSizeMeta || null; adData.adSizeMeta = ad.adSizeMeta || null;
adData.winClickTag = ad.winClickTag || null; adData.winClickTag = ad.winClickTag || null;
} }
@ -898,9 +1005,9 @@ var exports = {};
return; return;
} }
if ( adData.dummyId ) { if ( adData.postMessageId === POST_MSG_ID ) {
delete adData.dummyId; delete adData.postMessageId;
if ( isChildWin(myWin, ifrWin) ) { if ( isChildWin(myWin, ifrWin) ) {
if ( exports.utils.isFriendlyWindow(ifrWin) ) { if ( exports.utils.isFriendlyWindow(ifrWin) ) {
@ -917,6 +1024,40 @@ var exports = {};
} }
} }
function onVideoMessage(msg, sender, callback) {
var log;
if ( msg.event === 'new-video-ad' ) {
msg.assets.forEach(function(asset) {
});
log = _logGen.log('video', msg.assets);
} else {
log = _logGen.log('invalid-video', msg.assets);
}
msg.assets.forEach(function(a) {delete a.isVideo;});
log.displayAdFound = msg.displayAdFound;
log.requests = msg.requests;
log.data = msg.event_data;
log.doc.finalPageUrl = log.doc.url;
log.doc.url = exports.utils.normalizeUrl(msg.origUrl);
_onAdFound(log);
}
function addBackgroundListener(event, callback) {
if ( typeof chrome !== 'undefined' ) {
chrome.runtime.onMessage.addListener(function(msg) {
if ( msg.event === event ) {
callback(msg);
}
});
} else if ( window.self.port ) {
window.self.port.on(event, callback);
}
}
exports.coordinator = { exports.coordinator = {
init: function(onAdFound) { init: function(onAdFound) {
@ -925,19 +1066,21 @@ var exports = {};
} }
_onAdFound = onAdFound; _onAdFound = onAdFound;
window.addEventListener('message', onPostMessage, false);
if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
var log = _logGen.log('page'); var log = _logGen.log('page');
onAdFound(log); onAdFound(log);
}
window.addEventListener('message', onPostMessage, false);
if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
window.addEventListener('beforeunload', function(event) { window.addEventListener('beforeunload', function(event) {
var log = _logGen.log('unload'); var log = _logGen.log('unload');
log.timing = window.performance.timing; log.timing = window.performance.timing;
onAdFound(log); onAdFound(log);
}); });
addBackgroundListener('new-video-ad', onVideoMessage);
addBackgroundListener('new-invalid-video-ad', onVideoMessage);
} }
exports.utils.onDocLoaded(document, extractAdsWrapper); exports.utils.onDocLoaded(document, extractAdsWrapper);

@ -0,0 +1,584 @@
'use strict';
(function() {
var secBefore = 2000;
var secAfter = 5000;
var secBetweenDupAssets = 10e3;
var minVidSize = 500e3;
var maxVidSize = 25e6;
var maxContentRange = 25e6;
var videoExtensions = [
'af', '3gp', 'asf', 'avchd', 'avi', 'cam', 'dsh', 'flv', 'm1v', 'm2v',
'fla', 'flr', 'sol', 'm4v', 'mkv', 'wrap', 'mng', 'mov', 'mpeg', 'mpg',
'mpe', 'mp4', 'mxf', 'nsv', 'ogg', 'rm', 'svi', 'smi', 'wmv', 'webm'
];
var extensionsReg = new RegExp('\\.' + videoExtensions.join('$|\\.') + '$');
var videoContentTypesPrefixes = ['binary/octet-stream', 'video/', 'flv-application/', 'media'];
var bannedContentTypes = ['video/mp2t','video/f4m','video/f4f'];
var bannedFiletypes = ['ts'];
var bannedFiletypesReg = new RegExp('\\.' + bannedFiletypes.join('$|\\.') + '$');
var whitelistReqTypes = ['object', 'xmlhttprequest', 'other'];
var topVideoAssetDomains = [
'2mdn.net',
'innovid.com',
'serving-sys.com',
'btrll.com',
'teads.tv',
'tubemogul.com'
];
if ( !String.prototype.endsWith ) {
String.prototype.endsWith = function(searchString, position) {
var subjectString = this.toString();
if ( typeof position !== 'number' || !isFinite(position) ||
Math.floor(position) !== position || position > subjectString.length) {
position = subjectString.length;
}
position -= searchString.length;
var lastIndex = subjectString.indexOf(searchString, position);
return lastIndex !== -1 && lastIndex === position;
};
}
function isPixelRequest(request) {
return (request.type === 'image' || request.responseStatus === 204) &&
request.size <= 1000;
}
function isVpaidOrVastRequest(request) {
var lowerCaseUrl = request.url.toLowerCase();
return lowerCaseUrl.indexOf('vpaid') !== -1 || lowerCaseUrl.indexOf('vast') !== -1;
}
function hasValidRequestType(request) {
return whitelistReqTypes.indexOf(request.type) >= 0;
}
function stripQueryParams(url) {
return url.split('?',1)[0];
}
function parseHostnameFromUrl(url) {
var parser = document.createElement('a');
parser.href = url;
return parser.hostname;
}
function hasDomain(url, domain) {
return parseHostnameFromUrl(url).endsWith(domain);
}
function findHeader(headers, key) {
var header;
for ( var i = 0; i < headers.length; i += 1 ) {
header = headers[i];
if ( header.name.toLowerCase() === key ) {
return header;
}
}
return null;
}
function validVideoType(vtype) {
var goodType = videoContentTypesPrefixes.some(function(prefix) {
return vtype.indexOf(prefix) === 0;
});
return goodType;
}
function assetMsgKey(assetReq) {
var url = stripQueryParams(assetReq.url);
var key = assetReq.frameId + '-' + url;
return key;
}
var PageNetworkTrafficCollector = function(tabId) {
this.tabId = tabId;
this.displayAdFound = false;
this.requests = {};
this.msgsBeingSent = {};
this.assetsSeen = {};
this.allRedirects = {};
};
var globalPageContainer = {
collectors: {},
dyingCollectors: {},
cleanupCollector: function(tabId) {
if ( tabId in this.collectors ) {
delete globalPageContainer.collectors[tabId];
}
},
onNewNavigation: function(details) {
var tabId = details.tabId;
this.cleanupCollector(tabId);
this.collectors[tabId] = new PageNetworkTrafficCollector(tabId);
},
onNavigationCommitted: function(details) {
},
onNavigationCompleted: function(details) {
},
onTabClose: function(tabId, closeInfo) {
this.cleanupCollector(tabId);
delete this.collectors[tabId];
},
onDisplayAdFound: function(tabId) {
this.collectors[tabId].displayAdFound = true;
},
getRandId: function() {
return String(Math.floor(Math.random() * 1e9));
},
getCollector: function(tabId) {
if ( this.collectors.hasOwnProperty(tabId) ) {
return this.collectors[tabId];
}
return null;
},
forwardCall: function(details, collectorMemberFunction) {
var collector = this.getCollector(details.tabId);
if ( collector !== null ) {
collectorMemberFunction.apply(collector, [details]);
}
}
};
PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() {
var logMessage = Array.from(arguments).join(' ');
var message = {message: logMessage, event: 'console-log-message'};
chrome.tabs.sendMessage(this.tabId, message);
};
PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, isValidAd) {
var msg = {};
msg.assets = [];
msg.event_data = {};
if ( isValidAd ) {
msg.event = 'new-video-ad';
msg.requests = reqs;
msg.requests.sort(function(reqA, reqB) {return reqA.requestTimestamp - reqB.requestTimestamp;});
if ( assetReq ) {
msg.assets = [assetReq];
}
} else {
msg.requests = reqs.map(function(request) {
return parseHostnameFromUrl(request.url);
});
msg.assets = [{
url: parseHostnameFromUrl(assetReq.url),
contentType: assetReq.contentType,
size: assetReq.size
}];
msg.event = 'new-invalid-video-ad';
}
msg.origUrl = curPageUrl;
msg.displayAdFound = this.displayAdFound;
chrome.tabs.sendMessage(this.tabId, msg);
};
PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) {
return url + ':' + frameId;
};
PageNetworkTrafficCollector.prototype.seenBefore = function(request) {
var oldTime = this.assetsSeen[assetMsgKey(request)];
if ( oldTime && (request.requestTimestamp-oldTime < secBetweenDupAssets)){
return true;
}
return false;
};
PageNetworkTrafficCollector.prototype.recordSeenAsset = function(request) {
this.assetsSeen[assetMsgKey(request)] = request.requestTimestamp;
};
PageNetworkTrafficCollector.prototype.onBeforeRequest = function(details) {
var req = {
url: details.url,
type: details.type,
httpMethod: details.method,
frameId: details.frameId,
parentFrameId: details.parentFrameId,
requestTimestamp: details.timeStamp,
};
this.requests[details.requestId] = req;
};
PageNetworkTrafficCollector.prototype.onSendHeaders = function(details) {
var request, header;
request = this.requests[details.requestId];
header = request && findHeader(details.requestHeaders, 'x-requested-with');
if ( header && header.value.toLowerCase().indexOf('flash') > -1 ) {
request.from_flash = true;
}
};
PageNetworkTrafficCollector.prototype.onHeadersReceived = function(details) {
var getFrameDetails = {
tabId: details.tabId,
processId: null,
frameId: details.frameId
};
var pageNetworkTrafficController = this;
chrome.webNavigation.getFrame(getFrameDetails, function(frameDetails) {
if ( frameDetails && frameDetails.url ) {
pageNetworkTrafficController._onHeadersReceived(details, frameDetails);
}
});
};
PageNetworkTrafficCollector.prototype._onHeadersReceived = function(details, frameDetails) {
var contentSize, contentRange;
var request = this.requests[details.requestId];
if ( request ) {
var redirParent = this.allRedirects[this.getRedirKey(details.url, details.frameId)];
var header = request && findHeader(details.responseHeaders, 'content-type');
var contentType = header && header.value.toLowerCase();
if ( contentType){
request.contentType = contentType;
}
header = request && findHeader(details.responseHeaders, 'content-length');
contentSize = header && header.value;
if ( contentSize ) {
request.size = request.size || 0;
request.size += parseInt(contentSize);
}
header = request && findHeader(details.responseHeaders, 'content-range');
contentRange = header && header.value;
if ( contentRange ) {
request.contentRange = parseInt(contentRange.split('/')[1]);
}
var frameUrl = null;
if ( frameDetails && frameDetails.url ) {
frameUrl = frameDetails.url;
}
if ( !this.bannedRequest(request) &&
(this.isVideoReq(frameUrl, request) || (redirParent && redirParent.isVideo))) {
request.isVideo = true;
}
}
};
PageNetworkTrafficCollector.prototype.onBeforeRedirect = function(details) {
var request = this.requests[details.requestId];
if ( request ) {
if ( request.redirects ) {
request.redirects.push(details.redirectUrl);
} else {
request.redirects = [details.redirectUrl];
}
this.allRedirects[this.getRedirKey(details.redirectUrl, details.frameId)] = request;
}
};
PageNetworkTrafficCollector.prototype.isYoutubeMastheadRequest = function(url) {
var re = /video_masthead/;
return this.hasYoutubeDomain(url) && re.test(url);
};
PageNetworkTrafficCollector.prototype.isYoutubeVideoRequest = function(srcUrl, destUrl) {
if ( !this.hasYoutubeDomain(srcUrl) ) {
return false;
}
var re = /https?:\/\/r.*?\.googlevideo\.com\/videoplayback\?/;
return re.test(destUrl);
};
PageNetworkTrafficCollector.prototype.processResponse = function(requestDetails, frameDetails) {
var request;
if ( requestDetails ) {
request = this.requests[requestDetails.requestId];
if ( request ) {
request.responseStatus = requestDetails.statusCode;
request.responseTimestamp = requestDetails.timeStamp;
var frameUrl = null;
if ( frameDetails && frameDetails.url ) {
frameUrl = frameDetails.url;
}
var requestUrl = null;
if ( request.url ) {
requestUrl = request.url;
}
if ( this.isYoutubeAdReq(frameUrl, requestUrl) ) {
var videoId = this.parseYoutubeVideoIdFromUrl(requestUrl);
if ( videoId ) {
request.isYoutubeAd = true;
request.isVideo = true;
request.url = 'https://www.youtube.com/watch?v=' + this.parseYoutubeVideoIdFromUrl(requestUrl);
}
} else if ( !this.bannedRequest(request) &&
(this.isVideo || this.isVideoReq(frameUrl, request))) {
request.isVideo = true;
}
if ( request.isVideo ) {
var msgKey = assetMsgKey(request);
this.msgsBeingSent[msgKey] = request;
if ( !this.seenBefore(request) ) {
this.sendMsgWhenQuiet(msgKey);
}
this.recordSeenAsset(request);
}
}
}
};
PageNetworkTrafficCollector.prototype.onResponseStarted = function(responseDetails) {
if ( responseDetails.frameId < 0 ) {
responseDetails.frameId = 99999;
}
var getFrameDetails = {
tabId: responseDetails.tabId,
processId: null,
frameId: responseDetails.frameId
};
var pageNetworkTrafficController = this;
chrome.webNavigation.getFrame(getFrameDetails, function(frameDetails) {
if ( frameDetails && frameDetails.url ) {
pageNetworkTrafficController.processResponse(responseDetails, frameDetails);
}
});
};
PageNetworkTrafficCollector.prototype.hasBannedFiletype = function(request) {
var url = stripQueryParams(request.url);
if ( bannedFiletypesReg.exec(url) ) {
return true;
} else {
return false;
}
};
PageNetworkTrafficCollector.prototype.checkContentHeaders = function(request) {
if ( request.contentType && validVideoType(request.contentType) ) {
return true;
}
return false;
};
PageNetworkTrafficCollector.prototype.checkUrlExtension = function(request) {
var url = stripQueryParams(request.url);
if ( extensionsReg.exec(url) ) {
return true;
} else {
return false;
}
};
PageNetworkTrafficCollector.prototype.isVideoReq = function(srcUrl, request) {
if ( this.isYoutubeVideoRequest(srcUrl, request.url) ) {
return false;
}
return this.checkUrlExtension(request) || this.checkContentHeaders(request);
};
PageNetworkTrafficCollector.prototype.hasYoutubeDomain = function(url) {
var hostname = parseHostnameFromUrl(url) ;
if ( hostname === 'www.youtube.com' ) {
return true;
}
return false;
};
PageNetworkTrafficCollector.prototype.parseYoutubeVideoIdFromUrl = function(url) {
var re = /^https?:\/\/www\.youtube\.com\/get_video_info.*(?:\?|&)video_id=(.*?)(?:$|&)/;
var match = re.exec(url);
if ( match && match.length > 1 ) {
return match[1];
}
re = /^https?:\/\/www\.youtube\.com\/embed\/(.*?)(?:$|\?)/;
match = re.exec(url);
if ( match && match.length > 1 ) {
return match[1];
}
re = /^https?:\/\/www\.youtube\.com\/watch\?v=(.*$)/;
match = re.exec(url);
if ( match && match.length > 1 ) {
return match[1];
}
return null;
};
PageNetworkTrafficCollector.prototype.isYoutubeGetVideoInfoReq = function(url) {
var re = /^https?:\/\/www\.youtube\.com\/get_video_info\?/;
return re.test(url);
};
PageNetworkTrafficCollector.prototype.isYoutubeAdReq = function(srcUrl, destUrl) {
if ( !this.hasYoutubeDomain(srcUrl) ||
!this.isYoutubeGetVideoInfoReq(destUrl)) {
return false;
}
if ( this.parseYoutubeVideoIdFromUrl(srcUrl) ===
this.parseYoutubeVideoIdFromUrl(destUrl) &&
!this.isYoutubeMastheadRequest(destUrl)) {
return false;
}
return true;
};
PageNetworkTrafficCollector.prototype.bannedRequest = function(request) {
return this.bannedVideoType(request) || this.hasBannedFiletype(request) || this.bannedVideoSize(request);
};
PageNetworkTrafficCollector.prototype.bannedVideoType = function(request) {
var badType = false;
if ( request.contentType ) {
badType = bannedContentTypes.some(function(prefix) {
return request.contentType.indexOf(prefix) >= 0;
});
}
return badType;
};
PageNetworkTrafficCollector.prototype.bannedVideoSize = function(request) {
if ( request.size !== null ) {
if ( request.size < minVidSize || request.size > maxVidSize || request.contentRange > maxContentRange ) {
return true;
}
}
return false;
};
PageNetworkTrafficCollector.prototype.grabTagReqs = function(tabRequests, assetRequest) {
var minTimestamp, maxTimestamp;
minTimestamp = assetRequest.requestTimestamp - secBefore;
maxTimestamp = assetRequest.requestTimestamp + secAfter;
var filteredRequests = tabRequests.filter(function(request) {
return (request.requestTimestamp > minTimestamp &&
request.requestTimestamp < maxTimestamp &&
request.frameId === assetRequest.frameId &&
request.url !== assetRequest.url &&
(hasValidRequestType(request) ||
isPixelRequest(request)));
});
return filteredRequests;
};
PageNetworkTrafficCollector.prototype.isValidVideoAd = function(assetRequest, tagRequests) {
var hasVpaidOrVastRequest = tagRequests.some(function(tagRequest) {
return isVpaidOrVastRequest(tagRequest);
});
if ( hasVpaidOrVastRequest ) {
return true;
}
var hasTopVideoAssetDomain = topVideoAssetDomains.some(function(assetDomain) {
return hasDomain(assetRequest.url, assetDomain);
});
return hasTopVideoAssetDomain;
};
PageNetworkTrafficCollector.prototype.sendMsgWhenQuiet = function(msgKey) {
var _this = this,
origPageUrl, msgAssetReq;
msgAssetReq = this.msgsBeingSent[msgKey];
chrome.tabs.get(this.tabId, function(tab) {origPageUrl = tab.url;});
setTimeout(function() {
var rawRequests = [];
if ( globalPageContainer.collectors[_this.tabId] === _this ) {
for ( var reqId in _this.requests ) {
rawRequests.push(_this.requests[reqId]);
}
var tagReqs = _this.grabTagReqs(rawRequests, msgAssetReq);
if ( _this.isValidVideoAd(msgAssetReq, tagReqs) ) {
_this.sendToTab(msgAssetReq, tagReqs, origPageUrl, true);
} else {
_this.sendToTab(msgAssetReq, tagReqs, origPageUrl, false);
}
} else {
}
delete _this.msgsBeingSent[msgKey];
}, secAfter+secBefore);
};
PageNetworkTrafficCollector.prototype.existingMessage = function(candidateRequest) {
var frameMsg = this.msgsBeingSent[candidateRequest.frameId];
if ( frameMsg ) {
return frameMsg;
} else {
return null;
}
};
chrome.webRequest.onBeforeRequest.addListener(function(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRequest);
}, {urls: ['http://*/*', 'https://*/*']}, []);
chrome.webRequest.onSendHeaders.addListener(function(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onSendHeaders);
}, {urls: ['http://*/*', 'https://*/*']}, ['requestHeaders']);
chrome.webRequest.onHeadersReceived.addListener(function(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onHeadersReceived);
}, {urls: ['http://*/*', 'https://*/*']}, ['responseHeaders']);
chrome.webRequest.onBeforeRedirect.addListener(function(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRedirect);
}, {urls: ['http://*/*', 'https://*/*']}, []);
chrome.webRequest.onResponseStarted.addListener(function(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onResponseStarted);
}, {urls: ['http://*/*', 'https://*/*']}, ['responseHeaders']);
chrome.webNavigation.onBeforeNavigate.addListener(function(details) {
if ( details.frameId === 0 ) {
globalPageContainer.onNewNavigation(details);
}
}, {});
chrome.webNavigation.onCommitted.addListener(function(details) {
if ( details.frameId === 0 ) {
globalPageContainer.onNavigationCommitted(details);
}
});
chrome.webNavigation.onCompleted.addListener(function(details) {
if ( details.frameId === 0 ) {
globalPageContainer.onNavigationCompleted(details);
}
});
chrome.tabs.onRemoved.addListener(function(tabId, closeInfo) {
globalPageContainer.onTabClose(tabId, closeInfo);
});
chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) {
if ( message.event === 'new-ad' && message.data.event === 'ad' ) {
var tabId = sender.tab.id;
if ( tabId ) {
globalPageContainer.onDisplayAdFound(tabId);
}
}
});
})();