Ignore tags file

main
Elbert Alias 9 years ago
parent b0bc61bca7
commit 2bcb73b4ba

2
.gitignore vendored

@ -6,4 +6,6 @@ src/icons/converted/*
node_modules
npm-debug.log
tags
!.gitkeep

@ -34,7 +34,7 @@ ln -f $path/wappalyzer.js $path/drivers/firefox/lib
ln -f $path/apps.json $path/drivers/firefox/data
ln -f $path/icons/*.png $path/drivers/firefox/data/images/icons
ln -f $path/icons/*.svg $path/drivers/firefox/data/images/icons
ln -f $path/utils/*.js $path/drivers/firefox/data/js
ln -f $path/utils/iframe.js $path/drivers/firefox/data/js
ln -f $path/wappalyzer.js $path/drivers/chrome/js
ln -f $path/apps.json $path/drivers/chrome

@ -4,3 +4,4 @@ images/icons/*.png
images/icons/*.svg
js/wappalyzer.js
js/iframe.js
js/network.js

@ -8,6 +8,7 @@
<script src="js/wappalyzer.js"></script>
<script src="js/defaults.js"></script>
<script src="js/driver.js"></script>
<script src="js/network.js"></script>
</head>
<body>
</body>

@ -29,6 +29,6 @@
"js/inject.js"
],
"options_page": "options.html",
"permissions": [ "tabs", "webRequest", "http://*/*", "https://*/*" ],
"permissions": [ "tabs", "webRequest", "webNavigation", "http://*/*", "https://*/*" ],
"content_security_policy": "script-src 'self' https://ssl.google-analytics.com; object-src 'self'"
}

@ -7,18 +7,19 @@ var exports = {};
(function(exports) {
var utils = {
getReferrer: function() {
normalizeUrl: function(url) {
return this.hashUrl(document.referrer) || null;
return this.hashUrl(url) || null;
},
getPageUrl: function() {
return this.hashUrl(window.location.href) || null;
getReferrer: function() {
return this.normalizeUrl(document.referrer);
},
getPageUrl: function() {
return this.normalizeUrl(window.location.href);
},
hashUrl: function(url) {
var a,
result;
@ -121,6 +122,7 @@ var exports = {};
utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent);
utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME;
function LogGenerator() {
this.msgNum = 0;
this.pageMeta = {
@ -134,11 +136,21 @@ var exports = {};
LogGenerator.prototype = {
log: function(event, opt_assets, opt_pageTags) {
var opt_video_assets;
if ( event === 'video' ) {
opt_video_assets = opt_assets || [];
opt_assets = [];
} else {
opt_video_assets = [];
opt_assets = opt_assets || [];
}
var result = {
doc: this.pageMeta,
event: event,
assets: opt_assets || [],
video_assets: opt_video_assets,
assets: opt_assets,
version: '3',
mrev: '5dacb94-c',
msgNum: this.msgNum,
timestamp: new Date().getTime(),
pageVis: document.visibilityState,
@ -157,17 +169,19 @@ var exports = {};
(function(exports) {
var VALID_AD_SIZES = [
var SizeMatcher = {
VALID_AD_SIZES: [
[300, 50],
[320, 50],
[160, 600],
[300, 250],
[300, 600],
[300, 1050],
[336, 280],
[336, 850],
[468, 60],
[728, 90],
[728, 250],
[728, 270],
[970, 66],
[970, 90],
@ -176,14 +190,29 @@ var exports = {};
[970, 400],
[970, 415],
[1280, 100]
];
],
var PX_SIZE_TOL = 10;
var MIN_WINDOW_PX = 10;
var MAX_SEARCHES_PER_WINDOW = 10;
var MAX_SEARCHES_PER_ELEMENT = 2;
PX_SIZE_TOL: 10,
function makeSizeSet(validAdSizes, sizeTol) {
getMatchedAdSize: function(width, height) {
if ( !this.set ) {
this.set = this._makeSizeSet();
}
return this.set[Math.round(width) + 'x' + Math.round(height)];
},
elementIsAdShaped: function(el) {
return !!this.getMatchedAdSizeForElement(el);
},
getMatchedAdSizeForElement: function(el) {
var rect = el.getBoundingClientRect();
return this.getMatchedAdSize(rect.width, rect.height);
},
_makeSizeSet: function() {
var set = {};
var i;
var xfuz;
@ -192,10 +221,10 @@ var exports = {};
var width;
var height;
for ( i = 0; i < validAdSizes.length; i++ ) {
for ( xfuz = -sizeTol; xfuz <= sizeTol; xfuz++ ) {
for ( yfuz = -sizeTol; yfuz <= sizeTol; yfuz++ ) {
size = validAdSizes[i];
for ( i = 0; i < this.VALID_AD_SIZES.length; i++ ) {
for ( xfuz = -this.PX_SIZE_TOL; xfuz <= this.PX_SIZE_TOL; xfuz++ ) {
for ( yfuz = -this.PX_SIZE_TOL; yfuz <= this.PX_SIZE_TOL; yfuz++ ) {
size = this.VALID_AD_SIZES[i];
width = size[0] + xfuz;
height = size[1] + yfuz;
set[width + 'x' + height] = size;
@ -204,208 +233,278 @@ var exports = {};
}
return set;
}
};
var SIZE_SET = makeSizeSet(VALID_AD_SIZES, PX_SIZE_TOL);
var Throttler = {
MAX_SEARCHES_PER_WINDOW: 10,
MAX_SEARCHES_PER_ELEMENT: 2,
function elementIsAd(el) {
countSearch: function(el) {
if ( typeof el.searches !== 'number' ) {
el.searches = 0;
}
if ( el.searches >= MAX_SEARCHES_PER_ELEMENT ) {
return false;
}
el.searches += 1;
},
var isImgWithoutSrc = el.tagName === 'IMG' && !el.src;
var isImgWithoutAnchor = el.tagName === 'IMG' && !(el.parentNode.tagName === 'A' || el.getAttribute('onclick'));
return elementIsAdShaped(el) && !isImgWithoutSrc && !isImgWithoutAnchor;
throttle: function(el, max) {
if ( typeof el.searches === 'number' && el.searches >= max ) {
return true;
}
return false;
},
function isNewAd(el, win) {
return !el.mp_adFound && (win === win.top || !win.mp_adFound);
}
throttleElement: function(el) {
return this.throttle(el, this.MAX_SEARCHES_PER_ELEMENT);
},
function getFriendlyIframes(win) {
var iframes = win.document.querySelectorAll('iframe');
iframes = exports.utils.realArray(iframes);
var friendlyIframes = iframes.filter(function(ifr) {
return exports.utils.isFriendlyWindow(ifr.contentWindow);
});
return friendlyIframes;
}
throttleWin: function(win) {
return this.throttle(win, this.MAX_SEARCHES_PER_WINDOW);
},
function getMatchedAdSize(width, height) {
return SIZE_SET[width + 'x' + height];
getCount: function(el) {
return el.searches || 0;
}
};
function elementIsAdShaped(el) {
return !!getMatchedAdSizeForElement(el);
function TopSearcher(win) {
this.win = win;
this.doc = win.document;
}
function getMatchedAdSizeForElement(el) {
var rect = el.getBoundingClientRect();
return getMatchedAdSize(rect.width, rect.height);
}
TopSearcher.prototype.search = function() {
var candidates = exports.utils.realArray(this.doc.querySelectorAll('img, object, embed')),
html5Ad,
ads = [];
function containsLargeIframes(win) {
var iframes = win.document.querySelectorAll('iframe');
var rect;
var i;
for ( i = 0; i < iframes.length; i++ ) {
rect = iframes[i].getBoundingClientRect();
if ( rect.width > 10 || rect.height > 10 ) {
ads = ads.concat(candidates.filter(function(el) {
if ( !el.mpAdFound && !Throttler.throttleElement(el) ) {
Throttler.countSearch(el);
if ( (el.tagName !== 'IMG' || isStandardImage(el)) && SizeMatcher.elementIsAdShaped(el) ) {
el.mpAdFound = true;
return true;
}
}
return false;
}));
html5Ad = this._mainGetHTMLAd();
if ( html5Ad ) {
html5Ad.html5 = true;
html5Ad.mpAdFound = true;
ads.push(html5Ad);
}
function isValidHTML5Div(div, winSize) {
var elSize = getMatchedAdSizeForElement(div);
return ads;
};
if ( typeof div.checks !== 'number' ) {
div.checks = 1;
} else {
div.checks += 1;
TopSearcher.prototype._mainGetHTMLAd = function() {
var styles = this.doc.querySelectorAll('div > style, div > link[rel="stylesheet"]'),
i, div;
for ( i = 0; i < styles.length; i++ ) {
div = styles[i].parentNode;
if ( !div.mpAdFound && SizeMatcher.elementIsAdShaped(div) && this._jumpedOut(div) ) {
return div;
}
return (elSize &&
elSize[0] === winSize[0] && elSize[1] === winSize[1] &&
div.checks > 1);
}
};
var HTML5_SIGNAL_ELEMENTS = 'canvas, button, video, svg, img';
function iframeGetHTMLAd(win) {
var body = win.document.body,
elements, i, el, divs, div, numElements,
winSize, elSize;
if ( !body ) {
return null;
}
winSize = getMatchedAdSize(win.innerWidth, win.innerHeight);
TopSearcher.prototype._jumpedOut = function(el) {
var siblings, ifrs;
siblings = exports.utils.realArray(el.parentNode.children);
ifrs = siblings.filter(function(el) {
return el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0;
});
return ifrs.length > 0;
};
if ( !winSize ) {
return null;
}
function IframeSearcher(win) {
this.MIN_AD_AREA = 14000;
this.MIN_WINDOW_PX = 10;
elements = body.querySelectorAll(HTML5_SIGNAL_ELEMENTS);
this.win = win;
this.doc = win.document;
this.body = win.document.body;
this.winClickTag = win.clickTag;
this.adSizeMeta = this._getAdSizeMeta();
this.numElementsInBody = (this.body && this.body.querySelectorAll('*').length) || 0;
for ( i = 0; i < elements.length; i++ ) {
el = elements[i];
elSize = getMatchedAdSizeForElement(el);
if ( elSize && elSize[0] === winSize[0] && elSize[1] === winSize[1] ) {
return el;
this.shouldSearchWindow = false;
if ( !this.win.mpAdFound && this.body && !Throttler.throttleWin(this.win) ) {
this.winWidth = this.win.innerWidth;
this.winHeight = this.win.innerHeight;
if ( this._meetsMinAdSize(this.winWidth, this.winHeight) && !this._containsLargeIframes() ) {
this.shouldSearchWindow = true;
}
}
numElements = body.querySelectorAll('*').length;
if ( numElements < 5 ) {
return null;
}
divs = body.querySelectorAll('div');
IframeSearcher.prototype.search = function() {
var ad;
for ( i = 0; i < divs.length; i++ ) {
div = divs[i];
if ( isValidHTML5Div(div, winSize) ) {
return div;
if ( this.shouldSearchWindow ) {
ad = this._search();
if ( ad ) {
ad.mpAdFound = true;
win.mpAdFound = true;
return ad;
}
Throttler.countSearch(this.win);
}
return null;
}
};
function jumpedOut(el) {
var siblings, ifrs;
siblings = exports.utils.realArray(el.parentNode.children);
ifrs = siblings.filter(function(el) {
return el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0;
IframeSearcher.prototype._search = function() {
var _this = this,
stdCandidates,
html5Candidates,
stdEl,
html5El;
stdCandidates = this.body.querySelectorAll('img, object, embed');
stdEl = getFirst(stdCandidates, function(el) {
if ( !el.mpAdFound &&
!Throttler.throttleElement(el) &&
(el.tagName !== 'IMG' || isStandardImage(el)) &&
_this._elementIsAtLeastAsBigAsWindow(el))
{
return true;
}
Throttler.countSearch(el);
return false;
});
return ifrs.length > 0;
if ( stdEl ) {
return stdEl;
}
function mainGetHTMLAd(win) {
var styles = win.document.querySelectorAll('div > style, div > link[rel="stylesheet"]'),
i, div;
for ( i = 0; i < styles.length; i++ ) {
div = styles[i].parentNode;
if ( elementIsAdShaped(div) && jumpedOut(div) ) {
return div;
if ( this._isHTML5Iframe() ) {
html5Candidates = this.doc.querySelectorAll('body, canvas, button, video, svg, div');
html5El = getFirst(html5Candidates, function(el) {
if ( _this._elementIsAtLeastAsBigAsWindow(el) ) {
return true;
}
Throttler.countSearch(el);
return false;
});
}
if ( html5El ) {
html5El.html5 = true;
html5El.winClickTag = this.winClickTag;
html5El.adSizeMeta = this.adSizeMeta;
return html5El;
}
function findAds(win, opt_ads) {
return null;
};
if ( typeof win.searches !== 'number' ) {
win.searches = 0;
IframeSearcher.prototype._isHTML5Iframe = function() {
if ( this.winClickTag || this.adSizeMeta ) {
return true;
}
var ads = opt_ads || [];
var adsFound = 0;
if ( this.doc.querySelectorAll('canvas', 'button', 'video', 'svg').length > 0 ) {
return true;
}
if ( win.innerWidth <= MIN_WINDOW_PX || win.innerHeight <= MIN_WINDOW_PX ) {
win.searches++;
return ads;
if ( this.numElementsInBody >= 5 && Throttler.getCount(this.win) > 0 && this.doc.querySelectorAll('div').length > 0 ) {
return true;
}
if ( exports.utils.SCRIPT_IN_WINDOW_TOP || win.searches < MAX_SEARCHES_PER_WINDOW ) {
var adCandidates = win.document.querySelectorAll('img, object, embed');
adCandidates = exports.utils.realArray(adCandidates);
return false;
};
adCandidates.forEach(function(el) {
if ( elementIsAd(el) && isNewAd(el, win) ) {
el.mp_adFound = true;
el.inIframe = win !== win.top;
win.mp_adFound = true;
ads.push(el);
adsFound += 1;
}
});
IframeSearcher.prototype._elementIsAtLeastAsBigAsWindow = function(el) {
var rect = el.getBoundingClientRect(),
tol = 0.95;
var htmlAd, adSizeMeta;
if ( win === win.top ) {
htmlAd = mainGetHTMLAd(win);
} else {
if ( adsFound === 0 && !containsLargeIframes(win) ) {
htmlAd = iframeGetHTMLAd(win);
return rect.width >= (tol * this.winWidth) && rect.height >= (tol * this.winHeight);
};
IframeSearcher.prototype._meetsMinAdSize = function(width, height) {
return (width * height) >= this.MIN_AD_AREA;
};
IframeSearcher.prototype._containsLargeIframes = function() {
var iframes = this.doc.querySelectorAll('iframe');
var rect;
var i;
for ( i = 0; i < iframes.length; i++ ) {
rect = iframes[i].getBoundingClientRect();
if ( rect.width > this.MIN_WINDOW_PX || rect.height > this.MIN_WINDOW_PX ) {
return true;
}
}
return false;
};
if ( htmlAd && isNewAd(htmlAd, win) ) {
htmlAd.html5 = true;
htmlAd.inIframe = win !== win.top;
if ( htmlAd.inIframe ) {
adSizeMeta = win.document.querySelectorAll('meta[name="ad.size"]');
IframeSearcher.prototype._getAdSizeMeta = function() {
var adSizeMeta = this.doc.querySelectorAll('meta[name="ad.size"]');
if ( adSizeMeta.length > 0 ) {
htmlAd.adSizeMeta = adSizeMeta[0].content;
return adSizeMeta[0].content;
} else {
return null;
}
if ( win.clickTag ) {
htmlAd.winClickTag = win.clickTag;
};
function getFirst(arr, testFn) {
var i, el;
for ( i = 0; i < arr.length; i++ ) {
el = arr[i];
if ( testFn(el) ) {
return el;
}
}
htmlAd.mp_adFound = true;
win.mp_adFound = true;
ads.push(htmlAd);
return null;
}
win.searches += 1;
function isStandardImage(img) {
return img.src && (img.parentNode.tagName === 'A' || img.getAttribute('onclick'));
}
var iframes = getFriendlyIframes(win);
iframes.forEach(function(ifr) {
findAds(ifr.contentWindow, ads);
function getFriendlyIframes(win) {
var iframes = win.document.querySelectorAll('iframe');
iframes = exports.utils.realArray(iframes);
var friendlyIframes = iframes.filter(function(ifr) {
return exports.utils.isFriendlyWindow(ifr.contentWindow);
});
return friendlyIframes;
}
function findAds(win) {
var i,
iframes,
searcher,
ad,
ads = [];
if ( win === win.top ) {
searcher = new TopSearcher(win);
ads = ads.concat(searcher.search());
} else {
searcher = new IframeSearcher(win);
ad = searcher.search();
if ( ad ) {
ads.push(ad);
}
}
iframes = getFriendlyIframes(win);
for ( i = 0; i < iframes.length; i++ ) {
ads = ads.concat(findAds(iframes[i].contentWindow));
}
return ads;
}
exports.adfinder = {
getMatchedAdSize: getMatchedAdSize,
getMatchedAdSize: SizeMatcher.getMatchedAdSize.bind(SizeMatcher),
findAds: findAds
};
})(exports);
@ -463,11 +562,12 @@ var exports = {};
getPosition: function(el) {
var rect = el.getBoundingClientRect();
var win = exports.utils.elementWindow(el);
return {
width: rect.width,
height: rect.height,
left: rect.left + win.pageXOffset,
top: rect.top + win.pageYOffset
width: Math.round(rect.width),
height: Math.round(rect.height),
left: Math.round(rect.left + win.pageXOffset),
top: Math.round(rect.top + win.pageYOffset)
};
},
@ -682,7 +782,7 @@ var exports = {};
while ( true ) {
parentContainer = curContainer.parentElement;
if ( this.isValidContainer(parentContainer) ) {
if ( parentContainer && this.isValidContainer(parentContainer) ) {
curContainer = parentContainer;
} else {
return curContainer;
@ -693,12 +793,6 @@ var exports = {};
var tagfinder = {
prepToSend: function(adData) {
adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height);
delete adData.width;
delete adData.height;
},
setPositions: function(adData, opt_el, opt_winPos) {
var el = opt_el || adData.context;
var winPos = opt_winPos || {left: 0, top: 0};
@ -732,10 +826,12 @@ var exports = {};
while ( true ) {
highestContainer = mgr.highestContainer(curWin, referenceElement);
mgr.captureHTML(highestContainer);
if ( curWin === curWin.top ) {
break;
} else {
curWin.mpAdFound = true;
mgr.adData.serializedIframeContents = mgr.adData.context;
if ( exports.utils.isFriendlyWindow(curWin.parent) ) {
@ -763,6 +859,8 @@ var exports = {};
var _pageTags;
var INIT_MS_BW_SEARCHES = 2000;
var PAGE_TAG_RE = new RegExp('gpt|oascentral');
var POST_MSG_ID = '1484952787-11883-24071-12354-1549';
var AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)');
function getPageTags(doc) {
var scripts = doc.getElementsByTagName('script');
@ -778,7 +876,7 @@ var exports = {};
function messageAllParentFrames(adData) {
adData.dummyId = true;
adData.postMessageId = POST_MSG_ID;
adData = JSON.stringify(adData);
@ -795,18 +893,33 @@ var exports = {};
messageAllParentFrames(adData);
} else if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
exports.tagfinder.setPositions(adData);
exports.tagfinder.prepToSend(adData);
adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height);
if ( !adData.matchedSize ) {
if ( AD_SERVER_RE.exec(results.referenceElement.id) ) {
adData.matchedSize = [adData.width, adData.height];
adData.oddSize = true;
} else {
return;
}
}
delete adData.width;
delete adData.height;
adData.curPageUrl = exports.utils.getPageUrl();
_pageTags = _pageTags || getPageTags(document);
var log = _logGen.log('ad', [adData], _pageTags);
if ( _onAdFound ) {
_onAdFound(log, results.referenceElement);
}
}
}
}
function extractAdsWrapper() {
if ( exports.utils.SCRIPT_IN_WINDOW_TOP || document.readyState === 'complete' ) {
@ -819,26 +932,20 @@ var exports = {};
function extractAds() {
var ads = exports.adfinder.findAds(window);
if ( !ads ) {
return;
}
ads.forEach(function(ad) {
var startTime = new Date().getTime();
var adId = startTime + '-' + Math.floor(Math.random() * 10e12);
var adData = {
width: ad.offsetWidth,
height: ad.offsetHeight,
width: Math.round(ad.offsetWidth),
height: Math.round(ad.offsetHeight),
startTime: startTime,
adId: adId,
html5: ad.html5 || false,
inIframe: ad.inIframe
html5: ad.html5 || false
};
if ( ad.html5 && ad.inIframe ) {
if ( ad.html5 ) {
adData.adSizeMeta = ad.adSizeMeta || null;
adData.winClickTag = ad.winClickTag || null;
}
@ -898,9 +1005,9 @@ var exports = {};
return;
}
if ( adData.dummyId ) {
if ( adData.postMessageId === POST_MSG_ID ) {
delete adData.dummyId;
delete adData.postMessageId;
if ( isChildWin(myWin, ifrWin) ) {
if ( exports.utils.isFriendlyWindow(ifrWin) ) {
@ -917,6 +1024,40 @@ var exports = {};
}
}
function onVideoMessage(msg, sender, callback) {
var log;
if ( msg.event === 'new-video-ad' ) {
msg.assets.forEach(function(asset) {
});
log = _logGen.log('video', msg.assets);
} else {
log = _logGen.log('invalid-video', msg.assets);
}
msg.assets.forEach(function(a) {delete a.isVideo;});
log.displayAdFound = msg.displayAdFound;
log.requests = msg.requests;
log.data = msg.event_data;
log.doc.finalPageUrl = log.doc.url;
log.doc.url = exports.utils.normalizeUrl(msg.origUrl);
_onAdFound(log);
}
function addBackgroundListener(event, callback) {
if ( typeof chrome !== 'undefined' ) {
chrome.runtime.onMessage.addListener(function(msg) {
if ( msg.event === event ) {
callback(msg);
}
});
} else if ( window.self.port ) {
window.self.port.on(event, callback);
}
}
exports.coordinator = {
init: function(onAdFound) {
@ -925,19 +1066,21 @@ var exports = {};
}
_onAdFound = onAdFound;
window.addEventListener('message', onPostMessage, false);
if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
var log = _logGen.log('page');
onAdFound(log);
}
window.addEventListener('message', onPostMessage, false);
if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
window.addEventListener('beforeunload', function(event) {
var log = _logGen.log('unload');
log.timing = window.performance.timing;
onAdFound(log);
});
addBackgroundListener('new-video-ad', onVideoMessage);
addBackgroundListener('new-invalid-video-ad', onVideoMessage);
}
exports.utils.onDocLoaded(document, extractAdsWrapper);

@ -0,0 +1,584 @@
'use strict';
(function() {
var secBefore = 2000;
var secAfter = 5000;
var secBetweenDupAssets = 10e3;
var minVidSize = 500e3;
var maxVidSize = 25e6;
var maxContentRange = 25e6;
var videoExtensions = [
'af', '3gp', 'asf', 'avchd', 'avi', 'cam', 'dsh', 'flv', 'm1v', 'm2v',
'fla', 'flr', 'sol', 'm4v', 'mkv', 'wrap', 'mng', 'mov', 'mpeg', 'mpg',
'mpe', 'mp4', 'mxf', 'nsv', 'ogg', 'rm', 'svi', 'smi', 'wmv', 'webm'
];
var extensionsReg = new RegExp('\\.' + videoExtensions.join('$|\\.') + '$');
var videoContentTypesPrefixes = ['binary/octet-stream', 'video/', 'flv-application/', 'media'];
var bannedContentTypes = ['video/mp2t','video/f4m','video/f4f'];
var bannedFiletypes = ['ts'];
var bannedFiletypesReg = new RegExp('\\.' + bannedFiletypes.join('$|\\.') + '$');
var whitelistReqTypes = ['object', 'xmlhttprequest', 'other'];
var topVideoAssetDomains = [
'2mdn.net',
'innovid.com',
'serving-sys.com',
'btrll.com',
'teads.tv',
'tubemogul.com'
];
if ( !String.prototype.endsWith ) {
String.prototype.endsWith = function(searchString, position) {
var subjectString = this.toString();
if ( typeof position !== 'number' || !isFinite(position) ||
Math.floor(position) !== position || position > subjectString.length) {
position = subjectString.length;
}
position -= searchString.length;
var lastIndex = subjectString.indexOf(searchString, position);
return lastIndex !== -1 && lastIndex === position;
};
}
function isPixelRequest(request) {
return (request.type === 'image' || request.responseStatus === 204) &&
request.size <= 1000;
}
function isVpaidOrVastRequest(request) {
var lowerCaseUrl = request.url.toLowerCase();
return lowerCaseUrl.indexOf('vpaid') !== -1 || lowerCaseUrl.indexOf('vast') !== -1;
}
function hasValidRequestType(request) {
return whitelistReqTypes.indexOf(request.type) >= 0;
}
function stripQueryParams(url) {
return url.split('?',1)[0];
}
function parseHostnameFromUrl(url) {
var parser = document.createElement('a');
parser.href = url;
return parser.hostname;
}
function hasDomain(url, domain) {
return parseHostnameFromUrl(url).endsWith(domain);
}
function findHeader(headers, key) {
var header;
for ( var i = 0; i < headers.length; i += 1 ) {
header = headers[i];
if ( header.name.toLowerCase() === key ) {
return header;
}
}
return null;
}
function validVideoType(vtype) {
var goodType = videoContentTypesPrefixes.some(function(prefix) {
return vtype.indexOf(prefix) === 0;
});
return goodType;
}
function assetMsgKey(assetReq) {
var url = stripQueryParams(assetReq.url);
var key = assetReq.frameId + '-' + url;
return key;
}
var PageNetworkTrafficCollector = function(tabId) {
this.tabId = tabId;
this.displayAdFound = false;
this.requests = {};
this.msgsBeingSent = {};
this.assetsSeen = {};
this.allRedirects = {};
};
var globalPageContainer = {
collectors: {},
dyingCollectors: {},
cleanupCollector: function(tabId) {
if ( tabId in this.collectors ) {
delete globalPageContainer.collectors[tabId];
}
},
onNewNavigation: function(details) {
var tabId = details.tabId;
this.cleanupCollector(tabId);
this.collectors[tabId] = new PageNetworkTrafficCollector(tabId);
},
onNavigationCommitted: function(details) {
},
onNavigationCompleted: function(details) {
},
onTabClose: function(tabId, closeInfo) {
this.cleanupCollector(tabId);
delete this.collectors[tabId];
},
onDisplayAdFound: function(tabId) {
this.collectors[tabId].displayAdFound = true;
},
getRandId: function() {
return String(Math.floor(Math.random() * 1e9));
},
getCollector: function(tabId) {
if ( this.collectors.hasOwnProperty(tabId) ) {
return this.collectors[tabId];
}
return null;
},
forwardCall: function(details, collectorMemberFunction) {
var collector = this.getCollector(details.tabId);
if ( collector !== null ) {
collectorMemberFunction.apply(collector, [details]);
}
}
};
PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() {
var logMessage = Array.from(arguments).join(' ');
var message = {message: logMessage, event: 'console-log-message'};
chrome.tabs.sendMessage(this.tabId, message);
};
PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, isValidAd) {
var msg = {};
msg.assets = [];
msg.event_data = {};
if ( isValidAd ) {
msg.event = 'new-video-ad';
msg.requests = reqs;
msg.requests.sort(function(reqA, reqB) {return reqA.requestTimestamp - reqB.requestTimestamp;});
if ( assetReq ) {
msg.assets = [assetReq];
}
} else {
msg.requests = reqs.map(function(request) {
return parseHostnameFromUrl(request.url);
});
msg.assets = [{
url: parseHostnameFromUrl(assetReq.url),
contentType: assetReq.contentType,
size: assetReq.size
}];
msg.event = 'new-invalid-video-ad';
}
msg.origUrl = curPageUrl;
msg.displayAdFound = this.displayAdFound;
chrome.tabs.sendMessage(this.tabId, msg);
};
PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) {
return url + ':' + frameId;
};
PageNetworkTrafficCollector.prototype.seenBefore = function(request) {
var oldTime = this.assetsSeen[assetMsgKey(request)];
if ( oldTime && (request.requestTimestamp-oldTime < secBetweenDupAssets)){
return true;
}
return false;
};
PageNetworkTrafficCollector.prototype.recordSeenAsset = function(request) {
this.assetsSeen[assetMsgKey(request)] = request.requestTimestamp;
};
PageNetworkTrafficCollector.prototype.onBeforeRequest = function(details) {
var req = {
url: details.url,
type: details.type,
httpMethod: details.method,
frameId: details.frameId,
parentFrameId: details.parentFrameId,
requestTimestamp: details.timeStamp,
};
this.requests[details.requestId] = req;
};
PageNetworkTrafficCollector.prototype.onSendHeaders = function(details) {
var request, header;
request = this.requests[details.requestId];
header = request && findHeader(details.requestHeaders, 'x-requested-with');
if ( header && header.value.toLowerCase().indexOf('flash') > -1 ) {
request.from_flash = true;
}
};
PageNetworkTrafficCollector.prototype.onHeadersReceived = function(details) {
var getFrameDetails = {
tabId: details.tabId,
processId: null,
frameId: details.frameId
};
var pageNetworkTrafficController = this;
chrome.webNavigation.getFrame(getFrameDetails, function(frameDetails) {
if ( frameDetails && frameDetails.url ) {
pageNetworkTrafficController._onHeadersReceived(details, frameDetails);
}
});
};
PageNetworkTrafficCollector.prototype._onHeadersReceived = function(details, frameDetails) {
var contentSize, contentRange;
var request = this.requests[details.requestId];
if ( request ) {
var redirParent = this.allRedirects[this.getRedirKey(details.url, details.frameId)];
var header = request && findHeader(details.responseHeaders, 'content-type');
var contentType = header && header.value.toLowerCase();
if ( contentType){
request.contentType = contentType;
}
header = request && findHeader(details.responseHeaders, 'content-length');
contentSize = header && header.value;
if ( contentSize ) {
request.size = request.size || 0;
request.size += parseInt(contentSize);
}
header = request && findHeader(details.responseHeaders, 'content-range');
contentRange = header && header.value;
if ( contentRange ) {
request.contentRange = parseInt(contentRange.split('/')[1]);
}
var frameUrl = null;
if ( frameDetails && frameDetails.url ) {
frameUrl = frameDetails.url;
}
if ( !this.bannedRequest(request) &&
(this.isVideoReq(frameUrl, request) || (redirParent && redirParent.isVideo))) {
request.isVideo = true;
}
}
};
PageNetworkTrafficCollector.prototype.onBeforeRedirect = function(details) {
var request = this.requests[details.requestId];
if ( request ) {
if ( request.redirects ) {
request.redirects.push(details.redirectUrl);
} else {
request.redirects = [details.redirectUrl];
}
this.allRedirects[this.getRedirKey(details.redirectUrl, details.frameId)] = request;
}
};
PageNetworkTrafficCollector.prototype.isYoutubeMastheadRequest = function(url) {
var re = /video_masthead/;
return this.hasYoutubeDomain(url) && re.test(url);
};
PageNetworkTrafficCollector.prototype.isYoutubeVideoRequest = function(srcUrl, destUrl) {
if ( !this.hasYoutubeDomain(srcUrl) ) {
return false;
}
var re = /https?:\/\/r.*?\.googlevideo\.com\/videoplayback\?/;
return re.test(destUrl);
};
PageNetworkTrafficCollector.prototype.processResponse = function(requestDetails, frameDetails) {
var request;
if ( requestDetails ) {
request = this.requests[requestDetails.requestId];
if ( request ) {
request.responseStatus = requestDetails.statusCode;
request.responseTimestamp = requestDetails.timeStamp;
var frameUrl = null;
if ( frameDetails && frameDetails.url ) {
frameUrl = frameDetails.url;
}
var requestUrl = null;
if ( request.url ) {
requestUrl = request.url;
}
if ( this.isYoutubeAdReq(frameUrl, requestUrl) ) {
var videoId = this.parseYoutubeVideoIdFromUrl(requestUrl);
if ( videoId ) {
request.isYoutubeAd = true;
request.isVideo = true;
request.url = 'https://www.youtube.com/watch?v=' + this.parseYoutubeVideoIdFromUrl(requestUrl);
}
} else if ( !this.bannedRequest(request) &&
(this.isVideo || this.isVideoReq(frameUrl, request))) {
request.isVideo = true;
}
if ( request.isVideo ) {
var msgKey = assetMsgKey(request);
this.msgsBeingSent[msgKey] = request;
if ( !this.seenBefore(request) ) {
this.sendMsgWhenQuiet(msgKey);
}
this.recordSeenAsset(request);
}
}
}
};
PageNetworkTrafficCollector.prototype.onResponseStarted = function(responseDetails) {
if ( responseDetails.frameId < 0 ) {
responseDetails.frameId = 99999;
}
var getFrameDetails = {
tabId: responseDetails.tabId,
processId: null,
frameId: responseDetails.frameId
};
var pageNetworkTrafficController = this;
chrome.webNavigation.getFrame(getFrameDetails, function(frameDetails) {
if ( frameDetails && frameDetails.url ) {
pageNetworkTrafficController.processResponse(responseDetails, frameDetails);
}
});
};
PageNetworkTrafficCollector.prototype.hasBannedFiletype = function(request) {
var url = stripQueryParams(request.url);
if ( bannedFiletypesReg.exec(url) ) {
return true;
} else {
return false;
}
};
PageNetworkTrafficCollector.prototype.checkContentHeaders = function(request) {
if ( request.contentType && validVideoType(request.contentType) ) {
return true;
}
return false;
};
PageNetworkTrafficCollector.prototype.checkUrlExtension = function(request) {
var url = stripQueryParams(request.url);
if ( extensionsReg.exec(url) ) {
return true;
} else {
return false;
}
};
PageNetworkTrafficCollector.prototype.isVideoReq = function(srcUrl, request) {
if ( this.isYoutubeVideoRequest(srcUrl, request.url) ) {
return false;
}
return this.checkUrlExtension(request) || this.checkContentHeaders(request);
};
PageNetworkTrafficCollector.prototype.hasYoutubeDomain = function(url) {
var hostname = parseHostnameFromUrl(url) ;
if ( hostname === 'www.youtube.com' ) {
return true;
}
return false;
};
PageNetworkTrafficCollector.prototype.parseYoutubeVideoIdFromUrl = function(url) {
var re = /^https?:\/\/www\.youtube\.com\/get_video_info.*(?:\?|&)video_id=(.*?)(?:$|&)/;
var match = re.exec(url);
if ( match && match.length > 1 ) {
return match[1];
}
re = /^https?:\/\/www\.youtube\.com\/embed\/(.*?)(?:$|\?)/;
match = re.exec(url);
if ( match && match.length > 1 ) {
return match[1];
}
re = /^https?:\/\/www\.youtube\.com\/watch\?v=(.*$)/;
match = re.exec(url);
if ( match && match.length > 1 ) {
return match[1];
}
return null;
};
PageNetworkTrafficCollector.prototype.isYoutubeGetVideoInfoReq = function(url) {
var re = /^https?:\/\/www\.youtube\.com\/get_video_info\?/;
return re.test(url);
};
PageNetworkTrafficCollector.prototype.isYoutubeAdReq = function(srcUrl, destUrl) {
if ( !this.hasYoutubeDomain(srcUrl) ||
!this.isYoutubeGetVideoInfoReq(destUrl)) {
return false;
}
if ( this.parseYoutubeVideoIdFromUrl(srcUrl) ===
this.parseYoutubeVideoIdFromUrl(destUrl) &&
!this.isYoutubeMastheadRequest(destUrl)) {
return false;
}
return true;
};
PageNetworkTrafficCollector.prototype.bannedRequest = function(request) {
return this.bannedVideoType(request) || this.hasBannedFiletype(request) || this.bannedVideoSize(request);
};
PageNetworkTrafficCollector.prototype.bannedVideoType = function(request) {
var badType = false;
if ( request.contentType ) {
badType = bannedContentTypes.some(function(prefix) {
return request.contentType.indexOf(prefix) >= 0;
});
}
return badType;
};
PageNetworkTrafficCollector.prototype.bannedVideoSize = function(request) {
if ( request.size !== null ) {
if ( request.size < minVidSize || request.size > maxVidSize || request.contentRange > maxContentRange ) {
return true;
}
}
return false;
};
PageNetworkTrafficCollector.prototype.grabTagReqs = function(tabRequests, assetRequest) {
var minTimestamp, maxTimestamp;
minTimestamp = assetRequest.requestTimestamp - secBefore;
maxTimestamp = assetRequest.requestTimestamp + secAfter;
var filteredRequests = tabRequests.filter(function(request) {
return (request.requestTimestamp > minTimestamp &&
request.requestTimestamp < maxTimestamp &&
request.frameId === assetRequest.frameId &&
request.url !== assetRequest.url &&
(hasValidRequestType(request) ||
isPixelRequest(request)));
});
return filteredRequests;
};
PageNetworkTrafficCollector.prototype.isValidVideoAd = function(assetRequest, tagRequests) {
var hasVpaidOrVastRequest = tagRequests.some(function(tagRequest) {
return isVpaidOrVastRequest(tagRequest);
});
if ( hasVpaidOrVastRequest ) {
return true;
}
var hasTopVideoAssetDomain = topVideoAssetDomains.some(function(assetDomain) {
return hasDomain(assetRequest.url, assetDomain);
});
return hasTopVideoAssetDomain;
};
PageNetworkTrafficCollector.prototype.sendMsgWhenQuiet = function(msgKey) {
var _this = this,
origPageUrl, msgAssetReq;
msgAssetReq = this.msgsBeingSent[msgKey];
chrome.tabs.get(this.tabId, function(tab) {origPageUrl = tab.url;});
setTimeout(function() {
var rawRequests = [];
if ( globalPageContainer.collectors[_this.tabId] === _this ) {
for ( var reqId in _this.requests ) {
rawRequests.push(_this.requests[reqId]);
}
var tagReqs = _this.grabTagReqs(rawRequests, msgAssetReq);
if ( _this.isValidVideoAd(msgAssetReq, tagReqs) ) {
_this.sendToTab(msgAssetReq, tagReqs, origPageUrl, true);
} else {
_this.sendToTab(msgAssetReq, tagReqs, origPageUrl, false);
}
} else {
}
delete _this.msgsBeingSent[msgKey];
}, secAfter+secBefore);
};
PageNetworkTrafficCollector.prototype.existingMessage = function(candidateRequest) {
var frameMsg = this.msgsBeingSent[candidateRequest.frameId];
if ( frameMsg ) {
return frameMsg;
} else {
return null;
}
};
chrome.webRequest.onBeforeRequest.addListener(function(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRequest);
}, {urls: ['http://*/*', 'https://*/*']}, []);
chrome.webRequest.onSendHeaders.addListener(function(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onSendHeaders);
}, {urls: ['http://*/*', 'https://*/*']}, ['requestHeaders']);
chrome.webRequest.onHeadersReceived.addListener(function(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onHeadersReceived);
}, {urls: ['http://*/*', 'https://*/*']}, ['responseHeaders']);
chrome.webRequest.onBeforeRedirect.addListener(function(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRedirect);
}, {urls: ['http://*/*', 'https://*/*']}, []);
chrome.webRequest.onResponseStarted.addListener(function(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onResponseStarted);
}, {urls: ['http://*/*', 'https://*/*']}, ['responseHeaders']);
chrome.webNavigation.onBeforeNavigate.addListener(function(details) {
if ( details.frameId === 0 ) {
globalPageContainer.onNewNavigation(details);
}
}, {});
chrome.webNavigation.onCommitted.addListener(function(details) {
if ( details.frameId === 0 ) {
globalPageContainer.onNavigationCommitted(details);
}
});
chrome.webNavigation.onCompleted.addListener(function(details) {
if ( details.frameId === 0 ) {
globalPageContainer.onNavigationCompleted(details);
}
});
chrome.tabs.onRemoved.addListener(function(tabId, closeInfo) {
globalPageContainer.onTabClose(tabId, closeInfo);
});
chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) {
if ( message.event === 'new-ad' && message.data.event === 'ad' ) {
var tabId = sender.tab.id;
if ( tabId ) {
globalPageContainer.onDisplayAdFound(tabId);
}
}
});
})();