Fix jsonToDOM in Chrome

main
Elbert Alias 8 years ago
parent d882aca41d
commit 8df949c8cd

2
.gitignore vendored

@ -10,3 +10,5 @@ Thumbs.db
Desktop.ini Desktop.ini
*.DS_Store *.DS_Store
._* ._*
tags
tags.*

@ -2,7 +2,8 @@
"dependencies": { "dependencies": {
"file-type": "3.8.*", "file-type": "3.8.*",
"is-svg": "2.0.*", "is-svg": "2.0.*",
"read-chunk": "2.0.*" "read-chunk": "2.0.*",
"svg2png-many": "*"
} }
} }

@ -2,5 +2,3 @@
/images/icons/converted/* /images/icons/converted/*
/images/icons/* /images/icons/*
/js/wappalyzer.js /js/wappalyzer.js
/js/iframe.js
/js/network.js

@ -0,0 +1,975 @@
'use strict';
(function(win) {
var exports = {};
(function(exports) {
var utils = {
getReferrer: function() {
return this.hashUrl(document.referrer) || null;
},
getPageUrl: function() {
return this.hashUrl(window.location.href) || null;
},
hashUrl: function(url) {
var a,
result;
if ( !url || url.indexOf('http') !== 0 ) {
return null;
}
a = document.createElement('a');
a.href = url;
result = a.protocol + '//' + a.hostname + '/';
if ( a.pathname && a.pathname !== '/' ) {
result += this.hashCode(a.pathname);
}
if ( a.search ) {
result += '?' + this.hashCode(a.search);
}
if ( a.hash ) {
result += '#' + this.hashCode(a.hash);
}
return result;
},
hashCode: function(str) {
var hash = 0,
kar,
i;
if ( str.length === 0 ) {
return hash;
}
for ( i = 0; i < str.length; i++ ) {
kar = str.charCodeAt(i);
hash = ((hash << 5) - hash) + kar;
hash = hash & hash;
}
return hash + Math.pow(2, 32);
},
realArray: function(a) {
return Array.prototype.slice.apply(a);
},
onDocLoaded: function(doc, callback) {
if ( doc.readyState === 'loading' ) {
doc.addEventListener('DOMContentLoaded', callback);
} else {
callback();
}
},
SCRIPT_IN_WINDOW_TOP: window === window.top,
isFriendlyWindow: function(win) {
var href;
try {
href = win.location.href;
} catch(e) {
return false;
}
return true;
},
elementWindow: function(el) {
return el.ownerDocument.defaultView;
},
viewport: function(win) {
return {width: win.innerWidth, height: win.innerHeight};
},
parseQS: function(qs) {
if ( qs.indexOf('http') === 0 ) {
qs = qs.split('?')[1];
}
var i, kvs, key, val;
var dict = {};
qs = qs.split('&');
for ( i = 0; i < qs.length; i++ ) {
kvs = qs[i].split('=');
key = kvs[0];
val = kvs.slice(1).join('=');
try {
dict[key] = window.decodeURIComponent(val);
} catch (e) {
continue;
}
}
return dict;
},
};
utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent);
utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME;
function LogGenerator() {
this.msgNum = 0;
this.pageMeta = {
'url': utils.getPageUrl(),
'isHP': window.location.pathname === '/',
'referrer': utils.getReferrer(),
'rand': Math.floor(Math.random() * 10e12),
'startTime': new Date().getTime()
};
}
LogGenerator.prototype = {
log: function(event, opt_assets, opt_pageTags) {
var result = {
doc: this.pageMeta,
event: event,
assets: opt_assets || [],
version: '3',
msgNum: this.msgNum,
timestamp: new Date().getTime(),
pageVis: document.visibilityState,
pageFoc: document.hasFocus(),
pageTags: opt_pageTags || []
};
this.msgNum++;
return result;
}
};
utils.LogGenerator = LogGenerator;
exports.utils = utils;
})(exports);
(function(exports) {
var VALID_AD_SIZES = [
[160, 600],
[300, 250],
[300, 600],
[300, 1050],
[336, 280],
[336, 850],
[468, 60],
[728, 90],
[728, 270],
[970, 66],
[970, 90],
[970, 125],
[970, 250],
[970, 400],
[970, 415],
[1280, 100]
];
var PX_SIZE_TOL = 10;
var MIN_WINDOW_PX = 10;
var MAX_SEARCHES_PER_WINDOW = 10;
var MAX_SEARCHES_PER_ELEMENT = 2;
function makeSizeSet(validAdSizes, sizeTol) {
var set = {};
var i;
var xfuz;
var yfuz;
var size;
var width;
var height;
for ( i = 0; i < validAdSizes.length; i++ ) {
for ( xfuz = -sizeTol; xfuz <= sizeTol; xfuz++ ) {
for ( yfuz = -sizeTol; yfuz <= sizeTol; yfuz++ ) {
size = validAdSizes[i];
width = size[0] + xfuz;
height = size[1] + yfuz;
set[width + 'x' + height] = size;
}
}
}
return set;
}
var SIZE_SET = makeSizeSet(VALID_AD_SIZES, PX_SIZE_TOL);
function elementIsAd(el) {
if ( typeof el.searches !== 'number' ) {
el.searches = 0;
}
if ( el.searches >= MAX_SEARCHES_PER_ELEMENT ) {
return false;
}
el.searches += 1;
var isImgWithoutSrc = el.tagName === 'IMG' && !el.src;
var isImgWithoutAnchor = el.tagName === 'IMG' && !(el.parentNode.tagName === 'A' || el.getAttribute('onclick'));
return elementIsAdShaped(el) && !isImgWithoutSrc && !isImgWithoutAnchor;
}
function isNewAd(el, win) {
return !el.mp_adFound && (win === win.top || !win.mp_adFound);
}
function getFriendlyIframes(win) {
var iframes = win.document.querySelectorAll('iframe');
iframes = exports.utils.realArray(iframes);
var friendlyIframes = iframes.filter(function(ifr) {
return exports.utils.isFriendlyWindow(ifr.contentWindow);
});
return friendlyIframes;
}
function getMatchedAdSize(width, height) {
return SIZE_SET[width + 'x' + height];
}
function elementIsAdShaped(el) {
return !!getMatchedAdSizeForElement(el);
}
function getMatchedAdSizeForElement(el) {
var rect = el.getBoundingClientRect();
return getMatchedAdSize(rect.width, rect.height);
}
function containsLargeIframes(win) {
var iframes = win.document.querySelectorAll('iframe');
var rect;
var i;
for ( i = 0; i < iframes.length; i++ ) {
rect = iframes[i].getBoundingClientRect();
if ( rect.width > 10 || rect.height > 10 ) {
return true;
}
}
return false;
}
function isValidHTML5Div(div, winSize) {
var elSize = getMatchedAdSizeForElement(div);
if ( typeof div.checks !== 'number' ) {
div.checks = 1;
} else {
div.checks += 1;
}
return (elSize &&
elSize[0] === winSize[0] && elSize[1] === winSize[1] &&
div.checks > 1);
}
var HTML5_SIGNAL_ELEMENTS = 'canvas, button, video, svg, img';
function iframeGetHTMLAd(win) {
var body = win.document.body,
elements, i, el, divs, div, numElements,
winSize, elSize;
if ( !body ) {
return null;
}
winSize = getMatchedAdSize(win.innerWidth, win.innerHeight);
if ( !winSize ) {
return null;
}
elements = body.querySelectorAll(HTML5_SIGNAL_ELEMENTS);
for ( i = 0; i < elements.length; i++ ) {
el = elements[i];
elSize = getMatchedAdSizeForElement(el);
if ( elSize && elSize[0] === winSize[0] && elSize[1] === winSize[1] ) {
return el;
}
}
numElements = body.querySelectorAll('*').length;
if ( numElements < 5 ) {
return null;
}
divs = body.querySelectorAll('div');
for ( i = 0; i < divs.length; i++ ) {
div = divs[i];
if ( isValidHTML5Div(div, winSize) ) {
return div;
}
}
return null;
}
function jumpedOut(el) {
var siblings, ifrs;
siblings = exports.utils.realArray(el.parentNode.children);
ifrs = siblings.filter(function(el) {
return el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0;
});
return ifrs.length > 0;
}
function mainGetHTMLAd(win) {
var styles = win.document.querySelectorAll('div > style, div > link[rel="stylesheet"]'),
i, div;
for ( i = 0; i < styles.length; i++ ) {
div = styles[i].parentNode;
if ( elementIsAdShaped(div) && jumpedOut(div) ) {
return div;
}
}
}
function findAds(win, opt_ads) {
if ( typeof win.searches !== 'number' ) {
win.searches = 0;
}
var ads = opt_ads || [];
var adsFound = 0;
if ( win.innerWidth <= MIN_WINDOW_PX || win.innerHeight <= MIN_WINDOW_PX ) {
win.searches++;
return ads;
}
if ( exports.utils.SCRIPT_IN_WINDOW_TOP || win.searches < MAX_SEARCHES_PER_WINDOW ) {
var adCandidates = win.document.querySelectorAll('img, object, embed');
adCandidates = exports.utils.realArray(adCandidates);
adCandidates.forEach(function(el) {
if ( elementIsAd(el) && isNewAd(el, win) ) {
el.mp_adFound = true;
el.inIframe = win !== win.top;
win.mp_adFound = true;
ads.push(el);
adsFound += 1;
}
});
var htmlAd, adSizeMeta;
if ( win === win.top ) {
htmlAd = mainGetHTMLAd(win);
} else {
if ( adsFound === 0 && !containsLargeIframes(win) ) {
htmlAd = iframeGetHTMLAd(win);
}
}
if ( htmlAd && isNewAd(htmlAd, win) ) {
htmlAd.html5 = true;
htmlAd.inIframe = win !== win.top;
if ( htmlAd.inIframe ) {
adSizeMeta = win.document.querySelectorAll('meta[name="ad.size"]');
if ( adSizeMeta.length > 0 ) {
htmlAd.adSizeMeta = adSizeMeta[0].content;
}
if ( win.clickTag ) {
htmlAd.winClickTag = win.clickTag;
}
}
htmlAd.mp_adFound = true;
win.mp_adFound = true;
ads.push(htmlAd);
}
win.searches += 1;
}
var iframes = getFriendlyIframes(win);
iframes.forEach(function(ifr) {
findAds(ifr.contentWindow, ads);
});
return ads;
}
exports.adfinder = {
getMatchedAdSize: getMatchedAdSize,
findAds: findAds
};
})(exports);
(function(exports) {
var parser = {
TAGS_WITH_SRC_ATTR: {
'IMG': true,
'SCRIPT': true,
'IFRAME': true,
'EMBED': true
},
MAX_ATTR_LEN: 100,
getUrl: function(el, params) {
var url;
if ( this.TAGS_WITH_SRC_ATTR.hasOwnProperty(el.tagName) ) {
url = el.src;
} else if ( el.tagName === 'OBJECT' ) {
url = el.data || (params && params.movie) || null;
} else if ( el.tagName === 'A' ) {
url = el.href;
}
if ( url && url.indexOf('http') === 0 ) {
return url;
} else {
return null;
}
},
getParams: function(el) {
if ( el.tagName !== 'OBJECT' ) {
return null;
}
var i, child;
var params = {};
var children = el.children;
for ( i = 0; i < children.length; i++ ) {
child = children[i];
if ( child.tagName === 'PARAM' && child.name ) {
params[child.name.toLowerCase()] = child.value;
}
}
return params;
},
getPosition: function(el) {
var rect = el.getBoundingClientRect();
var win = exports.utils.elementWindow(el);
return {
width: rect.width,
height: rect.height,
left: rect.left + win.pageXOffset,
top: rect.top + win.pageYOffset
};
},
getFlashvars: function(el, params, url) {
var flashvars;
var urlQS = url && url.split('?')[1];
if ( el.tagName === 'EMBED' ) {
flashvars = el.getAttribute('flashvars') || urlQS;
} else if ( el.tagName === 'OBJECT' ) {
flashvars = params.flashvars || el.getAttribute('flashvars') || urlQS;
}
return (flashvars && exports.utils.parseQS(flashvars)) || null;
},
findClickThru: function(el, flashvars) {
var key;
if ( el.tagName === 'IMG' && el.parentElement.tagName === 'A' ) {
return el.parentElement.href;
} else if ( flashvars ) {
for ( key in flashvars ) {
if ( flashvars.hasOwnProperty(key) ) {
if ( key.toLowerCase().indexOf('clicktag') === 0 ) {
return flashvars[key];
}
}
}
}
return null;
},
getAttr: function(el, name) {
var val = el.getAttribute(name);
if ( val && val.slice && val.toString ) {
return val.slice(0, this.MAX_ATTR_LEN).toString();
} else {
return null;
}
},
putPropIfExists: function(obj, name, val) {
if ( val ) {
obj[name] = val;
}
},
putAttrIfExists: function(obj, el, name) {
var val = this.getAttr(el, name);
this.putPropIfExists(obj, name, val);
},
elementToJSON: function(el, opt_findClickThru) {
var pos = this.getPosition(el);
var params = this.getParams(el);
var url = this.getUrl(el, params);
var flashvars = this.getFlashvars(el, params, url);
var clickThru = opt_findClickThru && this.findClickThru(el, flashvars);
var json = {
tagName: el.tagName,
width: pos.width,
height: pos.height,
left: pos.left,
top: pos.top,
children: []
};
if ( params ) {
delete params.flashvars;
}
this.putAttrIfExists(json, el, 'id');
this.putAttrIfExists(json, el, 'class');
this.putAttrIfExists(json, el, 'name');
this.putPropIfExists(json, 'flashvars', flashvars);
this.putPropIfExists(json, 'url', url);
this.putPropIfExists(json, 'params', params);
this.putPropIfExists(json, 'clickThru', clickThru);
return json;
}
};
exports.parser = { elementToJSON: parser.elementToJSON.bind(parser) };
})(exports);
(function(exports) {
var ContextManager = function(adData) {
this.adData = adData;
};
ContextManager.prototype = {
CONTAINER_SIZE_TOL: 0.4,
ASPECT_RATIO_FOR_LEADERBOARDS: 2,
isValidContainer: function(el, opt_curWin) {
var cWidth = el.clientWidth;
var cHeight = el.clientHeight;
var adWidth = this.adData.width;
var adHeight = this.adData.height;
var winWidth = opt_curWin && opt_curWin.innerWidth;
var winHeight = opt_curWin && opt_curWin.innerHeight;
var similarWin = opt_curWin && this.withinTol(adWidth, winWidth) && this.withinTol(adHeight, winHeight);
var similarSizeX = this.withinTol(adWidth, cWidth);
var similarSizeY = this.withinTol(adHeight, cHeight);
var adAspect = adWidth / adHeight;
return similarWin || el.tagName === 'A' || (adAspect >= this.ASPECT_RATIO_FOR_LEADERBOARDS && similarSizeY) || (similarSizeX && similarSizeY);
},
withinTol: function(adlen, conlen) {
var pct = (conlen - adlen) / adlen;
return pct <= this.CONTAINER_SIZE_TOL;
},
serializeElements: function(el) {
if ( !el ) {
return;
}
var i;
var ifrWin;
var adId = this.adData.adId;
var elIsAd = false;
if ( adId && el[adId] && el[adId].isAd === true ) {
elIsAd = true;
}
var json = exports.parser.elementToJSON(el, elIsAd);
var childJSON;
if ( elIsAd ) {
json.adId = adId;
this.adData.element = {};
var keys = Object.keys(json);
for ( i = 0; i < keys.length; i++ ) {
var key = keys[i];
if ( key !== 'children' && key !== 'contents' ) {
this.adData.element[key] = json[key];
}
}
}
var children = exports.utils.realArray(el.children).filter(function(el) {
var param = el.tagName === 'PARAM';
var inlineScript = el.tagName === 'SCRIPT' && !(el.src && el.src.indexOf('http') >= 0);
var noScript = el.tagName === 'NOSCRIPT';
return !(param || inlineScript || noScript);
});
for ( i = 0; i < children.length; i++ ) {
childJSON = this.serializeElements(children[i]);
if ( childJSON ) {
json.children.push(childJSON);
}
}
if ( el.tagName === 'IFRAME' ) {
ifrWin = el.contentWindow;
if ( adId && el[adId] && el[adId].needsWindow ) {
json.contents = this.adData.serializedIframeContents;
el[adId].needsWindow = false;
delete this.adData.serializedIframeContents;
} else if ( exports.utils.isFriendlyWindow(ifrWin) ) {
childJSON = this.serializeElements(ifrWin.document.documentElement);
if ( childJSON ) {
json.contents = childJSON;
}
}
}
if ( json.children.length > 0 || json.adId || json.tagName === 'IFRAME' || json.url ) {
return json;
} else {
return null;
}
},
captureHTML: function(containerEl) {
this.adData.context = this.serializeElements(containerEl);
},
nodeCount: function(el) {
return el.getElementsByTagName('*').length + 1;
},
highestContainer: function(curWin, referenceElement) {
var curContainer = referenceElement;
var docEl = curWin.document.documentElement;
var parentContainer;
if ( curWin !== curWin.top && this.isValidContainer(docEl, curWin) ) {
return docEl;
}
while ( true ) {
parentContainer = curContainer.parentElement;
if ( this.isValidContainer(parentContainer) ) {
curContainer = parentContainer;
} else {
return curContainer;
}
}
}
};
var tagfinder = {
prepToSend: function(adData) {
adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height);
delete adData.width;
delete adData.height;
},
setPositions: function(adData, opt_el, opt_winPos) {
var el = opt_el || adData.context;
var winPos = opt_winPos || {left: 0, top: 0};
var ifrPos;
el.left += winPos.left;
el.top += winPos.top;
if ( el.children ) {
el.children.forEach(function(child) {
this.setPositions(adData, child, winPos);
}, this);
}
if ( el.contents ) {
ifrPos = {left: el.left, top: el.top};
this.setPositions(adData, el.contents, ifrPos);
}
if ( el.adId === adData.adId ) {
adData.element.left = el.left;
adData.element.top = el.top;
}
},
appendTags: function(adData, referenceElement) {
var mgr = new ContextManager(adData);
var curWin = exports.utils.elementWindow(referenceElement);
var highestContainer;
while ( true ) {
highestContainer = mgr.highestContainer(curWin, referenceElement);
mgr.captureHTML(highestContainer);
if ( curWin === curWin.top ) {
break;
} else {
mgr.adData.serializedIframeContents = mgr.adData.context;
if ( exports.utils.isFriendlyWindow(curWin.parent) ) {
referenceElement = curWin.frameElement;
referenceElement[mgr.adData.adId] = {needsWindow: true};
curWin = curWin.parent;
} else {
break;
}
}
}
return {
referenceElement:referenceElement,
highestContainer: highestContainer
};
}
};
exports.tagfinder = tagfinder;
})(exports);
(function(exports) {
var _onAdFound;
var _logGen = new exports.utils.LogGenerator();
var _pageTags;
var INIT_MS_BW_SEARCHES = 2000;
var PAGE_TAG_RE = new RegExp('gpt|oascentral');
function getPageTags(doc) {
var scripts = doc.getElementsByTagName('script');
var pageTags = [];
scripts = exports.utils.realArray(scripts);
scripts.forEach(function(script) {
if ( PAGE_TAG_RE.exec(script.src) ) {
pageTags.push({'tagName': 'SCRIPT', 'url': script.src});
}
});
return pageTags;
}
function messageAllParentFrames(adData) {
adData.dummyId = true;
adData = JSON.stringify(adData);
var win = window;
while ( win !== win.top ) {
win = win.parent;
win.postMessage(adData, '*');
}
}
function appendTagsAndSendToParent(adData, referenceElement) {
var results = exports.tagfinder.appendTags(adData, referenceElement);
if ( exports.utils.SCRIPT_IN_HOSTILE_IFRAME ) {
messageAllParentFrames(adData);
} else if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
exports.tagfinder.setPositions(adData);
exports.tagfinder.prepToSend(adData);
adData.curPageUrl = exports.utils.getPageUrl();
_pageTags = _pageTags || getPageTags(document);
var log = _logGen.log('ad', [adData], _pageTags);
if ( _onAdFound ) {
_onAdFound(log, results.referenceElement);
}
}
}
function extractAdsWrapper() {
if ( exports.utils.SCRIPT_IN_WINDOW_TOP || document.readyState === 'complete' ) {
extractAds();
}
setTimeout(function() {
extractAdsWrapper();
}, INIT_MS_BW_SEARCHES);
}
function extractAds() {
var ads = exports.adfinder.findAds(window);
if ( !ads ) {
return;
}
ads.forEach(function(ad) {
var startTime = new Date().getTime();
var adId = startTime + '-' + Math.floor(Math.random() * 10e12);
var adData = {
width: ad.offsetWidth,
height: ad.offsetHeight,
startTime: startTime,
adId: adId,
html5: ad.html5 || false,
inIframe: ad.inIframe
};
if ( ad.html5 && ad.inIframe ) {
adData.adSizeMeta = ad.adSizeMeta || null;
adData.winClickTag = ad.winClickTag || null;
}
ad[adId] = { isAd: true };
appendTagsAndSendToParent(adData, ad);
});
}
function isChildWin(myWin, otherWin) {
var parentWin = otherWin.parent;
while ( parentWin !== otherWin ) {
if ( parentWin === myWin ) {
return true;
}
otherWin = parentWin;
parentWin = parentWin.parent;
}
return false;
}
function iframeFromWindow(win, winToMatch) {
var i, ifr, ifrWin,
iframes = win.document.querySelectorAll('iframe');
for ( i = 0; i < iframes.length; i++ ) {
ifr = iframes[i];
if ( ifr.contentWindow === winToMatch ) {
return ifr;
}
}
for ( i = 0; i < iframes.length; i++ ) {
ifrWin = iframes[i].contentWindow;
if ( exports.utils.isFriendlyWindow(ifrWin) ) {
ifr = iframeFromWindow(ifrWin, winToMatch);
if ( ifr ) {
return ifr;
}
}
}
}
function onPostMessage(event) {
var adData,
ifrWin = event.source,
myWin = window.document.defaultView,
ifrTag;
try {
adData = JSON.parse(event.data);
} catch(e) {
return;
}
if ( adData.dummyId ) {
delete adData.dummyId;
if ( isChildWin(myWin, ifrWin) ) {
if ( exports.utils.isFriendlyWindow(ifrWin) ) {
ifrTag = ifrWin.frameElement;
} else {
ifrTag = iframeFromWindow(myWin, ifrWin);
}
if ( ifrTag ) {
ifrTag[adData.adId] = {needsWindow: true};
appendTagsAndSendToParent(adData, ifrTag);
}
}
}
}
exports.coordinator = {
init: function(onAdFound) {
if ( exports.utils.SCRIPT_IN_FRIENDLY_IFRAME ) {
return false;
}
_onAdFound = onAdFound;
if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
var log = _logGen.log('page');
onAdFound(log);
}
window.addEventListener('message', onPostMessage, false);
if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
window.addEventListener('beforeunload', function(event) {
var log = _logGen.log('unload');
log.timing = window.performance.timing;
onAdFound(log);
});
}
exports.utils.onDocLoaded(document, extractAdsWrapper);
}
};
})(exports);
if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
window.adparser = {
init: exports.coordinator.init,
};
} else {
exports.coordinator.init(function() {});
}
})(window);
(function(adparser) {
function sendToBackground(event, message) {
if ( window.self.port ) {
window.self.port.emit(event, message);
} else if ( typeof chrome !== 'undefined' ) {
chrome.extension.sendRequest(message);
}
}
function onAdFound(log) {
sendToBackground('ad_log', { id: 'ad_log', subject: log });
}
if ( window === window.top ) {
adparser.init(onAdFound);
}
})(window.adparser);

@ -16,7 +16,7 @@ function jsonToDOM(jsonTemplate, doc, nodes) {
// Array of elements? Parse each one... // Array of elements? Parse each one...
if (Array.isArray(elemNameOrArray)) { if (Array.isArray(elemNameOrArray)) {
var frag = doc.createDocumentFragment(); var frag = doc.createDocumentFragment();
Array.forEach(arguments, function(thisElem) { Array.prototype.forEach.call(arguments, function(thisElem) {
frag.appendChild(tag.apply(null, thisElem)); frag.appendChild(tag.apply(null, thisElem));
}); });
return frag; return frag;
@ -46,7 +46,7 @@ function jsonToDOM(jsonTemplate, doc, nodes) {
} }
// Create and append this element's children // Create and append this element's children
var childElems = Array.slice(arguments, 2); var childElems = Array.prototype.slice.call(arguments, 2);
childElems.forEach(function(childElem) { childElems.forEach(function(childElem) {
if (childElem != null) { if (childElem != null) {
elem.appendChild( elem.appendChild(

@ -0,0 +1,785 @@
'use strict';
(function() {
function isChrome() {
return (typeof chrome !== 'undefined' &&
window.navigator.userAgent.match(/Chrom(?:e|ium)\/([0-9\.]+)/));
}
var browserProxy;
if ( isChrome() ) {
browserProxy = chrome;
} else {
browserProxy = browser;
}
var MIN_FF_MAJOR_VERSION = 51;
var requiredBrowserApis = [
browserProxy.webNavigation,
browserProxy.tabs,
browserProxy.webRequest,
browserProxy.runtime
];
var areListenersRegistered = false;
var secBefore = 2000;
var secAfter = 5000;
var secBetweenDupAssets = 10e3;
var minVidSize = 500e3;
var maxVidSize = 25e6;
var maxContentRange = 25e6;
var videoExtensions = [
'af', '3gp', 'asf', 'avchd', 'avi', 'cam', 'dsh', 'flv', 'm1v', 'm2v',
'fla', 'flr', 'sol', 'm4v', 'mkv', 'wrap', 'mng', 'mov', 'mpeg', 'mpg',
'mpe', 'mp4', 'mxf', 'nsv', 'ogg', 'rm', 'svi', 'smi', 'wmv', 'webm'
];
var extensionsReg = new RegExp('\\.' + videoExtensions.join('$|\\.') + '$');
var videoContentTypesPrefixes = ['binary/octet-stream', 'video/', 'flv-application/', 'media'];
var bannedContentTypes = ['video/mp2t','video/f4m','video/f4f'];
var bannedFiletypes = ['ts'];
var bannedFiletypesReg = new RegExp('\\.' + bannedFiletypes.join('$|\\.') + '$');
var whitelistReqTypes = ['object', 'xmlhttprequest', 'other'];
var topVideoAssetDomains = [
'2mdn.net',
'adap.tv',
'adnxs.com',
'adsrvr.org',
'btrll.com',
'celtra.com',
'flashtalking.com',
'flite.com',
'innovid.com',
'jivox.com',
'mixpo.com',
'nytimes.com',
'playwire.com',
'selectmedia.asia',
'serving-sys.com',
'solvemedia.com',
'spotible.com',
'teads.tv',
'tribalfusion.com',
'tubemogul.com',
'videologygroup.com',
'washingtonpost.com'
];
if ( !String.prototype.endsWith ) {
String.prototype.endsWith = function(searchString, position) {
var subjectString = this.toString();
if ( typeof position !== 'number' || !isFinite(position) ||
Math.floor(position) !== position || position > subjectString.length) {
position = subjectString.length;
}
position -= searchString.length;
var lastIndex = subjectString.indexOf(searchString, position);
return lastIndex !== -1 && lastIndex === position;
};
}
function getFrame(getFrameDetails, callback) {
if ( typeof chrome !== 'undefined' ) {
chrome.webNavigation.getFrame(getFrameDetails, callback);
} else if ( typeof browser !== 'undefined' ) {
var gettingFrame = browser.webNavigation.getFrame(getFrameDetails);
gettingFrame.then(callback);
}
}
function ifBrowserValid(callback, elseCallback) {
if ( isChrome() ) {
callback();
} else if ( typeof browser !== 'undefined' ) {
try {
var gettingInfo = browser.runtime.getBrowserInfo();
gettingInfo.then(function(browserInfo) {
var browserVersion = parseInt(browserInfo.version.split('.')[0]);
if ( browserInfo.name === 'Firefox' &&
browserVersion >= MIN_FF_MAJOR_VERSION) {
callback();
} else {
elseCallback();
}
});
} catch (err) {
elseCallback();
}
} else {
elseCallback();
}
}
function isTrackingEnabled() {
return parseInt(localStorage.tracking, 10);
}
function isPixelRequest(request) {
return (request.type === 'image' || request.responseStatus === 204) &&
request.size <= 1000;
}
function isVpaidOrVastRequest(request) {
var lowerCaseUrl = request.url.toLowerCase();
return lowerCaseUrl.indexOf('vpaid') !== -1 || lowerCaseUrl.indexOf('vast') !== -1;
}
function hasValidRequestType(request) {
return whitelistReqTypes.indexOf(request.type) >= 0;
}
function stripQueryParams(url) {
return url.split('?', 1)[0];
}
function parseHostnameFromUrl(url) {
var parser = document.createElement('a');
parser.href = url;
return parser.hostname;
}
function hasDomain(url, domain) {
return parseHostnameFromUrl(url).endsWith(domain);
}
function findHeader(headers, key) {
var header;
for ( var i = 0; i < headers.length; i += 1 ) {
header = headers[i];
if ( header.name.toLowerCase() === key ) {
return header;
}
}
return null;
}
function validVideoType(vtype) {
var goodType = videoContentTypesPrefixes.some(function(prefix) {
return vtype.indexOf(prefix) === 0;
});
return goodType;
}
function assetMsgKey(assetReq) {
var url = stripQueryParams(assetReq.url);
var key = assetReq.frameId + '-' + url;
return key;
}
var PageNetworkTrafficCollector = function(tabId) {
this.tabId = tabId;
this.displayAdFound = false;
this.requests = {};
this.msgsBeingSent = {};
this.assetsSeen = {};
this.allRedirects = {};
};
var globalPageContainer = {
collectors: {},
dyingCollectors: {},
cleanupCollector: function(tabId) {
if ( tabId in this.collectors ) {
delete globalPageContainer.collectors[tabId];
}
},
onNewNavigation: function(details) {
var tabId = details.tabId;
this.cleanupCollector(tabId);
if ( isTrackingEnabled() ) {
if ( !areListenersRegistered ) {
registerListeners();
}
this.collectors[tabId] = new PageNetworkTrafficCollector(tabId);
} else {
if ( areListenersRegistered ) {
unregisterListeners();
}
}
},
onNavigationCommitted: function(details) {
},
onNavigationCompleted: function(details) {
},
onTabClose: function(tabId, closeInfo) {
this.cleanupCollector(tabId);
delete this.collectors[tabId];
},
onDisplayAdFound: function(tabId) {
this.collectors[tabId].displayAdFound = true;
},
getRandId: function() {
return String(Math.floor(Math.random() * 1e9));
},
getCollector: function(tabId) {
if ( this.collectors.hasOwnProperty(tabId) ) {
return this.collectors[tabId];
}
return null;
},
forwardCall: function(details, collectorMemberFunction) {
var collector = this.getCollector(details.tabId);
if ( collector !== null ) {
collectorMemberFunction.apply(collector, [details]);
}
}
};
PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() {
var logMessage = Array.from(arguments).join(' ');
var message = {message: logMessage, event: 'console-log-message'};
browserProxy.tabs.sendMessage(this.tabId, message);
};
PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, isValidAd) {
var msg = {};
msg.assets = [];
msg.event_data = {};
if ( isValidAd ) {
msg.event = 'new-video-ad';
msg.requests = reqs;
msg.requests.sort(function(reqA, reqB) {return reqA.requestTimestamp - reqB.requestTimestamp;});
if ( assetReq ) {
msg.assets = [assetReq];
}
} else {
msg.requests = reqs.map(function(request) {
return parseHostnameFromUrl(request.url);
});
msg.assets = [{
url: parseHostnameFromUrl(assetReq.url),
contentType: assetReq.contentType,
size: assetReq.size
}];
msg.event = 'new-invalid-video-ad';
}
msg.origUrl = curPageUrl;
msg.displayAdFound = this.displayAdFound;
browserProxy.tabs.sendMessage(this.tabId, msg);
};
PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) {
return url + ':' + frameId;
};
PageNetworkTrafficCollector.prototype.seenBefore = function(request) {
var oldTime = this.assetsSeen[assetMsgKey(request)];
if ( oldTime && (request.requestTimestamp-oldTime < secBetweenDupAssets)){
return true;
}
return false;
};
PageNetworkTrafficCollector.prototype.recordSeenAsset = function(request) {
this.assetsSeen[assetMsgKey(request)] = request.requestTimestamp;
};
PageNetworkTrafficCollector.prototype.onBeforeRequest = function(details) {
var req = {
url: details.url,
type: details.type,
httpMethod: details.method,
frameId: details.frameId,
parentFrameId: details.parentFrameId,
requestTimestamp: details.timeStamp,
};
this.requests[details.requestId] = req;
};
PageNetworkTrafficCollector.prototype.onSendHeaders = function(details) {
var request, header;
request = this.requests[details.requestId];
header = request && findHeader(details.requestHeaders, 'x-requested-with');
if ( header && header.value.toLowerCase().indexOf('flash') > -1 ) {
request.from_flash = true;
}
};
PageNetworkTrafficCollector.prototype.onHeadersReceived = function(details) {
var getFrameDetails = {
tabId: details.tabId,
processId: null,
frameId: details.frameId
};
var pageNetworkTrafficController = this;
getFrame(getFrameDetails, function(frameDetails) {
if ( frameDetails && frameDetails.url ) {
pageNetworkTrafficController._onHeadersReceived(details, frameDetails);
}
});
};
PageNetworkTrafficCollector.prototype._onHeadersReceived = function(details, frameDetails) {
var contentSize, contentRange;
var request = this.requests[details.requestId];
if ( request ) {
var redirParent = this.allRedirects[this.getRedirKey(details.url, details.frameId)];
var header = request && findHeader(details.responseHeaders, 'content-type');
var contentType = header && header.value.toLowerCase();
if ( contentType){
request.contentType = contentType;
}
header = request && findHeader(details.responseHeaders, 'content-length');
contentSize = header && header.value;
if ( contentSize ) {
request.size = request.size || 0;
request.size += parseInt(contentSize);
}
header = request && findHeader(details.responseHeaders, 'content-range');
contentRange = header && header.value;
if ( contentRange ) {
request.contentRange = parseInt(contentRange.split('/')[1]);
}
var frameUrl = null;
if ( frameDetails && frameDetails.url ) {
frameUrl = frameDetails.url;
}
if ( !this.bannedRequest(request) &&
(this.isVideoReq(frameUrl, request) || (redirParent && redirParent.isVideo))) {
request.isVideo = true;
}
}
};
PageNetworkTrafficCollector.prototype.onBeforeRedirect = function(details) {
var request = this.requests[details.requestId];
if ( request ) {
if ( request.redirects ) {
request.redirects.push(details.redirectUrl);
} else {
request.redirects = [details.redirectUrl];
}
this.allRedirects[this.getRedirKey(details.redirectUrl, details.frameId)] = request;
}
};
PageNetworkTrafficCollector.prototype.isYoutubeMastheadRequest = function(url) {
var re = /video_masthead/;
return this.hasYoutubeDomain(url) && re.test(url);
};
PageNetworkTrafficCollector.prototype.isYoutubeVideoRequest = function(srcUrl, destUrl) {
if ( !this.hasYoutubeDomain(srcUrl) ) {
return false;
}
var re = /https?:\/\/r.*?\.googlevideo\.com\/videoplayback\?/;
return re.test(destUrl);
};
PageNetworkTrafficCollector.prototype.processResponse = function(requestDetails, frameDetails) {
var request;
if ( requestDetails ) {
request = this.requests[requestDetails.requestId];
if ( request ) {
request.responseStatus = requestDetails.statusCode;
request.responseTimestamp = requestDetails.timeStamp;
var frameUrl = null;
if ( frameDetails && frameDetails.url ) {
frameUrl = frameDetails.url;
}
var requestUrl = null;
if ( request.url ) {
requestUrl = request.url;
}
if ( this.isYoutubeAdReq(frameUrl, requestUrl) ) {
var videoId = this.parseYoutubeVideoIdFromUrl(requestUrl);
if ( videoId ) {
request.isYoutubeAd = true;
request.isVideo = true;
request.url = 'https://www.youtube.com/watch?v=' + this.parseYoutubeVideoIdFromUrl(requestUrl);
}
} else if ( !this.bannedRequest(request) &&
(this.isVideo || this.isVideoReq(frameUrl, request))) {
request.isVideo = true;
}
if ( request.isVideo ) {
var msgKey = assetMsgKey(request);
this.msgsBeingSent[msgKey] = request;
if ( !this.seenBefore(request) ) {
this.sendMsgWhenQuiet(msgKey);
}
this.recordSeenAsset(request);
}
}
}
};
PageNetworkTrafficCollector.prototype.onResponseStarted = function(responseDetails) {
if ( responseDetails.frameId < 0 ) {
responseDetails.frameId = 99999;
}
var getFrameDetails = {
tabId: responseDetails.tabId,
processId: null,
frameId: responseDetails.frameId
};
var pageNetworkTrafficController = this;
getFrame(getFrameDetails, function(frameDetails) {
if ( frameDetails && frameDetails.url ) {
pageNetworkTrafficController.processResponse(responseDetails, frameDetails);
}
});
};
PageNetworkTrafficCollector.prototype.hasBannedFiletype = function(request) {
var url = stripQueryParams(request.url);
if ( bannedFiletypesReg.exec(url) ) {
return true;
} else {
return false;
}
};
PageNetworkTrafficCollector.prototype.checkContentHeaders = function(request) {
if ( request.contentType && validVideoType(request.contentType) ) {
return true;
}
return false;
};
PageNetworkTrafficCollector.prototype.checkUrlExtension = function(request) {
var url = stripQueryParams(request.url);
if ( extensionsReg.exec(url) ) {
return true;
} else {
return false;
}
};
PageNetworkTrafficCollector.prototype.isVideoReq = function(srcUrl, request) {
if ( this.isYoutubeVideoRequest(srcUrl, request.url) ) {
return false;
}
return this.checkUrlExtension(request) || this.checkContentHeaders(request);
};
PageNetworkTrafficCollector.prototype.hasYoutubeDomain = function(url) {
var hostname = parseHostnameFromUrl(url) ;
if ( hostname === 'www.youtube.com' ) {
return true;
}
return false;
};
PageNetworkTrafficCollector.prototype.parseYoutubeVideoIdFromUrl = function(url) {
var re = /^https?:\/\/www\.youtube\.com\/get_video_info.*(?:\?|&)video_id=(.*?)(?:$|&)/;
var match = re.exec(url);
if ( match && match.length > 1 ) {
return match[1];
}
re = /^https?:\/\/www\.youtube\.com\/embed\/(.*?)(?:$|\?)/;
match = re.exec(url);
if ( match && match.length > 1 ) {
return match[1];
}
re = /^https?:\/\/www\.youtube\.com\/watch\?v=(.*$)/;
match = re.exec(url);
if ( match && match.length > 1 ) {
return match[1];
}
return null;
};
PageNetworkTrafficCollector.prototype.isYoutubeGetVideoInfoReq = function(url) {
var re = /^https?:\/\/www\.youtube\.com\/get_video_info\?/;
return re.test(url);
};
PageNetworkTrafficCollector.prototype.isYoutubeAdReq = function(srcUrl, destUrl) {
if ( !this.hasYoutubeDomain(srcUrl) ||
!this.isYoutubeGetVideoInfoReq(destUrl)) {
return false;
}
if ( this.parseYoutubeVideoIdFromUrl(srcUrl) ===
this.parseYoutubeVideoIdFromUrl(destUrl) &&
!this.isYoutubeMastheadRequest(destUrl)) {
return false;
}
return true;
};
PageNetworkTrafficCollector.prototype.bannedRequest = function(request) {
return this.bannedVideoType(request) || this.hasBannedFiletype(request) || this.bannedVideoSize(request);
};
PageNetworkTrafficCollector.prototype.bannedVideoType = function(request) {
var badType = false;
if ( request.contentType ) {
badType = bannedContentTypes.some(function(prefix) {
return request.contentType.indexOf(prefix) >= 0;
});
}
return badType;
};
PageNetworkTrafficCollector.prototype.bannedVideoSize = function(request) {
if ( request.size !== null ) {
if ( request.size < minVidSize || request.size > maxVidSize || request.contentRange > maxContentRange ) {
return true;
}
}
return false;
};
PageNetworkTrafficCollector.prototype.grabTagReqs = function(tabRequests, assetRequest) {
var minTimestamp, maxTimestamp;
minTimestamp = assetRequest.requestTimestamp - secBefore;
maxTimestamp = assetRequest.requestTimestamp + secAfter;
var filteredRequests = tabRequests.filter(function(request) {
return (request.requestTimestamp > minTimestamp &&
request.requestTimestamp < maxTimestamp &&
request.frameId === assetRequest.frameId &&
request.url !== assetRequest.url &&
(hasValidRequestType(request) ||
isPixelRequest(request)));
});
return filteredRequests;
};
PageNetworkTrafficCollector.prototype.isValidVideoAd = function(assetRequest, tagRequests) {
var hasVpaidOrVastRequest = tagRequests.some(function(tagRequest) {
return isVpaidOrVastRequest(tagRequest);
});
if ( assetRequest.isYoutubeAd ) {
return true;
}
if ( hasVpaidOrVastRequest ) {
return true;
}
var hasTopVideoAssetDomain = topVideoAssetDomains.some(function(assetDomain) {
return hasDomain(assetRequest.url, assetDomain);
});
return hasTopVideoAssetDomain;
};
PageNetworkTrafficCollector.prototype.sendMsgWhenQuiet = function(msgKey) {
var _this = this,
origPageUrl, msgAssetReq;
msgAssetReq = this.msgsBeingSent[msgKey];
browserProxy.tabs.get(this.tabId, function(tab) {origPageUrl = tab.url;});
setTimeout(function() {
var rawRequests = [];
if ( globalPageContainer.collectors[_this.tabId] === _this ) {
for ( var reqId in _this.requests ) {
rawRequests.push(_this.requests[reqId]);
}
var tagReqs = _this.grabTagReqs(rawRequests, msgAssetReq);
if ( _this.isValidVideoAd(msgAssetReq, tagReqs) ) {
_this.sendToTab(msgAssetReq, tagReqs, origPageUrl, true);
} else {
_this.sendToTab(msgAssetReq, tagReqs, origPageUrl, false);
}
} else {
}
delete _this.msgsBeingSent[msgKey];
}, secAfter+secBefore);
};
PageNetworkTrafficCollector.prototype.existingMessage = function(candidateRequest) {
var frameMsg = this.msgsBeingSent[candidateRequest.frameId];
if ( frameMsg ) {
return frameMsg;
} else {
return null;
}
};
function onBeforeRequestListener(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRequest);
}
function onSendHeadersListener(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onSendHeaders);
}
function onHeadersReceivedListener(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onHeadersReceived);
}
function onBeforeRedirectListener(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRedirect);
}
function onResponseStartedListener(details) {
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onResponseStarted);
}
function onCommittedListener(details) {
if ( details.frameId === 0 ) {
globalPageContainer.onNavigationCommitted(details);
}
}
function onCompletedListener(details) {
if ( details.frameId === 0 ) {
globalPageContainer.onNavigationCompleted(details);
}
}
function onRemovedListener(tabId, closeInfo) {
globalPageContainer.onTabClose(tabId, closeInfo);
}
function onMessageListener(message, sender, sendResponse) {
if ( message.event === 'new-ad' && message.data.event === 'ad' ) {
var tabId = sender.tab.id;
if ( tabId ) {
globalPageContainer.onDisplayAdFound(tabId);
}
}
}
function registerListeners() {
browserProxy.webRequest.onBeforeRequest.addListener(
onBeforeRequestListener,
{urls: ['http://*/*', 'https://*/*']},
[]
);
browserProxy.webRequest.onSendHeaders.addListener(
onSendHeadersListener,
{urls: ['http://*/*', 'https://*/*']},
['requestHeaders']
);
browserProxy.webRequest.onHeadersReceived.addListener(
onHeadersReceivedListener,
{urls: ['http://*/*', 'https://*/*']},
['responseHeaders']
);
browserProxy.webRequest.onBeforeRedirect.addListener(
onBeforeRedirectListener,
{urls: ['http://*/*', 'https://*/*']},
[]
);
browserProxy.webRequest.onResponseStarted.addListener(
onResponseStartedListener,
{urls: ['http://*/*', 'https://*/*']},
['responseHeaders']
);
browserProxy.webNavigation.onCommitted.addListener(onCommittedListener);
browserProxy.webNavigation.onCompleted.addListener(onCompletedListener);
browserProxy.tabs.onRemoved.addListener(onRemovedListener);
browserProxy.runtime.onMessage.addListener(onMessageListener);
areListenersRegistered = true;
}
function unregisterListeners() {
browserProxy.webRequest.onBeforeRequest.removeListener(
onBeforeRequestListener
);
browserProxy.webRequest.onSendHeaders.removeListener(
onSendHeadersListener
);
browserProxy.webRequest.onHeadersReceived.removeListener(
onHeadersReceivedListener
);
browserProxy.webRequest.onBeforeRedirect.removeListener(
onBeforeRedirectListener
);
browserProxy.webRequest.onResponseStarted.removeListener(
onResponseStartedListener
);
browserProxy.webNavigation.onCommitted.removeListener(onCommittedListener);
browserProxy.webNavigation.onCompleted.removeListener(onCompletedListener);
browserProxy.tabs.onRemoved.removeListener(onRemovedListener);
browserProxy.runtime.onMessage.removeListener(onMessageListener);
areListenersRegistered = false;
}
function areRequiredBrowserApisAvailable() {
return requiredBrowserApis.every(function(api) {
return typeof api !== 'undefined';
});
}
if ( areRequiredBrowserApisAvailable() ) {
ifBrowserValid(
function() {
browserProxy.webNavigation.onBeforeNavigate.addListener(
function(details) {
if ( details.frameId === 0 ) {
globalPageContainer.onNewNavigation(details);
}
},
{
url: [{urlMatches: 'http://*/*'}, {urlMatches: 'https://*/*'}]
}
);
}, function() {
}
);
}
browserProxy.runtime.onMessage.addListener(function(request, sender, sendResponse) {
if ( request === 'is_browser_valid' ) {
ifBrowserValid(
sendResponse({'browser_valid': true}),
sendResponse({'browser_valid': false})
);
}
});
browserProxy.runtime.onMessage.addListener(function(request, sender, sendResponse) {
if ( request === 'is_tracking_enabled' ) {
sendResponse({'tracking_enabled': isTrackingEnabled()});
}
});
})();