';
- if ( detected != null && Object.keys(detected).length ) {
- for ( app in detected ) {
- if ( !hasOwn.call(detected, app) ) {
+ if (detected != null && Object.keys(detected).length) {
+ for (app in detected) {
+ if (!hasOwn.call(detected, app)) {
continue;
}
- var version = detected[app].version,
+ let version = detected[app].version,
confidence = detected[app].confidence;
- html +=
- '
' +
- '' +
- '' +
- ' + ')
' + app +
- '' +
- ( version ? ' ' + version : '' ) + ( confidence < 100 ? ' (' + confidence + '% sure)' : '' ) +
- '';
-
- for ( let i in wappalyzer.apps[app].cats ) {
- if ( !hasOwn.call(wappalyzer.apps[app].cats, i) ) {
+ html
+ += `]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"/i);
@@ -65,60 +170,62 @@ class Wappalyzer {
language = matches && matches.length ? matches[1] : null;
}
- Object.keys(this.apps).forEach(appName => {
- apps[appName] = this.detected[url.canonical] && this.detected[url.canonical][appName] ? this.detected[url.canonical][appName] : new Application(appName, this.apps[appName]);
+ Object.keys(this.apps).forEach((appName) => {
+ apps[appName] = this.detected[url.canonical] && this.detected[url.canonical][appName]
+ ? this.detected[url.canonical][appName]
+ : new Application(appName, this.apps[appName]);
- var app = apps[appName];
+ const app = apps[appName];
this.analyzeUrl(app, url);
- if ( data.html ) {
- promises.push(this.analyzeHtml(app, data.html));
- promises.push(this.analyzeMeta(app, data.html));
+ if (html) {
+ promises.push(this.analyzeHtml(app, html));
+ promises.push(this.analyzeMeta(app, html));
}
- if ( data.scripts ) {
- promises.push(this.analyzeScripts(app, data.scripts));
+ if (scripts) {
+ promises.push(this.analyzeScripts(app, scripts));
}
- if ( data.cookies ) {
- promises.push(this.analyzeCookies(app, data.cookies));
+ if (cookies) {
+ promises.push(this.analyzeCookies(app, cookies));
}
- if ( data.headers ) {
- promises.push(this.analyzeHeaders(app, data.headers));
+ if (headers) {
+ promises.push(this.analyzeHeaders(app, headers));
}
});
- if ( data.js ) {
- Object.keys(data.js).forEach(appName => {
- if (typeof data.js[appName] != 'function') {
- promises.push(this.analyzeJs(apps[appName], data.js[appName]));
+ if (js) {
+ Object.keys(js).forEach((appName) => {
+ if (typeof js[appName] !== 'function') {
+ promises.push(this.analyzeJs(apps[appName], js[appName]));
}
});
}
- return new Promise(resolve => {
+ return new Promise((resolve) => {
Promise.all(promises)
.then(() => {
- Object.keys(apps).forEach(appName => {
- let app = apps[appName];
+ Object.keys(apps).forEach((appName) => {
+ const app = apps[appName];
if (!app.detected || !app.getConfidence()) {
delete apps[app.name];
}
});
- this.resolveExcludes(apps, this.detected[url]);
+ resolveExcludes(apps, this.detected[url]);
this.resolveImplies(apps, url.canonical);
this.cacheDetectedApps(apps, url.canonical);
this.trackDetectedApps(apps, url, language);
- this.log('Processing ' + Object.keys(data).join(', ') + ' took ' + (( new Date() - startTime ) / 1000).toFixed(2) + 's (' + url.hostname + ')', 'core');
+ this.log(`Processing ${Object.keys(data).join(', ')} took ${((new Date() - startTime) / 1000).toFixed(2)}s (${url.hostname})`, 'core');
- if ( Object.keys(apps).length ) {
- this.log('Identified ' + Object.keys(apps).join(', ') + ' (' + url.hostname + ')', 'core');
+ if (Object.keys(apps).length) {
+ this.log(`Identified ${Object.keys(apps).join(', ')} (${url.hostname})`, 'core');
}
this.driver.displayApps(this.detected[url.canonical], { language }, context);
@@ -140,32 +247,34 @@ class Wappalyzer {
*/
robotsTxtAllows(url) {
return new Promise((resolve, reject) => {
- var parsed = this.parseUrl(url);
+ const parsed = this.parseUrl(url);
- if ( parsed.protocol !== 'http:' && parsed.protocol !== 'https:' ) {
- return reject();
+ if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+ reject();
+
+ return;
}
this.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
- .then(robotsTxt => {
- if ( robotsTxt.some(disallowedPath => parsed.pathname.indexOf(disallowedPath) === 0) ) {
+ .then((robotsTxt) => {
+ if (robotsTxt.some(disallowedPath => parsed.pathname.indexOf(disallowedPath) === 0)) {
return reject();
}
return resolve();
}, () => resolve());
});
- };
+ }
/**
* Parse a URL
*/
parseUrl(url) {
- var a = this.driver.document.createElement('a');
+ const a = this.driver.document.createElement('a');
a.href = url;
- a.canonical = a.protocol + '//' + a.host + a.pathname;
+ a.canonical = `${a.protocol}//${a.host}${a.pathname}`;
return a;
}
@@ -173,22 +282,21 @@ class Wappalyzer {
/**
*
*/
- parseRobotsTxt(robotsTxt) {
- var userAgent;
- var disallow = [];
+ static parseRobotsTxt(robotsTxt) {
+ const disallow = [];
- robotsTxt.split('\n').forEach(line => {
- var matches = /^User-agent:\s*(.+)$/i.exec(line);
+ let userAgent;
- if ( matches ) {
+ robotsTxt.split('\n').forEach((line) => {
+ let matches = /^User-agent:\s*(.+)$/i.exec(line);
+
+ if (matches) {
userAgent = matches[1].toLowerCase();
- } else {
- if ( userAgent === '*' || userAgent === 'wappalyzer' ) {
- matches = /^Disallow:\s*(.+)$/i.exec(line);
+ } else if (userAgent === '*' || userAgent === 'wappalyzer') {
+ matches = /^Disallow:\s*(.+)$/i.exec(line);
- if ( matches ) {
- disallow.push(matches[1]);
- }
+ if (matches) {
+ disallow.push(matches[1]);
}
}
});
@@ -200,66 +308,59 @@ class Wappalyzer {
*
*/
ping() {
- if ( Object.keys(this.hostnameCache).length > 100 ) {
+ if (Object.keys(this.hostnameCache).length > 100) {
this.driver.ping(this.hostnameCache);
this.hostnameCache = {};
}
- if ( this.adCache.length > 50 ) {
+ if (this.adCache.length > 50) {
this.driver.ping({}, this.adCache);
this.adCache = [];
}
}
- /**
- * Enclose string in array
- */
- asArray(value) {
- return value instanceof Array ? value : [ value ];
- }
-
/**
* Parse apps.json patterns
*/
parsePatterns(patterns) {
- if ( !patterns ) {
+ if (!patterns) {
return [];
}
- var parsed = {};
+ let parsed = {};
// Convert string to object containing array containing string
- if ( typeof patterns === 'string' || patterns instanceof Array ) {
+ if (typeof patterns === 'string' || patterns instanceof Array) {
patterns = {
- main: this.asArray(patterns)
+ main: asArray(patterns),
};
}
- Object.keys(patterns).forEach(key => {
+ Object.keys(patterns).forEach((key) => {
parsed[key] = [];
- this.asArray(patterns[key]).forEach(pattern => {
- var attrs = {};
+ asArray(patterns[key]).forEach((pattern) => {
+ const attrs = {};
pattern.split('\\;').forEach((attr, i) => {
- if ( i ) {
+ if (i) {
// Key value pairs
attr = attr.split(':');
- if ( attr.length > 1 ) {
+ if (attr.length > 1) {
attrs[attr.shift()] = attr.join(':');
}
} else {
attrs.string = attr;
try {
- attrs.regex = new RegExp(attr.replace('/', '\/'), 'i'); // Escape slashes in regular expression
+ attrs.regex = new RegExp(attr.replace('/', '\\/'), 'i'); // Escape slashes in regular expression
} catch (e) {
attrs.regex = new RegExp();
- this.log(e + ': ' + attr, 'error', 'core');
+ this.log(`${e}: ${attr}`, 'error', 'core');
}
}
});
@@ -269,7 +370,7 @@ class Wappalyzer {
});
// Convert back to array if the original pattern list was an array (or string)
- if ( 'main' in parsed ) {
+ if ('main' in parsed) {
parsed = parsed.main;
}
@@ -280,65 +381,45 @@ class Wappalyzer {
* Parse JavaScript patterns
*/
parseJsPatterns() {
- Object.keys(this.apps).forEach(appName => {
+ Object.keys(this.apps).forEach((appName) => {
if (this.apps[appName].js) {
this.jsPatterns[appName] = this.parsePatterns(this.apps[appName].js);
}
});
}
- resolveExcludes(apps, detected) {
- let excludes = [];
-
- // Exclude app in detected apps only
- Object.keys(Object.assign({}, apps, detected)).forEach(appName => {
- let app = apps[appName];
-
- if (app.props.excludes) {
- this.asArray(app.props.excludes).forEach(excluded => {
- excludes.push(excluded);
- });
- }
- });
-
- // Remove excluded applications
- Object.keys(apps).forEach(appName => {
- if (excludes.indexOf(appName) > -1) {
- delete apps[appName];
- }
- });
- }
-
resolveImplies(apps, url) {
- var checkImplies = true;
+ let checkImplies = true;
// Implied applications
// Run several passes as implied apps may imply other apps
- while ( checkImplies ) {
+ while (checkImplies) {
checkImplies = false;
- Object.keys(apps).forEach(appName => {
- var app = apps[appName];
+ Object.keys(apps).forEach((appName) => {
+ const app = apps[appName];
- if ( app && app.props.implies ) {
- this.asArray(app.props.implies).forEach(implied => {
- implied = this.parsePatterns(implied)[0];
+ if (app && app.props.implies) {
+ asArray(app.props.implies).forEach((implied) => {
+ [implied] = this.parsePatterns(implied);
- if ( !this.apps[implied.string] ) {
- this.log('Implied application ' + implied.string + ' does not exist', 'core', 'warn');
+ if (!this.apps[implied.string]) {
+ this.log(`Implied application ${implied.string} does not exist`, 'core', 'warn');
return;
}
- if ( !( implied.string in apps ) ) {
- apps[implied.string] = this.detected[url] && this.detected[url][implied.string] ? this.detected[url][implied.string] : new Application(implied.string, this.apps[implied.string], true);
+ if (!(implied.string in apps)) {
+ apps[implied.string] = this.detected[url] && this.detected[url][implied.string]
+ ? this.detected[url][implied.string]
+ : new Application(implied.string, this.apps[implied.string], true);
checkImplies = true;
}
// Apply app confidence to implied app
- Object.keys(app.confidence).forEach(id => {
- apps[implied.string].confidence[id + ' implied by ' + appName] = app.confidence[id] * ( implied.confidence === undefined ? 1 : implied.confidence / 100 );
+ Object.keys(app.confidence).forEach((id) => {
+ apps[implied.string].confidence[`${id} implied by ${appName}`] = app.confidence[id] * (implied.confidence === undefined ? 1 : implied.confidence / 100);
});
});
}
@@ -350,14 +431,17 @@ class Wappalyzer {
* Cache detected applications
*/
cacheDetectedApps(apps, url) {
- Object.keys(apps).forEach(appName => {
- let app = apps[appName];
+ Object.keys(apps).forEach((appName) => {
+ const app = apps[appName];
// Per URL
this.detected[url][appName] = app;
- Object.keys(app.confidence).forEach(id => this.detected[url][appName].confidence[id] = app.confidence[id]);
- })
+ Object.keys(app.confidence)
+ .forEach((id) => {
+ this.detected[url][appName].confidence[id] = app.confidence[id];
+ });
+ });
if (this.driver.ping instanceof Function) {
this.ping();
@@ -368,41 +452,44 @@ class Wappalyzer {
* Track detected applications
*/
trackDetectedApps(apps, url, language) {
- if ( !( this.driver.ping instanceof Function ) ) {
+ if (!(this.driver.ping instanceof Function)) {
return;
}
- const hostname = url.protocol + '//' + url.hostname;
+ const hostname = `${url.protocol}//${url.hostname}`;
- Object.keys(apps).forEach(appName => {
+ Object.keys(apps).forEach((appName) => {
const app = apps[appName];
- if ( this.detected[url.canonical][appName].getConfidence() >= 100 ) {
- if ( validation.hostname.test(url.hostname) && !validation.hostnameBlacklist.test(url.hostname) ) {
- if ( !( hostname in this.hostnameCache ) ) {
+ if (this.detected[url.canonical][appName].getConfidence() >= 100) {
+ if (
+ validation.hostname.test(url.hostname)
+ && !validation.hostnameBlacklist.test(url.hostname)
+ ) {
+ if (!(hostname in this.hostnameCache)) {
this.hostnameCache[hostname] = {
applications: {},
- meta: {}
+ meta: {},
};
}
- if ( !( appName in this.hostnameCache[hostname].applications ) ) {
+ if (!(appName in this.hostnameCache[hostname].applications)) {
this.hostnameCache[hostname].applications[appName] = {
- hits: 0
+ hits: 0,
};
}
- this.hostnameCache[hostname].applications[appName].hits ++;
+ this.hostnameCache[hostname].applications[appName].hits += 1;
- if ( apps[appName].version ) {
+ if (apps[appName].version) {
this.hostnameCache[hostname].applications[appName].version = app.version;
}
}
}
});
- if ( hostname in this.hostnameCache ) {
- this.hostnameCache[hostname].meta['language'] = language;
+ if (hostname in this.hostnameCache) {
+ this.hostnameCache[hostname].meta.language = language;
}
this.ping();
@@ -412,15 +499,15 @@ class Wappalyzer {
* Analyze URL
*/
analyzeUrl(app, url) {
- var patterns = this.parsePatterns(app.props.url);
+ const patterns = this.parsePatterns(app.props.url);
- if ( !patterns.length ) {
+ if (!patterns.length) {
return Promise.resolve();
}
- return this.asyncForEach(patterns, pattern => {
- if ( pattern.regex.test(url.canonical) ) {
- this.addDetected(app, pattern, 'url', url.canonical);
+ return asyncForEach(patterns, (pattern) => {
+ if (pattern.regex.test(url.canonical)) {
+ addDetected(app, pattern, 'url', url.canonical);
}
});
}
@@ -429,15 +516,15 @@ class Wappalyzer {
* Analyze HTML
*/
analyzeHtml(app, html) {
- var patterns = this.parsePatterns(app.props.html);
+ const patterns = this.parsePatterns(app.props.html);
- if ( !patterns.length ) {
+ if (!patterns.length) {
return Promise.resolve();
}
- return this.asyncForEach(patterns, pattern => {
- if ( pattern.regex.test(html) ) {
- this.addDetected(app, pattern, 'html', html);
+ return asyncForEach(patterns, (pattern) => {
+ if (pattern.regex.test(html)) {
+ addDetected(app, pattern, 'html', html);
}
});
}
@@ -446,16 +533,16 @@ class Wappalyzer {
* Analyze script tag
*/
analyzeScripts(app, scripts) {
- var patterns = this.parsePatterns(app.props.script);
+ const patterns = this.parsePatterns(app.props.script);
- if ( !patterns.length ) {
+ if (!patterns.length) {
return Promise.resolve();
}
- return this.asyncForEach(patterns, pattern => {
- scripts.forEach(uri => {
- if ( pattern.regex.test(uri) ) {
- this.addDetected(app, pattern, 'script', uri);
+ return asyncForEach(patterns, (pattern) => {
+ scripts.forEach((uri) => {
+ if (pattern.regex.test(uri)) {
+ addDetected(app, pattern, 'script', uri);
}
});
});
@@ -471,20 +558,20 @@ class Wappalyzer {
let matches;
- while ( patterns && ( matches = regex.exec(html) ) ) {
- for ( let meta in patterns ) {
- const r = new RegExp('(?:name|property)=["\']' + meta + '["\']', 'i');
+ while (patterns && (matches = regex.exec(html))) {
+ patterns.forEach((meta) => {
+ const r = new RegExp(`(?:name|property)=["']${meta}["']`, 'i');
- if ( r.test(matches[0]) ) {
- let content = matches[0].match(/content=("|')([^"']+)("|')/i);
+ if (r.test(matches[0])) {
+ const content = matches[0].match(/content=("|')([^"']+)("|')/i);
- promises.push(this.asyncForEach(patterns[meta], pattern => {
- if ( content && content.length === 4 && pattern.regex.test(content[2]) ) {
- this.addDetected(app, pattern, 'meta', content[2], meta);
+ promises.push(asyncForEach(patterns[meta], (pattern) => {
+ if (content && content.length === 4 && pattern.regex.test(content[2])) {
+ addDetected(app, pattern, 'meta', content[2], meta);
}
}));
}
- }
+ });
}
return promises ? Promise.all(promises) : Promise.resolve();
@@ -497,15 +584,15 @@ class Wappalyzer {
const patterns = this.parsePatterns(app.props.headers);
const promises = [];
- Object.keys(patterns).forEach(headerName => {
- if (typeof patterns[headerName] != 'function') {
- promises.push(this.asyncForEach(patterns[headerName], pattern => {
+ Object.keys(patterns).forEach((headerName) => {
+ if (typeof patterns[headerName] !== 'function') {
+ promises.push(asyncForEach(patterns[headerName], (pattern) => {
headerName = headerName.toLowerCase();
- if ( headerName in headers ) {
- headers[headerName].forEach(headerValue => {
- if ( pattern.regex.test(headerValue) ) {
- this.addDetected(app, pattern, 'headers', headerValue, headerName);
+ if (headerName in headers) {
+ headers[headerName].forEach((headerValue) => {
+ if (pattern.regex.test(headerValue)) {
+ addDetected(app, pattern, 'headers', headerValue, headerName);
}
});
}
@@ -523,15 +610,15 @@ class Wappalyzer {
const patterns = this.parsePatterns(app.props.cookies);
const promises = [];
- Object.keys(patterns).forEach(cookieName => {
- if (typeof patterns[cookieName] != 'function') {
+ Object.keys(patterns).forEach((cookieName) => {
+ if (typeof patterns[cookieName] !== 'function') {
cookieName = cookieName.toLowerCase();
- promises.push(this.asyncForEach(patterns[cookieName], pattern => {
- const cookie = cookies.find(cookie => cookie.name.toLowerCase() === cookieName);
+ promises.push(asyncForEach(patterns[cookieName], (pattern) => {
+ const cookie = cookies.find(_cookie => _cookie.name.toLowerCase() === cookieName);
- if ( cookie && pattern.regex.test(cookie.value) ) {
- this.addDetected(app, pattern, 'cookies', cookie.value, cookieName);
+ if (cookie && pattern.regex.test(cookie.value)) {
+ addDetected(app, pattern, 'cookies', cookie.value, cookieName);
}
}));
}
@@ -546,14 +633,14 @@ class Wappalyzer {
analyzeJs(app, results) {
const promises = [];
- Object.keys(results).forEach(string => {
- if (typeof results[string] != 'function') {
- promises.push(this.asyncForEach(Object.keys(results[string]), index => {
+ Object.keys(results).forEach((string) => {
+ if (typeof results[string] !== 'function') {
+ promises.push(asyncForEach(Object.keys(results[string]), (index) => {
const pattern = this.jsPatterns[app.name][string][index];
const value = results[string][index];
- if ( pattern && pattern.regex.test(value) ) {
- this.addDetected(app, pattern, 'js', value);
+ if (pattern && pattern.regex.test(value)) {
+ addDetected(app, pattern, 'js', value);
}
}));
}
@@ -561,76 +648,8 @@ class Wappalyzer {
return promises ? Promise.all(promises) : Promise.resolve();
}
-
- /**
- * Mark application as detected, set confidence and version
- */
- addDetected(app, pattern, type, value, key) {
- app.detected = true;
-
- // Set confidence level
- app.confidence[type + ' ' + ( key ? key + ' ' : '' ) + pattern.regex] = pattern.confidence === undefined ? 100 : parseInt(pattern.confidence);
-
- // Detect version number
- if ( pattern.version ) {
- var versions = [];
- var version = pattern.version;
- var matches = pattern.regex.exec(value);
-
- if ( matches ) {
- matches.forEach((match, i) => {
- // Parse ternary operator
- var ternary = new RegExp('\\\\' + i + '\\?([^:]+):(.*)$').exec(version);
-
- if ( ternary && ternary.length === 3 ) {
- version = version.replace(ternary[0], match ? ternary[1] : ternary[2]);
- }
-
- // Replace back references
- version = version.trim().replace(new RegExp('\\\\' + i, 'g'), match || '');
- });
-
- if ( version && versions.indexOf(version) === -1 ) {
- versions.push(version);
- }
-
- if ( versions.length ) {
- // Use the longest detected version number
- app.version = versions.reduce((a, b) => a.length > b.length ? a : b);
- }
- }
- }
- }
-}
-
-/**
- * Application class
- */
-class Application {
- constructor(name, props, detected) {
- this.confidence = {};
- this.confidenceTotal = 0;
- this.detected = Boolean(detected);
- this.excludes = [];
- this.name = name;
- this.props = props;
- this.version = '';
- }
-
- /**
- * Calculate confidence total
- */
- getConfidence() {
- var total = 0;
-
- for ( let id in this.confidence ) {
- total += this.confidence[id];
- }
-
- return this.confidenceTotal = Math.min(total, 100);
- }
}
-if ( typeof module === 'object' ) {
+if (typeof module === 'object') {
module.exports = Wappalyzer;
}
' + app +
- '' +
- ( version ? ' ' + version : '' ) + ( confidence < 100 ? ' (' + confidence + '% sure)' : '' ) +
- '';
-
- for ( let i in wappalyzer.apps[app].cats ) {
- if ( !hasOwn.call(wappalyzer.apps[app].cats, i) ) {
+ html
+ += ``
+ + ``
+ + ''
+ + `
${app
+ }${
+ version ? ` ${version}` : ''}${confidence < 100 ? ` (${confidence}% sure)` : ''
+ }`;
+
+ for (const i in wappalyzer.apps[app].cats) {
+ if (!hasOwn.call(wappalyzer.apps[app].cats, i)) {
continue;
}
category = wappalyzer.categories[wappalyzer.apps[app].cats[i]].name;
- html += '' + category + '';
+ html += `${category}`;
}
html += '
';
@@ -143,7 +143,7 @@
*/
function openTab(args) {
open(args.url);
- }
+ };
function slugify(string) {
return string.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/--+/g, '-').replace(/(?:^-|-$)/, '');
@@ -151,4 +151,4 @@
getPageContent();
getResponseHeaders();
-})();
+}());
diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js
index 664b4644f..d039c0eed 100644
--- a/src/drivers/npm/driver.js
+++ b/src/drivers/npm/driver.js
@@ -1,27 +1,79 @@
-'use strict';
-const Wappalyzer = require('./wappalyzer');
+
const url = require('url');
const fs = require('fs');
const path = require('path');
const Browser = require('zombie');
+const Wappalyzer = require('./wappalyzer');
-const json = JSON.parse(fs.readFileSync(path.resolve(__dirname + '/apps.json')));
+const json = JSON.parse(fs.readFileSync(path.resolve(`${__dirname}/apps.json`)));
const extensions = /^([^.]+$|\.(asp|aspx|cgi|htm|html|jsp|php)$)/;
const errorTypes = {
- RESPONSE_NOT_OK: 'Response was not ok',
- NO_RESPONSE: 'No response from server',
- NO_HTML_DOCUMENT: 'No HTML document',
+ RESPONSE_NOT_OK: 'Response was not ok',
+ NO_RESPONSE: 'No response from server',
+ NO_HTML_DOCUMENT: 'No HTML document',
};
+function sleep(ms) {
+ return ms ? new Promise(resolve => setTimeout(resolve, ms)) : Promise.resolve();
+}
+
+function getHeaders(browser) {
+ const headers = {};
+
+ const resource = browser.resources.length
+ ? browser.resources.filter(_resource => _resource.response).shift() : null;
+
+ if (resource) {
+ // eslint-disable-next-line no-underscore-dangle
+ resource.response.headers._headers.forEach((header) => {
+ if (!headers[header[0]]) {
+ headers[header[0]] = [];
+ }
+
+ headers[header[0]].push(header[1]);
+ });
+ }
+
+ return headers;
+}
+
+function getScripts(browser) {
+ if (!browser.document || !browser.document.scripts) {
+ return [];
+ }
+
+ const scripts = Array.prototype.slice
+ .apply(browser.document.scripts)
+ .filter(script => script.src)
+ .map(script => script.src);
+
+ return scripts;
+}
+
+function getCookies(browser) {
+ const cookies = [];
+
+ if (browser.cookies) {
+ browser.cookies.forEach(cookie => cookies.push({
+ name: cookie.key,
+ value: cookie.value,
+ domain: cookie.domain,
+ path: cookie.path,
+ }));
+ }
+
+ return cookies;
+}
+
class Driver {
constructor(pageUrl, options) {
this.options = Object.assign({}, {
- password: '',
- proxy: null,
- username: '',
+ password: '',
+ proxy: null,
+ username: '',
chunkSize: 5,
debug: false,
delay: 500,
@@ -58,46 +110,48 @@ class Driver {
this.wappalyzer.driver.log = (message, source, type) => this.log(message, source, type);
this.wappalyzer.driver.displayApps = (detected, meta, context) => this.displayApps(detected, meta, context);
- process.on('uncaughtException', e => this.wappalyzer.log('Uncaught exception: ' + e.message, 'driver', 'error'));
+ process.on('uncaughtException', e => this.wappalyzer.log(`Uncaught exception: ${e.message}`, 'driver', 'error'));
}
analyze() {
this.time = {
start: new Date().getTime(),
last: new Date().getTime(),
- }
+ };
return this.crawl(this.origPageUrl);
}
log(message, source, type) {
- this.options.debug && console.log('[wappalyzer ' + type + ']', '[' + source + ']', message);
+ if (this.options.debug) {
+ console.log(`[wappalyzer ${type}]`, `[${source}]`, message);
+ }
}
displayApps(detected, meta) {
this.meta = meta;
- Object.keys(detected).forEach(appName => {
+ Object.keys(detected).forEach((appName) => {
const app = detected[appName];
- var categories = [];
+ const categories = [];
- app.props.cats.forEach(id => {
- var category = {};
+ app.props.cats.forEach((id) => {
+ const category = {};
category[id] = json.categories[id].name;
- categories.push(category)
+ categories.push(category);
});
- if ( !this.apps.some(detectedApp => detectedApp.name === app.name) ) {
+ if (!this.apps.some(detectedApp => detectedApp.name === app.name)) {
this.apps.push({
name: app.name,
confidence: app.confidenceTotal.toString(),
version: app.version || null,
icon: app.props.icon || 'default.svg',
website: app.props.website,
- categories
+ categories,
});
}
});
@@ -105,55 +159,65 @@ class Driver {
fetch(pageUrl, index, depth) {
// Return when the URL is a duplicate or maxUrls has been reached
- if (this.analyzedPageUrls[pageUrl.href] || this.analyzedPageUrls.length >= this.options.maxUrls) {
+ if (
+ this.analyzedPageUrls[pageUrl.href]
+ || this.analyzedPageUrls.length >= this.options.maxUrls
+ ) {
return Promise.resolve();
}
this.analyzedPageUrls[pageUrl.href] = {
- status: 0,
- };
+ status: 0,
+ };
const timerScope = {
- last: new Date().getTime()
+ last: new Date().getTime(),
};
- this.timer('fetch; url: ' + pageUrl.href + '; depth: ' + depth + '; delay: ' + (this.options.delay * index) + 'ms', timerScope);
+ this.timer(`fetch; url: ${pageUrl.href}; depth: ${depth}; delay: ${this.options.delay * index}ms`, timerScope);
- return new Promise((resolve, reject) => this.sleep(this.options.delay * index).then(() => this.visit(pageUrl, timerScope, resolve, reject)));
+ return new Promise((resolve, reject) => {
+ sleep(this.options.delay * index)
+ .then(() => this.visit(pageUrl, timerScope, resolve, reject));
+ });
}
visit(pageUrl, timerScope, resolve, reject) {
const browser = new Browser({
- proxy: this.options.proxy,
+ proxy: this.options.proxy,
silent: true,
strictSSL: false,
userAgent: this.options.userAgent,
waitDuration: this.options.maxWait,
});
- browser.on('authenticate', auth => {
- auth.username = this.options.username;
- auth.password = this.options.password;
- });
+ browser.on('authenticate', (auth) => {
+ auth.username = this.options.username;
+ auth.password = this.options.password;
+ });
- this.timer('browser.visit start; url: ' + pageUrl.href, timerScope);
+ this.timer(`browser.visit start; url: ${pageUrl.href}`, timerScope);
browser.visit(pageUrl.href, () => {
- this.timer('browser.visit end; url: ' + pageUrl.href, timerScope);
+ this.timer(`browser.visit end; url: ${pageUrl.href}`, timerScope);
+
+ try {
+ if (!this.checkResponse(browser, pageUrl)) {
+ resolve();
- try {
- if (!this.checkResponse(browser, pageUrl)) {
- return resolve();
- }
- } catch(error) {
- return reject(error);
- }
+ return;
+ }
+ } catch (error) {
+ reject(error);
+
+ return;
+ }
- const headers = this.getHeaders(browser);
+ const headers = getHeaders(browser);
const html = this.getHtml(browser);
- const scripts = this.getScripts(browser);
+ const scripts = getScripts(browser);
const js = this.getJs(browser);
- const cookies = this.getCookies(browser);
+ const cookies = getCookies(browser);
this.wappalyzer.analyze(pageUrl, {
headers,
@@ -165,14 +229,14 @@ class Driver {
.then(() => {
const links = Array.prototype.reduce.call(
browser.document.getElementsByTagName('a'), (results, link) => {
- if ( link.protocol.match(/https?:/) && link.hostname === this.origPageUrl.hostname && extensions.test(link.pathname) ) {
+ if (link.protocol.match(/https?:/) && link.hostname === this.origPageUrl.hostname && extensions.test(link.pathname)) {
link.hash = '';
results.push(url.parse(link.href));
}
return results;
- }, []
+ }, [],
);
return resolve(links);
@@ -182,25 +246,26 @@ class Driver {
checkResponse(browser, pageUrl) {
// Validate response
- const resource = browser.resources.length ? browser.resources.filter(resource => resource.response).shift() : null;
+ const resource = browser.resources.length
+ ? browser.resources.filter(_resource => _resource.response).shift() : null;
- if ( !resource ) {
+ if (!resource) {
throw new Error('NO_RESPONSE');
}
- this.analyzedPageUrls[pageUrl.href].status = resource.response.status;
+ this.analyzedPageUrls[pageUrl.href].status = resource.response.status;
- if ( resource.response.status !== 200 ) {
+ if (resource.response.status !== 200) {
throw new Error('RESPONSE_NOT_OK');
}
- const headers = this.getHeaders(browser);
+ const headers = getHeaders(browser);
// Validate content type
- const contentType = headers.hasOwnProperty('content-type') ? headers['content-type'].shift() : null;
+ const contentType = headers['content-type'] ? headers['content-type'].shift() : null;
- if ( !contentType || !/\btext\/html\b/.test(contentType) ) {
- this.wappalyzer.log('Skipping; url: ' + pageUrl.href + '; content type: ' + contentType, 'driver');
+ if (!contentType || !/\btext\/html\b/.test(contentType)) {
+ this.wappalyzer.log(`Skipping; url: ${pageUrl.href}; content type: ${contentType}`, 'driver');
delete this.analyzedPageUrls[pageUrl.href];
@@ -208,80 +273,50 @@ class Driver {
}
// Validate document
- if ( !browser.document || !browser.document.documentElement ) {
+ if (!browser.document || !browser.document.documentElement) {
throw new Error('NO_HTML_DOCUMENT');
}
return true;
}
- getHeaders(browser) {
- const headers = {};
-
- const resource = browser.resources.length ? browser.resources.filter(resource => resource.response).shift() : null;
-
- if ( resource ) {
- resource.response.headers._headers.forEach(header => {
- if ( !headers[header[0]] ){
- headers[header[0]] = [];
- }
-
- headers[header[0]].push(header[1]);
- });
- }
-
- return headers;
- }
-
getHtml(browser) {
let html = '';
try {
html = browser.html()
.split('\n')
- .slice(0, this.options.htmlMaxRows / 2).concat(html.slice(html.length - this.options.htmlMaxRows / 2))
+ .slice(0, this.options.htmlMaxRows / 2)
+ .concat(html.slice(html.length - this.options.htmlMaxRows / 2))
.map(line => line.substring(0, this.options.htmlMaxCols))
.join('\n');
- } catch(error) {
+ } catch (error) {
this.wappalyzer.log(error.message, 'browser', 'error');
}
return html;
}
- getScripts(browser) {
- if ( !browser.document || !browser.document.scripts ) {
- return [];
- }
-
- const scripts = Array.prototype.slice
- .apply(browser.document.scripts)
- .filter(script => script.src)
- .map(script => script.src);
-
- return scripts;
- }
-
getJs(browser) {
const patterns = this.wappalyzer.jsPatterns;
const js = {};
- Object.keys(patterns).forEach(appName => {
+ Object.keys(patterns).forEach((appName) => {
js[appName] = {};
- Object.keys(patterns[appName]).forEach(chain => {
+ Object.keys(patterns[appName]).forEach((chain) => {
js[appName][chain] = {};
patterns[appName][chain].forEach((pattern, index) => {
const properties = chain.split('.');
- let value = properties.reduce((parent, property) => {
- return parent && parent.hasOwnProperty(property) ? parent[property] : null;
- }, browser.window);
+ let value = properties
+ .reduce((parent, property) => (parent && parent[property]
+ ? parent[property] : null), browser.window);
value = typeof value === 'string' || typeof value === 'number' ? value : !!value;
- if ( value ) {
+ if (value) {
js[appName][chain][index] = value;
}
});
@@ -291,81 +326,61 @@ class Driver {
return js;
}
- getCookies(browser) {
- const cookies = [];
-
- if ( browser.cookies ) {
- browser.cookies.forEach(cookie => cookies.push({
- name: cookie.key,
- value: cookie.value,
- domain: cookie.domain,
- path: cookie.path,
- }));
- }
-
- return cookies;
- }
-
crawl(pageUrl, index = 1, depth = 1) {
- pageUrl.canonical = pageUrl.protocol + '//' + pageUrl.host + pageUrl.pathname;
+ pageUrl.canonical = `${pageUrl.protocol}//${pageUrl.host}${pageUrl.pathname}`;
- return new Promise(resolve => {
+ return new Promise((resolve) => {
this.fetch(pageUrl, index, depth)
- .catch(error => {
- const type = error.message && errorTypes[error.message] ? error.message : 'UNKNOWN_ERROR';
- const message = error.message && errorTypes[error.message] ? errorTypes[error.message] : 'Unknown error';
-
- this.analyzedPageUrls[pageUrl.href].error = {
- type,
- message,
- };
-
- this.wappalyzer.log(`${message}; url: ${pageUrl.href}`, 'driver', 'error');
- })
- .then(links => {
- if ( links && this.options.recursive && depth < this.options.maxDepth ) {
+ .catch((error) => {
+ const type = error.message && errorTypes[error.message] ? error.message : 'UNKNOWN_ERROR';
+ const message = error.message && errorTypes[error.message] ? errorTypes[error.message] : 'Unknown error';
+
+ this.analyzedPageUrls[pageUrl.href].error = {
+ type,
+ message,
+ };
+
+ this.wappalyzer.log(`${message}; url: ${pageUrl.href}`, 'driver', 'error');
+ })
+ .then((links) => {
+ if (links && this.options.recursive && depth < this.options.maxDepth) {
return this.chunk(links.slice(0, this.options.maxUrls), depth + 1);
- } else {
- return Promise.resolve();
}
+ return Promise.resolve();
})
.then(() => {
resolve({
urls: this.analyzedPageUrls,
applications: this.apps,
- meta: this.meta
+ meta: this.meta,
});
});
});
}
chunk(links, depth, chunk = 0) {
- if ( links.length === 0 ) {
+ if (links.length === 0) {
return Promise.resolve();
}
const chunked = links.splice(0, this.options.chunkSize);
- return new Promise(resolve => {
+ return new Promise((resolve) => {
Promise.all(chunked.map((link, index) => this.crawl(link, index, depth)))
.then(() => this.chunk(links, depth, chunk + 1))
.then(() => resolve());
});
}
- sleep(ms) {
- return ms ? new Promise(resolve => setTimeout(resolve, ms)) : Promise.resolve();
- }
-
timer(message, scope) {
const time = new Date().getTime();
- const sinceStart = ( Math.round(( time - this.time.start ) / 10) / 100) + 's';
- const sinceLast = ( Math.round(( time - scope.last ) / 10) / 100) + 's';
+ const sinceStart = `${Math.round((time - this.time.start) / 10) / 100}s`;
+ const sinceLast = `${Math.round((time - scope.last) / 10) / 100}s`;
- this.wappalyzer.log('[timer] ' + message + '; lapsed: ' + sinceLast + ' / ' + sinceStart, 'driver');
+ this.wappalyzer.log(`[timer] ${message}; lapsed: ${sinceLast} / ${sinceStart}`, 'driver');
scope.last = time;
}
-};
+}
module.exports = Driver;
diff --git a/src/drivers/npm/index.js b/src/drivers/npm/index.js
index a638e2aea..cdff2ba3d 100755
--- a/src/drivers/npm/index.js
+++ b/src/drivers/npm/index.js
@@ -1,5 +1,5 @@
#!/usr/bin/env node
-'use strict';
+
const Wappalyzer = require('./driver');
@@ -7,36 +7,39 @@ const args = process.argv.slice(2);
const url = args.shift() || '';
-if ( !url ) {
+if (!url) {
process.stderr.write('No URL specified\n');
process.exit(1);
}
-let options = {};
+const options = {};
+
let arg;
-while ( arg = args.shift() ) {
- let matches = /--([^=]+)=(.+)/.exec(arg);
+do {
+ arg = args.shift();
- if ( matches ) {
- let key = matches[1].replace(/-\w/g, matches => matches[1].toUpperCase());
- let value = matches[2];
+ const matches = /--([^=]+)=(.+)/.exec(arg);
+
+ if (matches) {
+ const key = matches[1].replace(/-\w/g, _matches => _matches[1].toUpperCase());
+ const value = matches[2];
options[key] = value;
}
-}
+} while (arg);
const wappalyzer = new Wappalyzer(url, options);
wappalyzer.analyze()
- .then(json => {
- process.stdout.write(JSON.stringify(json) + '\n')
+ .then((json) => {
+ process.stdout.write(`${JSON.stringify(json)}\n`);
process.exit(0);
})
- .catch(error => {
- process.stderr.write(error + '\n')
+ .catch((error) => {
+ process.stderr.write(`${error}\n`);
process.exit(1);
});
diff --git a/src/drivers/webextension/html/background.html b/src/drivers/webextension/html/background.html
index 1c31887fc..8a03c7a9d 100644
--- a/src/drivers/webextension/html/background.html
+++ b/src/drivers/webextension/html/background.html
@@ -7,7 +7,7 @@
-
+
diff --git a/src/drivers/webextension/html/popup.html b/src/drivers/webextension/html/popup.html
index 9fc0278ad..b453b70c3 100644
--- a/src/drivers/webextension/html/popup.html
+++ b/src/drivers/webextension/html/popup.html
@@ -7,7 +7,7 @@
-
+
diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js
index be07c97c0..4d29c981a 100644
--- a/src/drivers/webextension/js/content.js
+++ b/src/drivers/webextension/js/content.js
@@ -1,12 +1,12 @@
/** global: browser */
/** global: XMLSerializer */
-if ( typeof browser !== 'undefined' && typeof document.body !== 'undefined' ) {
+if (typeof browser !== 'undefined' && typeof document.body !== 'undefined') {
try {
sendMessage('init', {});
// HTML
- var html = new XMLSerializer().serializeToString(document).split('\n');
+ let html = new XMLSerializer().serializeToString(document).split('\n');
html = html
.slice(0, 1000).concat(html.slice(html.length - 1000))
@@ -26,7 +26,7 @@ if ( typeof browser !== 'undefined' && typeof document.body !== 'undefined' ) {
const script = document.createElement('script');
script.onload = () => {
- const onMessage = event => {
+ const onMessage = (event) => {
if (event.data.id !== 'js') {
return;
}
@@ -40,11 +40,11 @@ if ( typeof browser !== 'undefined' && typeof document.body !== 'undefined' ) {
addEventListener('message', onMessage);
- sendMessage('get_js_patterns', {}, response => {
+ sendMessage('get_js_patterns', {}, (response) => {
if (response) {
postMessage({
id: 'patterns',
- patterns: response.patterns
+ patterns: response.patterns,
}, '*');
}
});
@@ -62,8 +62,8 @@ function sendMessage(id, subject, callback) {
(chrome || browser).runtime.sendMessage({
id,
subject,
- source: 'content.js'
- }, callback || ( () => {} ));
+ source: 'content.js',
+ }, callback || (() => {}));
}
// https://stackoverflow.com/a/44774834
diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js
index 988d53d12..6f339da05 100644
--- a/src/drivers/webextension/js/driver.js
+++ b/src/drivers/webextension/js/driver.js
@@ -2,17 +2,22 @@
* WebExtension driver
*/
+/* eslint-env browser */
+/* global browser, chrome, fetch, Wappalyzer */
+
/** global: browser */
+/** global: chrome */
+/** global: fetch */
/** global: Wappalyzer */
const wappalyzer = new Wappalyzer();
-var tabCache = {};
-var categoryOrder = [];
-var options = {};
-var robotsTxtQueue = {};
+const tabCache = {};
+let categoryOrder = [];
+const options = {};
+const robotsTxtQueue = {};
-browser.tabs.onRemoved.addListener(tabId => {
+browser.tabs.onRemoved.addListener((tabId) => {
tabCache[tabId] = null;
});
@@ -21,19 +26,19 @@ browser.tabs.onRemoved.addListener(tabId => {
*/
function getOption(name, defaultValue = null) {
return new Promise((resolve, reject) => {
- const callback = item => {
- options[name] = item.hasOwnProperty(name) ? item[name] : defaultValue;
+ const callback = (item) => {
+ options[name] = item[name] ? item[name] : defaultValue;
resolve(options[name]);
};
browser.storage.local.get(name)
.then(callback)
- .catch(error => {
- wappalyzer.log(error, 'driver', 'error')
+ .catch((error) => {
+ wappalyzer.log(error, 'driver', 'error');
reject();
- });
+ });
});
}
@@ -56,7 +61,7 @@ function setOption(name, value) {
function openTab(args) {
browser.tabs.create({
url: args.url,
- active: args.background === undefined || !args.background
+ active: args.background === undefined || !args.background,
});
}
@@ -66,7 +71,7 @@ function openTab(args) {
function post(url, body) {
fetch(url, {
method: 'POST',
- body: JSON.stringify(body)
+ body: JSON.stringify(body),
})
.then(response => wappalyzer.log(`POST ${url}: ${response.status}`, 'driver'))
.catch(error => wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error'));
@@ -74,7 +79,7 @@ function post(url, body) {
fetch('../apps.json')
.then(response => response.json())
- .then(json => {
+ .then((json) => {
wappalyzer.apps = json.apps;
wappalyzer.categories = json.categories;
@@ -87,21 +92,21 @@ fetch('../apps.json')
.catch(error => wappalyzer.log(`GET apps.json: ${error}`, 'driver', 'error'));
// Version check
-let version = browser.runtime.getManifest().version;
+const { version } = browser.runtime.getManifest();
getOption('version')
- .then(previousVersion => {
+ .then((previousVersion) => {
if (previousVersion === null) {
openTab({
- url: wappalyzer.config.websiteURL + 'installed'
+ url: `${wappalyzer.config.websiteURL}installed`,
});
} else if (version !== previousVersion) {
getOption('upgradeMessage', true)
- .then(upgradeMessage => {
+ .then((upgradeMessage) => {
if (upgradeMessage) {
openTab({
- url: wappalyzer.config.websiteURL + 'upgraded?v' + version,
- background: true
+ url: `${wappalyzer.config.websiteURL}upgraded?v${version}`,
+ background: true,
});
}
});
@@ -113,32 +118,37 @@ getOption('version')
getOption('dynamicIcon', true);
getOption('pinnedCategory');
-getOption('hostnameCache', {}).then(hostnameCache => wappalyzer.hostnameCache = hostnameCache);
+getOption('hostnameCache', {})
+ .then((hostnameCache) => {
+ wappalyzer.hostnameCache = hostnameCache;
+
+ return hostnameCache;
+ });
// Run content script on all tabs
-browser.tabs.query({ url: [ 'http://*/*', 'https://*/*' ] })
- .then(tabs => {
- tabs.forEach(tab => {
+browser.tabs.query({ url: ['http://*/*', 'https://*/*'] })
+ .then((tabs) => {
+ tabs.forEach((tab) => {
browser.tabs.executeScript(tab.id, {
- file: '../js/content.js'
+ file: '../js/content.js',
});
- })
+ });
})
.catch(error => wappalyzer.log(error, 'driver', 'error'));
// Capture response headers
-browser.webRequest.onCompleted.addListener(request => {
+browser.webRequest.onCompleted.addListener((request) => {
const headers = {};
if (request.responseHeaders) {
const url = wappalyzer.parseUrl(request.url);
- browser.tabs.query({ url: [ url.href ] })
- .then(tabs => {
+ browser.tabs.query({ url: [url.href] })
+ .then((tabs) => {
const tab = tabs[0] || null;
if (tab) {
- request.responseHeaders.forEach(header => {
+ request.responseHeaders.forEach((header) => {
const name = header.name.toLowerCase();
headers[name] = headers[name] || [];
@@ -153,32 +163,32 @@ browser.webRequest.onCompleted.addListener(request => {
})
.catch(error => wappalyzer.log(error, 'driver', 'error'));
}
-}, { urls: [ 'http://*/*', 'https://*/*' ], types: [ 'main_frame' ] }, [ 'responseHeaders' ]);
+}, { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, ['responseHeaders']);
// Listen for messages
(chrome || browser).runtime.onMessage.addListener((message, sender, sendResponse) => {
- if (typeof message.id != 'undefined') {
+ if (typeof message.id !== 'undefined') {
if (message.id !== 'log') {
- wappalyzer.log('Message' + (message.source ? ' from ' + message.source : '') + ': ' + message.id, 'driver');
+ wappalyzer.log(`Message${message.source ? ` from ${message.source}` : ''}: ${message.id}`, 'driver');
}
- let url = wappalyzer.parseUrl(sender.tab ? sender.tab.url : '');
+ const url = wappalyzer.parseUrl(sender.tab ? sender.tab.url : '');
let response;
- switch ( message.id ) {
+ switch (message.id) {
case 'log':
wappalyzer.log(message.subject, message.source);
break;
case 'init':
- browser.cookies.getAll({ domain: '.' + url.hostname })
+ browser.cookies.getAll({ domain: `.${url.hostname}` })
.then(cookies => wappalyzer.analyze(url, { cookies }, { tab: sender.tab }));
break;
case 'analyze':
wappalyzer.analyze(url, message.subject, { tab: sender.tab });
- setOption('hostnameCache', wappalyzer.hostnameCache);
+ setOption('hostnameCache', wappalyzer.hostnameCache);
break;
case 'ad_log':
@@ -200,7 +210,7 @@ browser.webRequest.onCompleted.addListener(request => {
break;
case 'get_js_patterns':
response = {
- patterns: wappalyzer.jsPatterns
+ patterns: wappalyzer.jsPatterns,
};
break;
@@ -226,14 +236,14 @@ wappalyzer.driver.log = (message, source, type) => {
* Display apps
*/
wappalyzer.driver.displayApps = (detected, meta, context) => {
- let tab = context.tab;
+ const { tab } = context;
if (tab === undefined) {
return;
}
tabCache[tab.id] = tabCache[tab.id] || {
- detected: []
+ detected: [],
};
tabCache[tab.id].detected = detected;
@@ -241,11 +251,11 @@ wappalyzer.driver.displayApps = (detected, meta, context) => {
let found = false;
// Find the main application to display
- [ options.pinnedCategory ].concat(categoryOrder).forEach(match => {
- Object.keys(detected).forEach(appName => {
- let app = detected[appName];
+ [options.pinnedCategory].concat(categoryOrder).forEach((match) => {
+ Object.keys(detected).forEach((appName) => {
+ const app = detected[appName];
- app.props.cats.forEach(category => {
+ app.props.cats.forEach((category) => {
if (category === match && !found) {
let icon = app.props.icon || 'default.svg';
@@ -254,15 +264,15 @@ wappalyzer.driver.displayApps = (detected, meta, context) => {
}
if (/\.svg$/i.test(icon)) {
- icon = 'converted/' + icon.replace(/\.svg$/, '.png');
+ icon = `converted/${icon.replace(/\.svg$/, '.png')}`;
}
try {
browser.pageAction.setIcon({
tabId: tab.id,
- path: '../images/icons/' + icon
+ path: `../images/icons/${icon}`,
});
- } catch(e) {
+ } catch (e) {
// Firefox for Android does not support setIcon see https://bugzilla.mozilla.org/show_bug.cgi?id=1331746
}
@@ -284,35 +294,39 @@ wappalyzer.driver.displayApps = (detected, meta, context) => {
* Fetch and cache robots.txt for host
*/
wappalyzer.driver.getRobotsTxt = (host, secure = false) => {
- if (robotsTxtQueue.hasOwnProperty(host)) {
+ if (robotsTxtQueue[host]) {
return robotsTxtQueue[host];
}
- robotsTxtQueue[host] = new Promise(resolve => {
+ robotsTxtQueue[host] = new Promise((resolve) => {
getOption('tracking', true)
- .then(tracking => {
+ .then((tracking) => {
if (!tracking) {
- return resolve([]);
+ resolve([]);
+
+ return;
}
getOption('robotsTxtCache')
- .then(robotsTxtCache => {
+ .then((robotsTxtCache) => {
robotsTxtCache = robotsTxtCache || {};
- if ( host in robotsTxtCache ) {
- return resolve(robotsTxtCache[host]);
+ if (host in robotsTxtCache) {
+ resolve(robotsTxtCache[host]);
+
+ return;
}
const timeout = setTimeout(() => resolve([]), 3000);
- fetch('http' + (secure ? 's' : '') + '://' + host + '/robots.txt', { redirect: 'follow' })
- .then(response => {
+ fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { redirect: 'follow' })
+ .then((response) => {
clearTimeout(timeout);
return response.ok ? response.text() : '';
})
- .then(robotsTxt => {
- robotsTxtCache[host] = wappalyzer.parseRobotsTxt(robotsTxt);
+ .then((robotsTxt) => {
+ robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt);
setOption('robotsTxtCache', robotsTxtCache);
@@ -322,7 +336,7 @@ wappalyzer.driver.getRobotsTxt = (host, secure = false) => {
});
});
})
- .finally(() => delete robotsTxtQueue[host]);
+ .finally(() => delete robotsTxtQueue[host]);
return robotsTxtQueue[host];
};
@@ -332,7 +346,7 @@ wappalyzer.driver.getRobotsTxt = (host, secure = false) => {
*/
wappalyzer.driver.ping = (hostnameCache = {}, adCache = []) => {
getOption('tracking', true)
- .then(tracking => {
+ .then((tracking) => {
if (tracking) {
if (Object.keys(hostnameCache).length) {
post('https://api.wappalyzer.com/ping/v1/', hostnameCache);
diff --git a/src/drivers/webextension/js/iframe.js b/src/drivers/webextension/js/iframe.js
deleted file mode 100644
index 883ebea71..000000000
--- a/src/drivers/webextension/js/iframe.js
+++ /dev/null
@@ -1,1185 +0,0 @@
-'use strict';
-
-(function(win) {
-
-var exports = {};
-
-(function(exports) {
-
- var utils = {
- normalizeUrl: function(url) {
-
- return this.hashUrl(url) || null;
-
- },
-
- getReferrer: function() {
- return this.normalizeUrl(document.referrer);
- },
-
- getPageUrl: function() {
- return this.normalizeUrl(window.location.href);
- },
- hashUrl: function(url) {
- var a,
- result;
-
- if ( !url || url.indexOf('http') !== 0 ) {
- return null;
- }
-
- a = document.createElement('a');
- a.href = url;
-
- result = a.protocol + '//' + a.hostname + '/';
-
- if ( a.pathname && a.pathname !== '/' ) {
- result += this.hashCode(a.pathname);
- }
-
- if ( a.search ) {
- result += '?' + this.hashCode(a.search);
- }
-
- if ( a.hash ) {
- result += '#' + this.hashCode(a.hash);
- }
-
- return result;
- },
-
- hashCode: function(str) {
- var hash = 0,
- kar,
- i;
-
- if ( str.length === 0 ) {
- return hash;
- }
-
- for ( i = 0; i < str.length; i++ ) {
- kar = str.charCodeAt(i);
- hash = ((hash << 5) - hash) + kar;
- hash = hash & hash;
- }
-
- return hash + Math.pow(2, 32);
- },
-
- realArray: function(a) {
- return Array.prototype.slice.apply(a);
- },
-
- onDocLoaded: function(doc, callback) {
- if ( doc.readyState === 'loading' ) {
- doc.addEventListener('DOMContentLoaded', callback);
- } else {
- callback();
- }
- },
-
- SCRIPT_IN_WINDOW_TOP: window === window.top,
-
- isFriendlyWindow: function(win) {
-
- var href;
- try {
- href = win.location.href;
- } catch(e) {
- return false;
- }
- return true;
- },
-
- elementWindow: function(el) {
- return el.ownerDocument.defaultView;
- },
-
- viewport: function(win) {
- return {width: win.innerWidth, height: win.innerHeight};
- },
-
- parseQS: function(qs) {
- if ( qs.indexOf('http') === 0 ) {
- qs = qs.split('?')[1];
- }
- var i, kvs, key, val;
- var dict = {};
- qs = qs.split('&');
- for ( i = 0; i < qs.length; i++ ) {
- kvs = qs[i].split('=');
- key = kvs[0];
- val = kvs.slice(1).join('=');
- try {
- dict[key] = window.decodeURIComponent(val);
- } catch (e) {
-
- continue;
- }
- }
- return dict;
- },
- sendToBackground: function(message, event, responseMessage, onResponse) {
- if ( typeof browser !== 'undefined' ) {
- var response = browser.runtime.sendMessage(message);
- response.then(onResponse);
- } else if ( typeof chrome !== 'undefined' ) {
- chrome.runtime.sendMessage(message, onResponse);
- } else if ( window.self.port ) {
- window.self.port.on(responseMessage, onResponse);
- window.self.port.emit(event, message);
- }
- },
-
- askIfTrackingEnabled: function(callback, elseCallback) {
-
- this.sendToBackground(
- 'is_tracking_enabled',
- '',
- 'tracking_enabled_response',
- function(message) {
- if ( message && message.tracking_enabled ) {
-
- callback();
- } else {
-
- elseCallback();
- }
- }
- );
-
- }
- };
-
- utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent);
- utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME;
-
- function LogGenerator() {
- this.msgNum = 0;
- this.pageMeta = {
- 'url': utils.getPageUrl(),
- 'isHP': window.location.pathname === '/',
- 'referrer': utils.getReferrer(),
- 'rand': Math.floor(Math.random() * 10e12),
- 'startTime': new Date().getTime()
- };
- }
-
- LogGenerator.prototype = {
- log: function(event, opt_assets, opt_pageTags) {
- var opt_video_assets;
- if ( event === 'video' || event === 'invalid-video' ) {
- opt_video_assets = opt_assets || [];
- opt_assets = [];
- } else {
- opt_video_assets = [];
- opt_assets = opt_assets || [];
- }
- var result = {
- doc: this.pageMeta,
- event: event,
- video_assets: opt_video_assets,
- assets: opt_assets,
- version: '3',
- mrev: '4aeaa5a-c',
- msgNum: this.msgNum,
- timestamp: new Date().getTime(),
- pageVis: document.visibilityState,
- pageFoc: document.hasFocus(),
- pageTags: opt_pageTags || []
- };
- this.msgNum++;
- return result;
- }
- };
-
- utils.LogGenerator = LogGenerator;
-
- exports.utils = utils;
-})(exports);
-
-(function(exports) {
-
- var SizeMatcher = {
- VALID_AD_SIZES: [
- [300, 50],
- [320, 50],
- [160, 600],
- [300, 250],
- [300, 600],
- [300, 1050],
- [336, 280],
- [336, 850],
- [468, 60],
- [728, 90],
- [728, 250],
- [728, 270],
- [970, 66],
- [970, 90],
- [970, 125],
- [970, 250],
- [970, 400],
- [970, 415],
- [1280, 100]
- ],
-
- PX_SIZE_TOL: 10,
-
- getMatchedAdSize: function(width, height) {
-
- if ( !this.set ) {
- this.set = this._makeSizeSet();
- }
-
- return this.set[Math.round(width) + 'x' + Math.round(height)];
- },
-
- elementIsAdShaped: function(el) {
- return !!this.getMatchedAdSizeForElement(el);
- },
-
- getMatchedAdSizeForElement: function(el) {
- var rect = el.getBoundingClientRect();
- return this.getMatchedAdSize(rect.width, rect.height);
- },
-
- _makeSizeSet: function() {
- var set = {};
- var i;
- var xfuz;
- var yfuz;
- var size;
- var width;
- var height;
-
- for ( i = 0; i < this.VALID_AD_SIZES.length; i++ ) {
- for ( xfuz = -this.PX_SIZE_TOL; xfuz <= this.PX_SIZE_TOL; xfuz++ ) {
- for ( yfuz = -this.PX_SIZE_TOL; yfuz <= this.PX_SIZE_TOL; yfuz++ ) {
- size = this.VALID_AD_SIZES[i];
- width = size[0] + xfuz;
- height = size[1] + yfuz;
- set[width + 'x' + height] = size;
- }
- }
- }
- return set;
- }
- };
-
- var Throttler = {
- MAX_SEARCHES_PER_WINDOW: 10,
- MAX_SEARCHES_PER_ELEMENT: 2,
-
- countSearch: function(el) {
- if ( typeof el.searches !== 'number' ) {
- el.searches = 0;
- }
-
- el.searches += 1;
- },
-
- throttle: function(el, max) {
- if ( typeof el.searches === 'number' && el.searches >= max ) {
- return true;
- }
- return false;
- },
-
- throttleElement: function(el) {
- return this.throttle(el, this.MAX_SEARCHES_PER_ELEMENT);
- },
-
- throttleWin: function(win) {
- return this.throttle(win, this.MAX_SEARCHES_PER_WINDOW);
- },
-
- getCount: function(el) {
- return el.searches || 0;
- }
- };
-
- function TopSearcher(win) {
- this.win = win;
- this.doc = win.document;
- }
-
- TopSearcher.prototype.search = function() {
- var candidates = exports.utils.realArray(this.doc.querySelectorAll('img, object, embed')),
- html5Ad,
- ads = [];
-
- ads = ads.concat(candidates.filter(function(el) {
- if ( !el.mpAdFound && !Throttler.throttleElement(el) ) {
- Throttler.countSearch(el);
- if ( (el.tagName !== 'IMG' || isStandardImage(el)) && SizeMatcher.elementIsAdShaped(el) ) {
- el.mpAdFound = true;
- return true;
- }
- }
- return false;
- }));
-
- html5Ad = this._mainGetHTMLAd();
- if ( html5Ad ) {
- html5Ad.html5 = true;
- html5Ad.mpAdFound = true;
- ads.push(html5Ad);
- }
-
- return ads;
- };
-
- TopSearcher.prototype._mainGetHTMLAd = function() {
- var styles = this.doc.querySelectorAll('div > style, div > link[rel="stylesheet"]'),
- i, div;
- for ( i = 0; i < styles.length; i++ ) {
- div = styles[i].parentNode;
- if ( !div.mpAdFound && SizeMatcher.elementIsAdShaped(div) && this._jumpedOut(div) ) {
- return div;
- }
- }
- };
-
- TopSearcher.prototype._jumpedOut = function(el) {
- var siblings, ifrs;
- siblings = exports.utils.realArray(el.parentNode.children);
- ifrs = siblings.filter(function(el) {
- return el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0;
- });
- return ifrs.length > 0;
- };
-
- function IframeSearcher(win) {
- this.MIN_AD_AREA = 14000;
- this.MIN_WINDOW_PX = 10;
-
- this.win = win;
- this.doc = win.document;
- this.body = win.document.body;
- this.winClickTag = win.clickTag;
- this.adSizeMeta = this._getAdSizeMeta();
- this.numElementsInBody = (this.body && this.body.querySelectorAll('*').length) || 0;
-
- this.shouldSearchWindow = false;
- if ( !this.win.mpAdFound && this.body && !Throttler.throttleWin(this.win) ) {
- this.winWidth = this.win.innerWidth;
- this.winHeight = this.win.innerHeight;
- if ( this._meetsMinAdSize(this.winWidth, this.winHeight) && !this._containsLargeIframes() ) {
- this.shouldSearchWindow = true;
- }
- }
-
- }
-
- IframeSearcher.prototype.search = function() {
- var ad;
-
- if ( this.shouldSearchWindow ) {
- ad = this._search();
- if ( ad ) {
- ad.mpAdFound = true;
- win.mpAdFound = true;
- return ad;
- }
- Throttler.countSearch(this.win);
- }
-
- return null;
- };
-
- IframeSearcher.prototype._search = function() {
- var _this = this,
- stdCandidates,
- html5Candidates,
- stdEl,
- html5El;
-
- stdCandidates = this.body.querySelectorAll('img, object, embed');
-
- stdEl = getFirst(stdCandidates, function(el) {
- if ( !el.mpAdFound &&
- !Throttler.throttleElement(el) &&
- (el.tagName !== 'IMG' || isStandardImage(el)) &&
- _this._elementIsAtLeastAsBigAsWindow(el))
- {
- return true;
- }
- Throttler.countSearch(el);
- return false;
- });
-
- if ( stdEl ) {
- return stdEl;
- }
-
- if ( this._isHTML5Iframe() ) {
- html5Candidates = this.doc.querySelectorAll('body, canvas, button, video, svg, div');
- html5El = getFirst(html5Candidates, function(el) {
-
- if ( _this._elementIsAtLeastAsBigAsWindow(el) ) {
- return true;
- }
- Throttler.countSearch(el);
- return false;
- });
- }
-
- if ( html5El ) {
- html5El.html5 = true;
- html5El.winClickTag = this.winClickTag;
- html5El.adSizeMeta = this.adSizeMeta;
- return html5El;
- }
-
- return null;
- };
-
- IframeSearcher.prototype._isHTML5Iframe = function() {
- if ( this.winClickTag || this.adSizeMeta ) {
- return true;
- }
-
- if ( this.doc.querySelectorAll('canvas', 'button', 'video', 'svg').length > 0 ) {
- return true;
- }
-
- if ( this.numElementsInBody >= 5 && Throttler.getCount(this.win) > 0 && this.doc.querySelectorAll('div').length > 0 ) {
- return true;
- }
-
- return false;
- };
-
- IframeSearcher.prototype._elementIsAtLeastAsBigAsWindow = function(el) {
- var rect = el.getBoundingClientRect(),
- tol = 0.95;
-
- return rect.width >= (tol * this.winWidth) && rect.height >= (tol * this.winHeight);
- };
-
- IframeSearcher.prototype._meetsMinAdSize = function(width, height) {
- return (width * height) >= this.MIN_AD_AREA;
- };
-
- IframeSearcher.prototype._containsLargeIframes = function() {
- var iframes = this.doc.querySelectorAll('iframe');
- var rect;
- var i;
- for ( i = 0; i < iframes.length; i++ ) {
- rect = iframes[i].getBoundingClientRect();
- if ( rect.width > this.MIN_WINDOW_PX || rect.height > this.MIN_WINDOW_PX ) {
- return true;
- }
- }
- return false;
- };
-
- IframeSearcher.prototype._getAdSizeMeta = function() {
- var adSizeMeta = this.doc.querySelectorAll('meta[name="ad.size"]');
- if ( adSizeMeta.length > 0 ) {
- return adSizeMeta[0].content;
- } else {
- return null;
- }
- };
-
- function getFirst(arr, testFn) {
- var i, el;
- for ( i = 0; i < arr.length; i++ ) {
- el = arr[i];
- if ( testFn(el) ) {
- return el;
- }
- }
- return null;
- }
-
- function isStandardImage(img) {
-
- return img.src && (img.parentNode.tagName === 'A' || img.getAttribute('onclick'));
- }
-
- function getFriendlyIframes(win) {
- var iframes = win.document.querySelectorAll('iframe');
- iframes = exports.utils.realArray(iframes);
- var friendlyIframes = iframes.filter(function(ifr) {
- return exports.utils.isFriendlyWindow(ifr.contentWindow);
- });
- return friendlyIframes;
- }
-
- function findAds(win) {
- var i,
- iframes,
- searcher,
- ad,
- ads = [];
-
- if ( win === win.top ) {
- searcher = new TopSearcher(win);
- ads = ads.concat(searcher.search());
- } else {
- searcher = new IframeSearcher(win);
- ad = searcher.search();
- if ( ad ) {
- ads.push(ad);
- }
- }
-
- iframes = getFriendlyIframes(win);
- for ( i = 0; i < iframes.length; i++ ) {
- ads = ads.concat(findAds(iframes[i].contentWindow));
- }
-
- return ads;
- }
-
- exports.adfinder = {
- getMatchedAdSize: SizeMatcher.getMatchedAdSize.bind(SizeMatcher),
- findAds: findAds
- };
-})(exports);
-
-(function(exports) {
-
- var parser = {
- TAGS_WITH_SRC_ATTR: {
- 'IMG': true,
- 'SCRIPT': true,
- 'IFRAME': true,
- 'EMBED': true
- },
-
- MAX_ATTR_LEN: 100,
-
- getUrl: function(el, params) {
- var url;
-
- if ( this.TAGS_WITH_SRC_ATTR.hasOwnProperty(el.tagName) ) {
- url = el.src;
-
- } else if ( el.tagName === 'OBJECT' ) {
- url = el.data || (params && params.movie) || null;
-
- } else if ( el.tagName === 'A' ) {
- url = el.href;
- }
-
- if ( url && url.indexOf('http') === 0 ) {
- return url;
- } else {
- return null;
- }
- },
-
- getParams: function(el) {
- if ( el.tagName !== 'OBJECT' ) {
- return null;
- }
-
- var i, child;
- var params = {};
- var children = el.children;
- for ( i = 0; i < children.length; i++ ) {
- child = children[i];
- if ( child.tagName === 'PARAM' && child.name ) {
-
- params[child.name.toLowerCase()] = child.value;
- }
- }
- return params;
- },
-
- getPosition: function(el) {
- var rect = el.getBoundingClientRect();
- var win = exports.utils.elementWindow(el);
-
- return {
- width: Math.round(rect.width),
- height: Math.round(rect.height),
- left: Math.round(rect.left + win.pageXOffset),
- top: Math.round(rect.top + win.pageYOffset)
- };
- },
-
- getFlashvars: function(el, params, url) {
- var flashvars;
- var urlQS = url && url.split('?')[1];
-
- if ( el.tagName === 'EMBED' ) {
- flashvars = el.getAttribute('flashvars') || urlQS;
-
- } else if ( el.tagName === 'OBJECT' ) {
- flashvars = params.flashvars || el.getAttribute('flashvars') || urlQS;
- }
-
- return (flashvars && exports.utils.parseQS(flashvars)) || null;
- },
-
- findClickThru: function(el, flashvars) {
- var key;
- if ( el.tagName === 'IMG' && el.parentElement.tagName === 'A' ) {
- return el.parentElement.href;
- } else if ( flashvars ) {
- for ( key in flashvars ) {
- if ( flashvars.hasOwnProperty(key) ) {
-
- if ( key.toLowerCase().indexOf('clicktag') === 0 ) {
- return flashvars[key];
- }
- }
- }
- }
- return null;
- },
-
- getAttr: function(el, name) {
- var val = el.getAttribute(name);
-
- if ( val && val.slice && val.toString ) {
-
- return val.slice(0, this.MAX_ATTR_LEN).toString();
- } else {
- return null;
- }
- },
-
- putPropIfExists: function(obj, name, val) {
- if ( val ) {
- obj[name] = val;
- }
- },
-
- putAttrIfExists: function(obj, el, name) {
- var val = this.getAttr(el, name);
- this.putPropIfExists(obj, name, val);
- },
-
- elementToJSON: function(el, opt_findClickThru) {
- var pos = this.getPosition(el);
- var params = this.getParams(el);
- var url = this.getUrl(el, params);
- var flashvars = this.getFlashvars(el, params, url);
- var clickThru = opt_findClickThru && this.findClickThru(el, flashvars);
- var json = {
- tagName: el.tagName,
- width: pos.width,
- height: pos.height,
- left: pos.left,
- top: pos.top,
- children: []
- };
-
- if ( params ) {
-
- delete params.flashvars;
- }
-
- this.putAttrIfExists(json, el, 'id');
- this.putAttrIfExists(json, el, 'class');
- this.putAttrIfExists(json, el, 'name');
-
- this.putPropIfExists(json, 'flashvars', flashvars);
- this.putPropIfExists(json, 'url', url);
- this.putPropIfExists(json, 'params', params);
- this.putPropIfExists(json, 'clickThru', clickThru);
-
- return json;
- }
- };
-
- exports.parser = { elementToJSON: parser.elementToJSON.bind(parser) };
-})(exports);
-
-(function(exports) {
-
- var ContextManager = function(adData) {
- this.adData = adData;
- };
-
- ContextManager.prototype = {
- CONTAINER_SIZE_TOL: 0.4,
- ASPECT_RATIO_FOR_LEADERBOARDS: 2,
-
- isValidContainer: function(el, opt_curWin) {
-
- var cWidth = el.clientWidth;
- var cHeight = el.clientHeight;
-
- var adWidth = this.adData.width;
- var adHeight = this.adData.height;
-
- var winWidth = opt_curWin && opt_curWin.innerWidth;
- var winHeight = opt_curWin && opt_curWin.innerHeight;
- var similarWin = opt_curWin && this.withinTol(adWidth, winWidth) && this.withinTol(adHeight, winHeight);
-
- var similarSizeX = this.withinTol(adWidth, cWidth);
- var similarSizeY = this.withinTol(adHeight, cHeight);
- var adAspect = adWidth / adHeight;
-
- return similarWin || el.tagName === 'A' || (adAspect >= this.ASPECT_RATIO_FOR_LEADERBOARDS && similarSizeY) || (similarSizeX && similarSizeY);
- },
-
- withinTol: function(adlen, conlen) {
- var pct = (conlen - adlen) / adlen;
-
- return pct <= this.CONTAINER_SIZE_TOL;
- },
-
- serializeElements: function(el) {
- if ( !el ) {
- return;
- }
- var i;
- var ifrWin;
- var adId = this.adData.adId;
- var elIsAd = false;
-
- if ( adId && el[adId] && el[adId].isAd === true ) {
- elIsAd = true;
- }
-
- var json = exports.parser.elementToJSON(el, elIsAd);
- var childJSON;
-
- if ( elIsAd ) {
- json.adId = adId;
- this.adData.element = {};
-
- var keys = Object.keys(json);
- for ( i = 0; i < keys.length; i++ ) {
- var key = keys[i];
- if ( key !== 'children' && key !== 'contents' ) {
- this.adData.element[key] = json[key];
- }
- }
- }
-
- var children = exports.utils.realArray(el.children).filter(function(el) {
- var param = el.tagName === 'PARAM';
- var inlineScript = el.tagName === 'SCRIPT' && !(el.src && el.src.indexOf('http') >= 0);
- var noScript = el.tagName === 'NOSCRIPT';
- return !(param || inlineScript || noScript);
- });
-
- for ( i = 0; i < children.length; i++ ) {
- childJSON = this.serializeElements(children[i]);
- if ( childJSON ) {
- json.children.push(childJSON);
- }
- }
-
- if ( el.tagName === 'IFRAME' ) {
- ifrWin = el.contentWindow;
-
- if ( adId && el[adId] && el[adId].needsWindow ) {
-
- json.contents = this.adData.serializedIframeContents;
- el[adId].needsWindow = false;
- delete this.adData.serializedIframeContents;
-
- } else if ( exports.utils.isFriendlyWindow(ifrWin) ) {
-
- childJSON = this.serializeElements(ifrWin.document.documentElement);
- if ( childJSON ) {
- json.contents = childJSON;
- }
- }
- }
-
- if ( json.children.length > 0 || json.adId || json.tagName === 'IFRAME' || json.url ) {
- return json;
- } else {
- return null;
- }
- },
-
- captureHTML: function(containerEl) {
- this.adData.context = this.serializeElements(containerEl);
- },
-
- nodeCount: function(el) {
- return el.getElementsByTagName('*').length + 1;
- },
-
- highestContainer: function(curWin, referenceElement) {
- var curContainer = referenceElement;
- var docEl = curWin.document.documentElement;
- var parentContainer;
-
- if ( curWin !== curWin.top && this.isValidContainer(docEl, curWin) ) {
- return docEl;
- }
-
- while ( true ) {
- parentContainer = curContainer.parentElement;
- if ( parentContainer && this.isValidContainer(parentContainer) ) {
- curContainer = parentContainer;
- } else {
- return curContainer;
- }
- }
- }
- };
-
- var tagfinder = {
-
- setPositions: function(adData, opt_el, opt_winPos) {
- var el = opt_el || adData.context;
- var winPos = opt_winPos || {left: 0, top: 0};
- var ifrPos;
-
- el.left += winPos.left;
- el.top += winPos.top;
-
- if ( el.children ) {
- el.children.forEach(function(child) {
- this.setPositions(adData, child, winPos);
- }, this);
- }
-
- if ( el.contents ) {
- ifrPos = {left: el.left, top: el.top};
- this.setPositions(adData, el.contents, ifrPos);
- }
-
- if ( el.adId === adData.adId ) {
- adData.element.left = el.left;
- adData.element.top = el.top;
- }
- },
-
- appendTags: function(adData, referenceElement) {
- var mgr = new ContextManager(adData);
- var curWin = exports.utils.elementWindow(referenceElement);
- var highestContainer;
-
- while ( true ) {
- highestContainer = mgr.highestContainer(curWin, referenceElement);
- mgr.captureHTML(highestContainer);
- if ( curWin === curWin.top ) {
- break;
- } else {
-
- curWin.mpAdFound = true;
-
- mgr.adData.serializedIframeContents = mgr.adData.context;
-
- if ( exports.utils.isFriendlyWindow(curWin.parent) ) {
- referenceElement = curWin.frameElement;
- referenceElement[mgr.adData.adId] = {needsWindow: true};
- curWin = curWin.parent;
- } else {
- break;
- }
- }
- }
- return {
- referenceElement:referenceElement,
- highestContainer: highestContainer
- };
- }
- };
-
- exports.tagfinder = tagfinder;
-})(exports);
-
-(function(exports) {
- var _onAdFound;
- var _logGen = new exports.utils.LogGenerator();
- var _pageTags;
- var INIT_MS_BW_SEARCHES = 2000;
- var PAGE_TAG_RE = new RegExp('gpt|oascentral');
- var POST_MSG_ID = '1519242200-10756-12873-1462-13403';
- var AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)');
-
- function getPageTags(doc) {
- var scripts = doc.getElementsByTagName('script');
- var pageTags = [];
- scripts = exports.utils.realArray(scripts);
- scripts.forEach(function(script) {
- if ( PAGE_TAG_RE.exec(script.src) ) {
- pageTags.push({'tagName': 'SCRIPT', 'url': script.src});
- }
- });
- return pageTags;
- }
-
- function messageAllParentFrames(adData) {
-
- adData.postMessageId = POST_MSG_ID;
-
- adData = JSON.stringify(adData);
-
- var win = window;
- while ( win !== win.top ) {
- win = win.parent;
- win.postMessage(adData, '*');
- }
- }
-
- function appendTagsAndSendToParent(adData, referenceElement) {
- var results = exports.tagfinder.appendTags(adData, referenceElement);
- if ( exports.utils.SCRIPT_IN_HOSTILE_IFRAME ) {
- messageAllParentFrames(adData);
-
- } else if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
-
- exports.tagfinder.setPositions(adData);
-
- adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height);
- if ( !adData.matchedSize ) {
-
- if ( AD_SERVER_RE.exec(results.referenceElement.id) ) {
- adData.matchedSize = [adData.width, adData.height];
- adData.oddSize = true;
- } else {
-
- return;
- }
- }
- delete adData.width;
- delete adData.height;
- adData.curPageUrl = exports.utils.getPageUrl();
- _pageTags = _pageTags || getPageTags(document);
- var log = _logGen.log('ad', [adData], _pageTags);
-
- if ( _onAdFound ) {
-
- _onAdFound(log, results.referenceElement);
-
- }
- }
- }
-
- function extractAdsWrapper() {
- if ( exports.utils.SCRIPT_IN_WINDOW_TOP || document.readyState === 'complete' ) {
- extractAds();
- }
- setTimeout(
- function() { extractAdsWrapper(); }, INIT_MS_BW_SEARCHES
- );
- }
-
- function extractAds() {
- var ads = exports.adfinder.findAds(window);
- ads.forEach(function(ad) {
-
- var startTime = new Date().getTime();
- var adId = startTime + '-' + Math.floor(Math.random() * 10e12);
-
- var adData = {
- width: Math.round(ad.offsetWidth),
- height: Math.round(ad.offsetHeight),
- startTime: startTime,
- adId: adId,
- html5: ad.html5 || false
- };
-
- if ( ad.html5 ) {
- adData.adSizeMeta = ad.adSizeMeta || null;
- adData.winClickTag = ad.winClickTag || null;
- }
-
- ad[adId] = { isAd: true };
-
- appendTagsAndSendToParent(adData, ad);
- });
- }
-
- function isChildWin(myWin, otherWin) {
- var parentWin = otherWin.parent;
- while ( parentWin !== otherWin ) {
- if ( parentWin === myWin ) {
- return true;
- }
- otherWin = parentWin;
- parentWin = parentWin.parent;
- }
- return false;
- }
-
- function iframeFromWindow(win, winToMatch) {
- var i, ifr, ifrWin,
- iframes = win.document.querySelectorAll('iframe');
-
- for ( i = 0; i < iframes.length; i++ ) {
- ifr = iframes[i];
- if ( ifr.contentWindow === winToMatch ) {
- return ifr;
- }
- }
-
- for ( i = 0; i < iframes.length; i++ ) {
- ifrWin = iframes[i].contentWindow;
- if ( exports.utils.isFriendlyWindow(ifrWin) ) {
- ifr = iframeFromWindow(ifrWin, winToMatch);
- if ( ifr ) {
- return ifr;
- }
- }
- }
- }
-
- function onPostMessage(event) {
- var adData,
- ifrWin = event.source,
-
- myWin = window.document.defaultView,
- ifrTag;
-
- try {
-
- adData = JSON.parse(event.data);
- } catch(e) {
-
- return;
- }
-
- if ( adData.postMessageId === POST_MSG_ID ) {
-
- delete adData.postMessageId;
-
- event.stopImmediatePropagation();
-
- if ( isChildWin(myWin, ifrWin) ) {
- if ( exports.utils.isFriendlyWindow(ifrWin) ) {
- ifrTag = ifrWin.frameElement;
- } else {
- ifrTag = iframeFromWindow(myWin, ifrWin);
- }
-
- if ( ifrTag ) {
- ifrTag[adData.adId] = {needsWindow: true};
- appendTagsAndSendToParent(adData, ifrTag);
- }
- }
- }
- }
-
- function onVideoMessage(msg, sender, callback) {
- var log;
- if ( msg.event === 'new-video-ad' ) {
- msg.assets.forEach(function(asset) {
-
- });
- log = _logGen.log('video', msg.assets);
- } else {
- log = _logGen.log('invalid-video', msg.assets);
- }
-
- msg.assets.forEach(function(a) {delete a.isVideo;});
- log.displayAdFound = msg.displayAdFound;
- log.requests = msg.requests;
- log.data = msg.event_data;
-
- log.doc.finalPageUrl = log.doc.url;
- log.doc.url = exports.utils.normalizeUrl(msg.origUrl);
-
- _onAdFound(log);
- }
-
- function addBackgroundListener(event, callback) {
- if ( typeof browser !== 'undefined' ) {
- browser.runtime.onMessage.addListener(function(msg) {
- if ( msg.event === event ) {
- callback(msg);
- }
- });
- } else if ( typeof chrome !== 'undefined' ) {
- chrome.runtime.onMessage.addListener(function(msg) {
- if ( msg.event === event ) {
- callback(msg);
- }
- });
- } else if ( window.self.port ) {
- window.self.port.on(event, callback);
- }
- }
-
- exports.coordinator = {
- addPostMessageListener: function() {
- if ( !exports.utils.SCRIPT_IN_FRIENDLY_IFRAME ) {
- window.addEventListener('message', onPostMessage, false);
- }
- },
-
- blockedRobotsMsgGen: function(sendFcn, origUrl) {
-
- if ( origUrl.indexOf('google.com/_/chrome/newtab') === -1 ) {
- var onBlockedRobotsMessage = function() {
- var log;
- log = _logGen.log('invalid-robotstxt', []);
- log.doc.finalPageUrl = log.doc.url;
- log.doc.url = exports.utils.normalizeUrl(origUrl);
-
- sendFcn(log);
- };
- return onBlockedRobotsMessage;
- } else {
- return function() {};
- }
- },
-
- init: function(onAdFound) {
-
- if ( exports.utils.SCRIPT_IN_FRIENDLY_IFRAME ) {
- return false;
- }
-
- _onAdFound = onAdFound;
- if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
- var log = _logGen.log('page');
- onAdFound(log);
-
- window.addEventListener('beforeunload', function(event) {
- var log = _logGen.log('unload');
- log.timing = window.performance.timing;
- onAdFound(log);
- });
-
- addBackgroundListener('new-video-ad', onVideoMessage);
- addBackgroundListener('new-invalid-video-ad', onVideoMessage);
-
- }
-
- exports.utils.onDocLoaded(document, extractAdsWrapper);
- }
- };
-
-})(exports);
-
-if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
- window.adparser = {
- init: exports.coordinator.init,
- addPostMessageListener: exports.coordinator.addPostMessageListener,
- askIfTrackingEnabled: exports.utils.askIfTrackingEnabled,
- blockedRobotsMsgGen: exports.coordinator.blockedRobotsMsgGen,
- inWindowTop: exports.utils.SCRIPT_IN_WINDOW_TOP,
- sendToBackground: exports.utils.sendToBackground
- };
-} else {
- exports.coordinator.addPostMessageListener();
- exports.utils.askIfTrackingEnabled(
- function() {
- exports.coordinator.init(function() {});
- },
- function() {}
- );
-}
-})(window);
-(function(adparser, pageUrl) {
- function onAdFound(log) {
- adparser.sendToBackground({ id: 'ad_log', subject: log }, 'ad_log', '', function(){});
- }
-
- if ( adparser && adparser.inWindowTop ) {
- adparser.addPostMessageListener();
- adparser.askIfTrackingEnabled(
- function() {
- adparser.init(onAdFound);
- },
- adparser.blockedRobotsMsgGen(onAdFound, pageUrl)
- )
- }
-})(window.adparser, window.location.href);
diff --git a/src/drivers/webextension/js/inject.js b/src/drivers/webextension/js/inject.js
index 322c21374..a2458e1bd 100644
--- a/src/drivers/webextension/js/inject.js
+++ b/src/drivers/webextension/js/inject.js
@@ -1,12 +1,12 @@
(() => {
- try {
- const detectJs = chain => {
+ try {
+ const detectJs = (chain) => {
const properties = chain.split('.');
let value = properties.length ? window : null;
- for (let i = 0; i < properties.length; i ++) {
- let property = properties[i];
+ for (let i = 0; i < properties.length; i++) {
+ const property = properties[i];
if (value && value.hasOwnProperty(property)) {
value = value[property];
@@ -20,7 +20,7 @@
return typeof value === 'string' || typeof value === 'number' ? value : !!value;
};
- const onMessage = event => {
+ const onMessage = (event) => {
if (event.data.id !== 'patterns') {
return;
}
@@ -31,15 +31,15 @@
const js = {};
- for (let appName in patterns) {
+ for (const appName in patterns) {
if (patterns.hasOwnProperty(appName)) {
js[appName] = {};
- for (let chain in patterns[appName]) {
+ for (const chain in patterns[appName]) {
if (patterns[appName].hasOwnProperty(chain)) {
js[appName][chain] = {};
- for (let index in patterns[appName][chain]) {
+ for (const index in patterns[appName][chain]) {
const value = detectJs(chain);
if (value && patterns[appName][chain].hasOwnProperty(index)) {
@@ -55,7 +55,7 @@
};
addEventListener('message', onMessage);
- } catch(e) {
+ } catch (e) {
// Fail quietly
}
})();
diff --git a/src/drivers/webextension/js/jsontodom.js b/src/drivers/webextension/js/jsontodom.js
deleted file mode 100644
index c6506c4fb..000000000
--- a/src/drivers/webextension/js/jsontodom.js
+++ /dev/null
@@ -1,63 +0,0 @@
-jsonToDOM.namespaces = {
- html: "http://www.w3.org/1999/xhtml",
- xul: "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"
-};
-
-jsonToDOM.defaultNamespace = jsonToDOM.namespaces.html;
-
-function jsonToDOM(jsonTemplate, doc, nodes) {
- function namespace(name) {
- var reElemNameParts = /^(?:(.*):)?(.*)$/.exec(name);
- return { namespace: jsonToDOM.namespaces[reElemNameParts[1]], shortName: reElemNameParts[2] };
- }
-
- // Note that 'elemNameOrArray' is: either the full element name (eg. [html:]div) or an array of elements in JSON notation
- function tag(elemNameOrArray, elemAttr) {
- // Array of elements? Parse each one...
- if (Array.isArray(elemNameOrArray)) {
- var frag = doc.createDocumentFragment();
- Array.prototype.forEach.call(arguments, function(thisElem) {
- frag.appendChild(tag.apply(null, thisElem));
- });
- return frag;
- }
-
- // Single element? Parse element namespace prefix (if none exists, default to defaultNamespace), and create element
- var elemNs = namespace(elemNameOrArray);
- var elem = doc.createElementNS(elemNs.namespace || jsonToDOM.defaultNamespace, elemNs.shortName);
-
- // Set element's attributes and/or callback functions (eg. onclick)
- for (var key in elemAttr) {
- var val = elemAttr[key];
- if (nodes && key == "key") {
- nodes[val] = elem;
- continue;
- }
-
- var attrNs = namespace(key);
- if (typeof val == "function") {
- // Special case for function attributes; don't just add them as 'on...' attributes, but as events, using addEventListener
- elem.addEventListener(key.replace(/^on/, ""), val, false);
- }
- else {
- // Note that the default namespace for XML attributes is, and should be, blank (ie. they're not in any namespace)
- elem.setAttributeNS(attrNs.namespace || "", attrNs.shortName, val);
- }
- }
-
- // Create and append this element's children
- var childElems = Array.prototype.slice.call(arguments, 2);
- childElems.forEach(function(childElem) {
- if (childElem != null) {
- elem.appendChild(
- childElem instanceof doc.defaultView.Node ? childElem :
- Array.isArray(childElem) ? tag.apply(null, childElem) :
- doc.createTextNode(childElem));
- }
- });
-
- return elem;
- }
-
- return tag.apply(null, jsonTemplate);
-}
diff --git a/src/drivers/webextension/js/lib/iframe.js b/src/drivers/webextension/js/lib/iframe.js
new file mode 100644
index 000000000..2185a709e
--- /dev/null
+++ b/src/drivers/webextension/js/lib/iframe.js
@@ -0,0 +1,1140 @@
+
+
+(function (win) {
+ const exports = {};
+
+ (function (exports) {
+ const utils = {
+ normalizeUrl(url) {
+ return this.hashUrl(url) || null;
+ },
+
+ getReferrer() {
+ return this.normalizeUrl(document.referrer);
+ },
+
+ getPageUrl() {
+ return this.normalizeUrl(window.location.href);
+ },
+ hashUrl(url) {
+ let a,
+ result;
+
+ if (!url || url.indexOf('http') !== 0) {
+ return null;
+ }
+
+ a = document.createElement('a');
+ a.href = url;
+
+ result = `${a.protocol}//${a.hostname}/`;
+
+ if (a.pathname && a.pathname !== '/') {
+ result += this.hashCode(a.pathname);
+ }
+
+ if (a.search) {
+ result += `?${this.hashCode(a.search)}`;
+ }
+
+ if (a.hash) {
+ result += `#${this.hashCode(a.hash)}`;
+ }
+
+ return result;
+ },
+
+ hashCode(str) {
+ let hash = 0,
+ kar,
+ i;
+
+ if (str.length === 0) {
+ return hash;
+ }
+
+ for (i = 0; i < str.length; i++) {
+ kar = str.charCodeAt(i);
+ hash = ((hash << 5) - hash) + kar;
+ hash &= hash;
+ }
+
+ return hash + Math.pow(2, 32);
+ },
+
+ realArray(a) {
+ return Array.prototype.slice.apply(a);
+ },
+
+ onDocLoaded(doc, callback) {
+ if (doc.readyState === 'loading') {
+ doc.addEventListener('DOMContentLoaded', callback);
+ } else {
+ callback();
+ }
+ },
+
+ SCRIPT_IN_WINDOW_TOP: window === window.top,
+
+ isFriendlyWindow(win) {
+ let href;
+ try {
+ href = win.location.href;
+ } catch (e) {
+ return false;
+ }
+ return true;
+ },
+
+ elementWindow(el) {
+ return el.ownerDocument.defaultView;
+ },
+
+ viewport(win) {
+ return { width: win.innerWidth, height: win.innerHeight };
+ },
+
+ parseQS(qs) {
+ if (qs.indexOf('http') === 0) {
+ qs = qs.split('?')[1];
+ }
+ let i,
+ kvs,
+ key,
+ val;
+ const dict = {};
+ qs = qs.split('&');
+ for (i = 0; i < qs.length; i++) {
+ kvs = qs[i].split('=');
+ key = kvs[0];
+ val = kvs.slice(1).join('=');
+ try {
+ dict[key] = window.decodeURIComponent(val);
+ } catch (e) {
+ continue;
+ }
+ }
+ return dict;
+ },
+ sendToBackground(message, event, responseMessage, onResponse) {
+ if (typeof browser !== 'undefined') {
+ const response = browser.runtime.sendMessage(message);
+ response.then(onResponse);
+ } else if (typeof chrome !== 'undefined') {
+ chrome.runtime.sendMessage(message, onResponse);
+ } else if (window.self.port) {
+ window.self.port.on(responseMessage, onResponse);
+ window.self.port.emit(event, message);
+ }
+ },
+
+ askIfTrackingEnabled(callback, elseCallback) {
+ this.sendToBackground(
+ 'is_tracking_enabled',
+ '',
+ 'tracking_enabled_response',
+ (message) => {
+ if (message && message.tracking_enabled) {
+ callback();
+ } else {
+ elseCallback();
+ }
+ },
+ );
+ },
+ };
+
+ utils.SCRIPT_IN_FRIENDLY_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && utils.isFriendlyWindow(window.parent);
+ utils.SCRIPT_IN_HOSTILE_IFRAME = !utils.SCRIPT_IN_WINDOW_TOP && !utils.SCRIPT_IN_FRIENDLY_IFRAME;
+
+ function LogGenerator() {
+ this.msgNum = 0;
+ this.pageMeta = {
+ url: utils.getPageUrl(),
+ isHP: window.location.pathname === '/',
+ referrer: utils.getReferrer(),
+ rand: Math.floor(Math.random() * 10e12),
+ startTime: new Date().getTime(),
+ };
+ }
+
+ LogGenerator.prototype = {
+ log(event, opt_assets, opt_pageTags) {
+ let opt_video_assets;
+ if (event === 'video' || event === 'invalid-video') {
+ opt_video_assets = opt_assets || [];
+ opt_assets = [];
+ } else {
+ opt_video_assets = [];
+ opt_assets = opt_assets || [];
+ }
+ const result = {
+ doc: this.pageMeta,
+ event,
+ video_assets: opt_video_assets,
+ assets: opt_assets,
+ version: '3',
+ mrev: '4aeaa5a-c',
+ msgNum: this.msgNum,
+ timestamp: new Date().getTime(),
+ pageVis: document.visibilityState,
+ pageFoc: document.hasFocus(),
+ pageTags: opt_pageTags || [],
+ };
+ this.msgNum++;
+ return result;
+ },
+ };
+
+ utils.LogGenerator = LogGenerator;
+
+ exports.utils = utils;
+ }(exports));
+
+ (function (exports) {
+ const SizeMatcher = {
+ VALID_AD_SIZES: [
+ [300, 50],
+ [320, 50],
+ [160, 600],
+ [300, 250],
+ [300, 600],
+ [300, 1050],
+ [336, 280],
+ [336, 850],
+ [468, 60],
+ [728, 90],
+ [728, 250],
+ [728, 270],
+ [970, 66],
+ [970, 90],
+ [970, 125],
+ [970, 250],
+ [970, 400],
+ [970, 415],
+ [1280, 100],
+ ],
+
+ PX_SIZE_TOL: 10,
+
+ getMatchedAdSize(width, height) {
+ if (!this.set) {
+ this.set = this._makeSizeSet();
+ }
+
+ return this.set[`${Math.round(width)}x${Math.round(height)}`];
+ },
+
+ elementIsAdShaped(el) {
+ return !!this.getMatchedAdSizeForElement(el);
+ },
+
+ getMatchedAdSizeForElement(el) {
+ const rect = el.getBoundingClientRect();
+ return this.getMatchedAdSize(rect.width, rect.height);
+ },
+
+ _makeSizeSet() {
+ const set = {};
+ let i;
+ let xfuz;
+ let yfuz;
+ let size;
+ let width;
+ let height;
+
+ for (i = 0; i < this.VALID_AD_SIZES.length; i++) {
+ for (xfuz = -this.PX_SIZE_TOL; xfuz <= this.PX_SIZE_TOL; xfuz++) {
+ for (yfuz = -this.PX_SIZE_TOL; yfuz <= this.PX_SIZE_TOL; yfuz++) {
+ size = this.VALID_AD_SIZES[i];
+ width = size[0] + xfuz;
+ height = size[1] + yfuz;
+ set[`${width}x${height}`] = size;
+ }
+ }
+ }
+ return set;
+ },
+ };
+
+ const Throttler = {
+ MAX_SEARCHES_PER_WINDOW: 10,
+ MAX_SEARCHES_PER_ELEMENT: 2,
+
+ countSearch(el) {
+ if (typeof el.searches !== 'number') {
+ el.searches = 0;
+ }
+
+ el.searches += 1;
+ },
+
+ throttle(el, max) {
+ if (typeof el.searches === 'number' && el.searches >= max) {
+ return true;
+ }
+ return false;
+ },
+
+ throttleElement(el) {
+ return this.throttle(el, this.MAX_SEARCHES_PER_ELEMENT);
+ },
+
+ throttleWin(win) {
+ return this.throttle(win, this.MAX_SEARCHES_PER_WINDOW);
+ },
+
+ getCount(el) {
+ return el.searches || 0;
+ },
+ };
+
+ function TopSearcher(win) {
+ this.win = win;
+ this.doc = win.document;
+ }
+
+ TopSearcher.prototype.search = function () {
+ let candidates = exports.utils.realArray(this.doc.querySelectorAll('img, object, embed')),
+ html5Ad,
+ ads = [];
+
+ ads = ads.concat(candidates.filter((el) => {
+ if (!el.mpAdFound && !Throttler.throttleElement(el)) {
+ Throttler.countSearch(el);
+ if ((el.tagName !== 'IMG' || isStandardImage(el)) && SizeMatcher.elementIsAdShaped(el)) {
+ el.mpAdFound = true;
+ return true;
+ }
+ }
+ return false;
+ }));
+
+ html5Ad = this._mainGetHTMLAd();
+ if (html5Ad) {
+ html5Ad.html5 = true;
+ html5Ad.mpAdFound = true;
+ ads.push(html5Ad);
+ }
+
+ return ads;
+ };
+
+ TopSearcher.prototype._mainGetHTMLAd = function () {
+ let styles = this.doc.querySelectorAll('div > style, div > link[rel="stylesheet"]'),
+ i,
+ div;
+ for (i = 0; i < styles.length; i++) {
+ div = styles[i].parentNode;
+ if (!div.mpAdFound && SizeMatcher.elementIsAdShaped(div) && this._jumpedOut(div)) {
+ return div;
+ }
+ }
+ };
+
+ TopSearcher.prototype._jumpedOut = function (el) {
+ let siblings,
+ ifrs;
+ siblings = exports.utils.realArray(el.parentNode.children);
+ ifrs = siblings.filter(el => el.tagName === 'IFRAME' && el.offsetWidth === 0 && el.offsetHeight === 0);
+ return ifrs.length > 0;
+ };
+
+ function IframeSearcher(win) {
+ this.MIN_AD_AREA = 14000;
+ this.MIN_WINDOW_PX = 10;
+
+ this.win = win;
+ this.doc = win.document;
+ this.body = win.document.body;
+ this.winClickTag = win.clickTag;
+ this.adSizeMeta = this._getAdSizeMeta();
+ this.numElementsInBody = (this.body && this.body.querySelectorAll('*').length) || 0;
+
+ this.shouldSearchWindow = false;
+ if (!this.win.mpAdFound && this.body && !Throttler.throttleWin(this.win)) {
+ this.winWidth = this.win.innerWidth;
+ this.winHeight = this.win.innerHeight;
+ if (this._meetsMinAdSize(this.winWidth, this.winHeight) && !this._containsLargeIframes()) {
+ this.shouldSearchWindow = true;
+ }
+ }
+ }
+
+ IframeSearcher.prototype.search = function () {
+ let ad;
+
+ if (this.shouldSearchWindow) {
+ ad = this._search();
+ if (ad) {
+ ad.mpAdFound = true;
+ win.mpAdFound = true;
+ return ad;
+ }
+ Throttler.countSearch(this.win);
+ }
+
+ return null;
+ };
+
+ IframeSearcher.prototype._search = function () {
+ let _this = this,
+ stdCandidates,
+ html5Candidates,
+ stdEl,
+ html5El;
+
+ stdCandidates = this.body.querySelectorAll('img, object, embed');
+
+ stdEl = getFirst(stdCandidates, (el) => {
+ if (!el.mpAdFound
+ && !Throttler.throttleElement(el)
+ && (el.tagName !== 'IMG' || isStandardImage(el))
+ && _this._elementIsAtLeastAsBigAsWindow(el)) {
+ return true;
+ }
+ Throttler.countSearch(el);
+ return false;
+ });
+
+ if (stdEl) {
+ return stdEl;
+ }
+
+ if (this._isHTML5Iframe()) {
+ html5Candidates = this.doc.querySelectorAll('body, canvas, button, video, svg, div');
+ html5El = getFirst(html5Candidates, (el) => {
+ if (_this._elementIsAtLeastAsBigAsWindow(el)) {
+ return true;
+ }
+ Throttler.countSearch(el);
+ return false;
+ });
+ }
+
+ if (html5El) {
+ html5El.html5 = true;
+ html5El.winClickTag = this.winClickTag;
+ html5El.adSizeMeta = this.adSizeMeta;
+ return html5El;
+ }
+
+ return null;
+ };
+
+ IframeSearcher.prototype._isHTML5Iframe = function () {
+ if (this.winClickTag || this.adSizeMeta) {
+ return true;
+ }
+
+ if (this.doc.querySelectorAll('canvas', 'button', 'video', 'svg').length > 0) {
+ return true;
+ }
+
+ if (this.numElementsInBody >= 5 && Throttler.getCount(this.win) > 0 && this.doc.querySelectorAll('div').length > 0) {
+ return true;
+ }
+
+ return false;
+ };
+
+ IframeSearcher.prototype._elementIsAtLeastAsBigAsWindow = function (el) {
+ let rect = el.getBoundingClientRect(),
+ tol = 0.95;
+
+ return rect.width >= (tol * this.winWidth) && rect.height >= (tol * this.winHeight);
+ };
+
+ IframeSearcher.prototype._meetsMinAdSize = function (width, height) {
+ return (width * height) >= this.MIN_AD_AREA;
+ };
+
+ IframeSearcher.prototype._containsLargeIframes = function () {
+ const iframes = this.doc.querySelectorAll('iframe');
+ let rect;
+ let i;
+ for (i = 0; i < iframes.length; i++) {
+ rect = iframes[i].getBoundingClientRect();
+ if (rect.width > this.MIN_WINDOW_PX || rect.height > this.MIN_WINDOW_PX) {
+ return true;
+ }
+ }
+ return false;
+ };
+
+ IframeSearcher.prototype._getAdSizeMeta = function () {
+ const adSizeMeta = this.doc.querySelectorAll('meta[name="ad.size"]');
+ if (adSizeMeta.length > 0) {
+ return adSizeMeta[0].content;
+ }
+ return null;
+ };
+
+ function getFirst(arr, testFn) {
+ let i,
+ el;
+ for (i = 0; i < arr.length; i++) {
+ el = arr[i];
+ if (testFn(el)) {
+ return el;
+ }
+ }
+ return null;
+ }
+
+ function isStandardImage(img) {
+ return img.src && (img.parentNode.tagName === 'A' || img.getAttribute('onclick'));
+ }
+
+ function getFriendlyIframes(win) {
+ let iframes = win.document.querySelectorAll('iframe');
+ iframes = exports.utils.realArray(iframes);
+ const friendlyIframes = iframes.filter(ifr => exports.utils.isFriendlyWindow(ifr.contentWindow));
+ return friendlyIframes;
+ }
+
+ function findAds(win) {
+ let i,
+ iframes,
+ searcher,
+ ad,
+ ads = [];
+
+ if (win === win.top) {
+ searcher = new TopSearcher(win);
+ ads = ads.concat(searcher.search());
+ } else {
+ searcher = new IframeSearcher(win);
+ ad = searcher.search();
+ if (ad) {
+ ads.push(ad);
+ }
+ }
+
+ iframes = getFriendlyIframes(win);
+ for (i = 0; i < iframes.length; i++) {
+ ads = ads.concat(findAds(iframes[i].contentWindow));
+ }
+
+ return ads;
+ }
+
+ exports.adfinder = {
+ getMatchedAdSize: SizeMatcher.getMatchedAdSize.bind(SizeMatcher),
+ findAds,
+ };
+ }(exports));
+
+ (function (exports) {
+ const parser = {
+ TAGS_WITH_SRC_ATTR: {
+ IMG: true,
+ SCRIPT: true,
+ IFRAME: true,
+ EMBED: true,
+ },
+
+ MAX_ATTR_LEN: 100,
+
+ getUrl(el, params) {
+ let url;
+
+ if (this.TAGS_WITH_SRC_ATTR.hasOwnProperty(el.tagName)) {
+ url = el.src;
+ } else if (el.tagName === 'OBJECT') {
+ url = el.data || (params && params.movie) || null;
+ } else if (el.tagName === 'A') {
+ url = el.href;
+ }
+
+ if (url && url.indexOf('http') === 0) {
+ return url;
+ }
+ return null;
+ },
+
+ getParams(el) {
+ if (el.tagName !== 'OBJECT') {
+ return null;
+ }
+
+ let i,
+ child;
+ const params = {};
+ const children = el.children;
+ for (i = 0; i < children.length; i++) {
+ child = children[i];
+ if (child.tagName === 'PARAM' && child.name) {
+ params[child.name.toLowerCase()] = child.value;
+ }
+ }
+ return params;
+ },
+
+ getPosition(el) {
+ const rect = el.getBoundingClientRect();
+ const win = exports.utils.elementWindow(el);
+
+ return {
+ width: Math.round(rect.width),
+ height: Math.round(rect.height),
+ left: Math.round(rect.left + win.pageXOffset),
+ top: Math.round(rect.top + win.pageYOffset),
+ };
+ },
+
+ getFlashvars(el, params, url) {
+ let flashvars;
+ const urlQS = url && url.split('?')[1];
+
+ if (el.tagName === 'EMBED') {
+ flashvars = el.getAttribute('flashvars') || urlQS;
+ } else if (el.tagName === 'OBJECT') {
+ flashvars = params.flashvars || el.getAttribute('flashvars') || urlQS;
+ }
+
+ return (flashvars && exports.utils.parseQS(flashvars)) || null;
+ },
+
+ findClickThru(el, flashvars) {
+ let key;
+ if (el.tagName === 'IMG' && el.parentElement.tagName === 'A') {
+ return el.parentElement.href;
+ } if (flashvars) {
+ for (key in flashvars) {
+ if (flashvars.hasOwnProperty(key)) {
+ if (key.toLowerCase().indexOf('clicktag') === 0) {
+ return flashvars[key];
+ }
+ }
+ }
+ }
+ return null;
+ },
+
+ getAttr(el, name) {
+ const val = el.getAttribute(name);
+
+ if (val && val.slice && val.toString) {
+ return val.slice(0, this.MAX_ATTR_LEN).toString();
+ }
+ return null;
+ },
+
+ putPropIfExists(obj, name, val) {
+ if (val) {
+ obj[name] = val;
+ }
+ },
+
+ putAttrIfExists(obj, el, name) {
+ const val = this.getAttr(el, name);
+ this.putPropIfExists(obj, name, val);
+ },
+
+ elementToJSON(el, opt_findClickThru) {
+ const pos = this.getPosition(el);
+ const params = this.getParams(el);
+ const url = this.getUrl(el, params);
+ const flashvars = this.getFlashvars(el, params, url);
+ const clickThru = opt_findClickThru && this.findClickThru(el, flashvars);
+ const json = {
+ tagName: el.tagName,
+ width: pos.width,
+ height: pos.height,
+ left: pos.left,
+ top: pos.top,
+ children: [],
+ };
+
+ if (params) {
+ delete params.flashvars;
+ }
+
+ this.putAttrIfExists(json, el, 'id');
+ this.putAttrIfExists(json, el, 'class');
+ this.putAttrIfExists(json, el, 'name');
+
+ this.putPropIfExists(json, 'flashvars', flashvars);
+ this.putPropIfExists(json, 'url', url);
+ this.putPropIfExists(json, 'params', params);
+ this.putPropIfExists(json, 'clickThru', clickThru);
+
+ return json;
+ },
+ };
+
+ exports.parser = { elementToJSON: parser.elementToJSON.bind(parser) };
+ }(exports));
+
+ (function (exports) {
+ const ContextManager = function (adData) {
+ this.adData = adData;
+ };
+
+ ContextManager.prototype = {
+ CONTAINER_SIZE_TOL: 0.4,
+ ASPECT_RATIO_FOR_LEADERBOARDS: 2,
+
+ isValidContainer(el, opt_curWin) {
+ const cWidth = el.clientWidth;
+ const cHeight = el.clientHeight;
+
+ const adWidth = this.adData.width;
+ const adHeight = this.adData.height;
+
+ const winWidth = opt_curWin && opt_curWin.innerWidth;
+ const winHeight = opt_curWin && opt_curWin.innerHeight;
+ const similarWin = opt_curWin && this.withinTol(adWidth, winWidth) && this.withinTol(adHeight, winHeight);
+
+ const similarSizeX = this.withinTol(adWidth, cWidth);
+ const similarSizeY = this.withinTol(adHeight, cHeight);
+ const adAspect = adWidth / adHeight;
+
+ return similarWin || el.tagName === 'A' || (adAspect >= this.ASPECT_RATIO_FOR_LEADERBOARDS && similarSizeY) || (similarSizeX && similarSizeY);
+ },
+
+ withinTol(adlen, conlen) {
+ const pct = (conlen - adlen) / adlen;
+
+ return pct <= this.CONTAINER_SIZE_TOL;
+ },
+
+ serializeElements(el) {
+ if (!el) {
+ return;
+ }
+ let i;
+ let ifrWin;
+ const adId = this.adData.adId;
+ let elIsAd = false;
+
+ if (adId && el[adId] && el[adId].isAd === true) {
+ elIsAd = true;
+ }
+
+ const json = exports.parser.elementToJSON(el, elIsAd);
+ let childJSON;
+
+ if (elIsAd) {
+ json.adId = adId;
+ this.adData.element = {};
+
+ const keys = Object.keys(json);
+ for (i = 0; i < keys.length; i++) {
+ const key = keys[i];
+ if (key !== 'children' && key !== 'contents') {
+ this.adData.element[key] = json[key];
+ }
+ }
+ }
+
+ const children = exports.utils.realArray(el.children).filter((el) => {
+ const param = el.tagName === 'PARAM';
+ const inlineScript = el.tagName === 'SCRIPT' && !(el.src && el.src.indexOf('http') >= 0);
+ const noScript = el.tagName === 'NOSCRIPT';
+ return !(param || inlineScript || noScript);
+ });
+
+ for (i = 0; i < children.length; i++) {
+ childJSON = this.serializeElements(children[i]);
+ if (childJSON) {
+ json.children.push(childJSON);
+ }
+ }
+
+ if (el.tagName === 'IFRAME') {
+ ifrWin = el.contentWindow;
+
+ if (adId && el[adId] && el[adId].needsWindow) {
+ json.contents = this.adData.serializedIframeContents;
+ el[adId].needsWindow = false;
+ delete this.adData.serializedIframeContents;
+ } else if (exports.utils.isFriendlyWindow(ifrWin)) {
+ childJSON = this.serializeElements(ifrWin.document.documentElement);
+ if (childJSON) {
+ json.contents = childJSON;
+ }
+ }
+ }
+
+ if (json.children.length > 0 || json.adId || json.tagName === 'IFRAME' || json.url) {
+ return json;
+ }
+ return null;
+ },
+
+ captureHTML(containerEl) {
+ this.adData.context = this.serializeElements(containerEl);
+ },
+
+ nodeCount(el) {
+ return el.getElementsByTagName('*').length + 1;
+ },
+
+ highestContainer(curWin, referenceElement) {
+ let curContainer = referenceElement;
+ const docEl = curWin.document.documentElement;
+ let parentContainer;
+
+ if (curWin !== curWin.top && this.isValidContainer(docEl, curWin)) {
+ return docEl;
+ }
+
+ while (true) {
+ parentContainer = curContainer.parentElement;
+ if (parentContainer && this.isValidContainer(parentContainer)) {
+ curContainer = parentContainer;
+ } else {
+ return curContainer;
+ }
+ }
+ },
+ };
+
+ const tagfinder = {
+
+ setPositions(adData, opt_el, opt_winPos) {
+ const el = opt_el || adData.context;
+ const winPos = opt_winPos || { left: 0, top: 0 };
+ let ifrPos;
+
+ el.left += winPos.left;
+ el.top += winPos.top;
+
+ if (el.children) {
+ el.children.forEach(function (child) {
+ this.setPositions(adData, child, winPos);
+ }, this);
+ }
+
+ if (el.contents) {
+ ifrPos = { left: el.left, top: el.top };
+ this.setPositions(adData, el.contents, ifrPos);
+ }
+
+ if (el.adId === adData.adId) {
+ adData.element.left = el.left;
+ adData.element.top = el.top;
+ }
+ },
+
+ appendTags(adData, referenceElement) {
+ const mgr = new ContextManager(adData);
+ let curWin = exports.utils.elementWindow(referenceElement);
+ let highestContainer;
+
+ while (true) {
+ highestContainer = mgr.highestContainer(curWin, referenceElement);
+ mgr.captureHTML(highestContainer);
+ if (curWin === curWin.top) {
+ break;
+ } else {
+ curWin.mpAdFound = true;
+
+ mgr.adData.serializedIframeContents = mgr.adData.context;
+
+ if (exports.utils.isFriendlyWindow(curWin.parent)) {
+ referenceElement = curWin.frameElement;
+ referenceElement[mgr.adData.adId] = { needsWindow: true };
+ curWin = curWin.parent;
+ } else {
+ break;
+ }
+ }
+ }
+ return {
+ referenceElement,
+ highestContainer,
+ };
+ },
+ };
+
+ exports.tagfinder = tagfinder;
+ }(exports));
+
+ (function (exports) {
+ let _onAdFound;
+ const _logGen = new exports.utils.LogGenerator();
+ let _pageTags;
+ const INIT_MS_BW_SEARCHES = 2000;
+ const PAGE_TAG_RE = new RegExp('gpt|oascentral');
+ const POST_MSG_ID = '1519242200-10756-12873-1462-13403';
+ const AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)');
+
+ function getPageTags(doc) {
+ let scripts = doc.getElementsByTagName('script');
+ const pageTags = [];
+ scripts = exports.utils.realArray(scripts);
+ scripts.forEach((script) => {
+ if (PAGE_TAG_RE.exec(script.src)) {
+ pageTags.push({ tagName: 'SCRIPT', url: script.src });
+ }
+ });
+ return pageTags;
+ }
+
+ function messageAllParentFrames(adData) {
+ adData.postMessageId = POST_MSG_ID;
+
+ adData = JSON.stringify(adData);
+
+ let win = window;
+ while (win !== win.top) {
+ win = win.parent;
+ win.postMessage(adData, '*');
+ }
+ }
+
+ function appendTagsAndSendToParent(adData, referenceElement) {
+ const results = exports.tagfinder.appendTags(adData, referenceElement);
+ if (exports.utils.SCRIPT_IN_HOSTILE_IFRAME) {
+ messageAllParentFrames(adData);
+ } else if (exports.utils.SCRIPT_IN_WINDOW_TOP) {
+ exports.tagfinder.setPositions(adData);
+
+ adData.matchedSize = exports.adfinder.getMatchedAdSize(adData.width, adData.height);
+ if (!adData.matchedSize) {
+ if (AD_SERVER_RE.exec(results.referenceElement.id)) {
+ adData.matchedSize = [adData.width, adData.height];
+ adData.oddSize = true;
+ } else {
+ return;
+ }
+ }
+ delete adData.width;
+ delete adData.height;
+ adData.curPageUrl = exports.utils.getPageUrl();
+ _pageTags = _pageTags || getPageTags(document);
+ const log = _logGen.log('ad', [adData], _pageTags);
+
+ if (_onAdFound) {
+ _onAdFound(log, results.referenceElement);
+ }
+ }
+ }
+
+ function extractAdsWrapper() {
+ if (exports.utils.SCRIPT_IN_WINDOW_TOP || document.readyState === 'complete') {
+ extractAds();
+ }
+ setTimeout(
+ () => { extractAdsWrapper(); }, INIT_MS_BW_SEARCHES,
+ );
+ }
+
+ function extractAds() {
+ const ads = exports.adfinder.findAds(window);
+ ads.forEach((ad) => {
+ const startTime = new Date().getTime();
+ const adId = `${startTime}-${Math.floor(Math.random() * 10e12)}`;
+
+ const adData = {
+ width: Math.round(ad.offsetWidth),
+ height: Math.round(ad.offsetHeight),
+ startTime,
+ adId,
+ html5: ad.html5 || false,
+ };
+
+ if (ad.html5) {
+ adData.adSizeMeta = ad.adSizeMeta || null;
+ adData.winClickTag = ad.winClickTag || null;
+ }
+
+ ad[adId] = { isAd: true };
+
+ appendTagsAndSendToParent(adData, ad);
+ });
+ }
+
+ function isChildWin(myWin, otherWin) {
+ let parentWin = otherWin.parent;
+ while (parentWin !== otherWin) {
+ if (parentWin === myWin) {
+ return true;
+ }
+ otherWin = parentWin;
+ parentWin = parentWin.parent;
+ }
+ return false;
+ }
+
+ function iframeFromWindow(win, winToMatch) {
+ let i,
+ ifr,
+ ifrWin,
+ iframes = win.document.querySelectorAll('iframe');
+
+ for (i = 0; i < iframes.length; i++) {
+ ifr = iframes[i];
+ if (ifr.contentWindow === winToMatch) {
+ return ifr;
+ }
+ }
+
+ for (i = 0; i < iframes.length; i++) {
+ ifrWin = iframes[i].contentWindow;
+ if (exports.utils.isFriendlyWindow(ifrWin)) {
+ ifr = iframeFromWindow(ifrWin, winToMatch);
+ if (ifr) {
+ return ifr;
+ }
+ }
+ }
+ }
+
+ function onPostMessage(event) {
+ let adData,
+ ifrWin = event.source,
+
+ myWin = window.document.defaultView,
+ ifrTag;
+
+ try {
+ adData = JSON.parse(event.data);
+ } catch (e) {
+ return;
+ }
+
+ if (adData.postMessageId === POST_MSG_ID) {
+ delete adData.postMessageId;
+
+ event.stopImmediatePropagation();
+
+ if (isChildWin(myWin, ifrWin)) {
+ if (exports.utils.isFriendlyWindow(ifrWin)) {
+ ifrTag = ifrWin.frameElement;
+ } else {
+ ifrTag = iframeFromWindow(myWin, ifrWin);
+ }
+
+ if (ifrTag) {
+ ifrTag[adData.adId] = { needsWindow: true };
+ appendTagsAndSendToParent(adData, ifrTag);
+ }
+ }
+ }
+ }
+
+ function onVideoMessage(msg, sender, callback) {
+ let log;
+ if (msg.event === 'new-video-ad') {
+ msg.assets.forEach((asset) => {
+
+ });
+ log = _logGen.log('video', msg.assets);
+ } else {
+ log = _logGen.log('invalid-video', msg.assets);
+ }
+
+ msg.assets.forEach((a) => { delete a.isVideo; });
+ log.displayAdFound = msg.displayAdFound;
+ log.requests = msg.requests;
+ log.data = msg.event_data;
+
+ log.doc.finalPageUrl = log.doc.url;
+ log.doc.url = exports.utils.normalizeUrl(msg.origUrl);
+
+ _onAdFound(log);
+ }
+
+ function addBackgroundListener(event, callback) {
+ if (typeof browser !== 'undefined') {
+ browser.runtime.onMessage.addListener((msg) => {
+ if (msg.event === event) {
+ callback(msg);
+ }
+ });
+ } else if (typeof chrome !== 'undefined') {
+ chrome.runtime.onMessage.addListener((msg) => {
+ if (msg.event === event) {
+ callback(msg);
+ }
+ });
+ } else if (window.self.port) {
+ window.self.port.on(event, callback);
+ }
+ }
+
+ exports.coordinator = {
+ addPostMessageListener() {
+ if (!exports.utils.SCRIPT_IN_FRIENDLY_IFRAME) {
+ window.addEventListener('message', onPostMessage, false);
+ }
+ },
+
+ blockedRobotsMsgGen(sendFcn, origUrl) {
+ if (origUrl.indexOf('google.com/_/chrome/newtab') === -1) {
+ const onBlockedRobotsMessage = function () {
+ let log;
+ log = _logGen.log('invalid-robotstxt', []);
+ log.doc.finalPageUrl = log.doc.url;
+ log.doc.url = exports.utils.normalizeUrl(origUrl);
+
+ sendFcn(log);
+ };
+ return onBlockedRobotsMessage;
+ }
+ return function () {};
+ },
+
+ init(onAdFound) {
+ if (exports.utils.SCRIPT_IN_FRIENDLY_IFRAME) {
+ return false;
+ }
+
+ _onAdFound = onAdFound;
+ if (exports.utils.SCRIPT_IN_WINDOW_TOP) {
+ const log = _logGen.log('page');
+ onAdFound(log);
+
+ window.addEventListener('beforeunload', (event) => {
+ const log = _logGen.log('unload');
+ log.timing = window.performance.timing;
+ onAdFound(log);
+ });
+
+ addBackgroundListener('new-video-ad', onVideoMessage);
+ addBackgroundListener('new-invalid-video-ad', onVideoMessage);
+ }
+
+ exports.utils.onDocLoaded(document, extractAdsWrapper);
+ },
+ };
+ }(exports));
+
+ if (exports.utils.SCRIPT_IN_WINDOW_TOP) {
+ window.adparser = {
+ init: exports.coordinator.init,
+ addPostMessageListener: exports.coordinator.addPostMessageListener,
+ askIfTrackingEnabled: exports.utils.askIfTrackingEnabled,
+ blockedRobotsMsgGen: exports.coordinator.blockedRobotsMsgGen,
+ inWindowTop: exports.utils.SCRIPT_IN_WINDOW_TOP,
+ sendToBackground: exports.utils.sendToBackground,
+ };
+ } else {
+ exports.coordinator.addPostMessageListener();
+ exports.utils.askIfTrackingEnabled(
+ () => {
+ exports.coordinator.init(() => {});
+ },
+ () => {},
+ );
+ }
+}(window));
+(function (adparser, pageUrl) {
+ function onAdFound(log) {
+ adparser.sendToBackground({ id: 'ad_log', subject: log }, 'ad_log', '', () => {});
+ }
+
+ if (adparser && adparser.inWindowTop) {
+ adparser.addPostMessageListener();
+ adparser.askIfTrackingEnabled(
+ () => {
+ adparser.init(onAdFound);
+ },
+ adparser.blockedRobotsMsgGen(onAdFound, pageUrl),
+ );
+ }
+}(window.adparser, window.location.href));
diff --git a/src/drivers/webextension/js/lib/jsontodom.js b/src/drivers/webextension/js/lib/jsontodom.js
new file mode 100644
index 000000000..24d9e4c29
--- /dev/null
+++ b/src/drivers/webextension/js/lib/jsontodom.js
@@ -0,0 +1,63 @@
+jsonToDOM.namespaces = {
+ html: 'http://www.w3.org/1999/xhtml',
+ xul: 'http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul',
+};
+
+jsonToDOM.defaultNamespace = jsonToDOM.namespaces.html;
+
+function jsonToDOM(jsonTemplate, doc, nodes) {
+ function namespace(name) {
+ const reElemNameParts = /^(?:(.*):)?(.*)$/.exec(name);
+ return { namespace: jsonToDOM.namespaces[reElemNameParts[1]], shortName: reElemNameParts[2] };
+ }
+
+ // Note that 'elemNameOrArray' is: either the full element name (eg. [html:]div) or an array of elements in JSON notation
+ function tag(elemNameOrArray, elemAttr) {
+ // Array of elements? Parse each one...
+ if (Array.isArray(elemNameOrArray)) {
+ const frag = doc.createDocumentFragment();
+ Array.prototype.forEach.call(arguments, (thisElem) => {
+ frag.appendChild(tag(...thisElem));
+ });
+ return frag;
+ }
+
+ // Single element? Parse element namespace prefix (if none exists, default to defaultNamespace), and create element
+ const elemNs = namespace(elemNameOrArray);
+ const elem = doc.createElementNS(elemNs.namespace || jsonToDOM.defaultNamespace, elemNs.shortName);
+
+ // Set element's attributes and/or callback functions (eg. onclick)
+ for (const key in elemAttr) {
+ const val = elemAttr[key];
+ if (nodes && key == 'key') {
+ nodes[val] = elem;
+ continue;
+ }
+
+ const attrNs = namespace(key);
+ if (typeof val === 'function') {
+ // Special case for function attributes; don't just add them as 'on...' attributes, but as events, using addEventListener
+ elem.addEventListener(key.replace(/^on/, ''), val, false);
+ } else {
+ // Note that the default namespace for XML attributes is, and should be, blank (ie. they're not in any namespace)
+ elem.setAttributeNS(attrNs.namespace || '', attrNs.shortName, val);
+ }
+ }
+
+ // Create and append this element's children
+ const childElems = Array.prototype.slice.call(arguments, 2);
+ childElems.forEach((childElem) => {
+ if (childElem != null) {
+ elem.appendChild(
+ childElem instanceof doc.defaultView.Node ? childElem
+ : Array.isArray(childElem) ? tag(...childElem)
+ : doc.createTextNode(childElem),
+ );
+ }
+ });
+
+ return elem;
+ }
+
+ return tag(...jsonTemplate);
+}
diff --git a/src/drivers/webextension/js/lib/network.js b/src/drivers/webextension/js/lib/network.js
new file mode 100644
index 000000000..81cc45a54
--- /dev/null
+++ b/src/drivers/webextension/js/lib/network.js
@@ -0,0 +1,793 @@
+
+(function () {
+ function isChrome() {
+ return (typeof chrome !== 'undefined'
+ && window.navigator.userAgent.match(/Chrom(?:e|ium)\/([0-9\.]+)/));
+ }
+
+ let browserProxy;
+ if (isChrome()) {
+ browserProxy = chrome;
+ } else {
+ browserProxy = browser;
+ }
+
+ const MIN_FF_MAJOR_VERSION = 51;
+
+ const requiredBrowserApis = [
+ browserProxy.webNavigation,
+ browserProxy.tabs,
+ browserProxy.webRequest,
+ browserProxy.runtime,
+ ];
+ let areListenersRegistered = false;
+ const secBefore = 2000;
+ const secAfter = 5000;
+ const secBetweenDupAssets = 10e3;
+ const minVidSize = 500e3;
+ const maxVidSize = 25e6;
+ const maxContentRange = 25e6;
+ const videoExtensions = [
+ 'af', '3gp', 'asf', 'avchd', 'avi', 'cam', 'dsh', 'flv', 'm1v', 'm2v',
+ 'fla', 'flr', 'sol', 'm4v', 'mkv', 'wrap', 'mng', 'mov', 'mpeg', 'mpg',
+ 'mpe', 'mp4', 'mxf', 'nsv', 'ogg', 'rm', 'svi', 'smi', 'wmv', 'webm',
+ ];
+ const extensionsReg = new RegExp(`\\.${videoExtensions.join('$|\\.')}$`);
+ const videoContentTypesPrefixes = ['binary/octet-stream', 'video/', 'flv-application/', 'media'];
+
+ const bannedContentTypes = ['video/mp2t', 'video/f4m', 'video/f4f'];
+ const bannedFiletypes = ['ts'];
+ const bannedFiletypesReg = new RegExp(`\\.${bannedFiletypes.join('$|\\.')}$`);
+ const whitelistReqTypes = ['object', 'xmlhttprequest', 'other'];
+
+ const topVideoAssetDomains = [
+ '2mdn.net',
+ 'adap.tv',
+ 'adnxs.com',
+ 'adsrvr.org',
+ 'btrll.com',
+ 'celtra.com',
+ 'flashtalking.com',
+ 'flite.com',
+ 'innovid.com',
+ 'jivox.com',
+ 'mixpo.com',
+ 'nytimes.com',
+ 'playwire.com',
+ 'selectmedia.asia',
+ 'serving-sys.com',
+ 'solvemedia.com',
+ 'spotible.com',
+ 'teads.tv',
+ 'tribalfusion.com',
+ 'tubemogul.com',
+ 'videologygroup.com',
+ 'washingtonpost.com',
+ ];
+
+ const robotsTxtAllows = wappalyzer.robotsTxtAllows.bind(wappalyzer);
+ if (!String.prototype.endsWith) {
+ String.prototype.endsWith = function (searchString, position) {
+ const subjectString = this.toString();
+ if (typeof position !== 'number' || !isFinite(position)
+ || Math.floor(position) !== position || position > subjectString.length) {
+ position = subjectString.length;
+ }
+ position -= searchString.length;
+ const lastIndex = subjectString.indexOf(searchString, position);
+ return lastIndex !== -1 && lastIndex === position;
+ };
+ }
+
+ function getFrame(getFrameDetails, callback) {
+ if (typeof chrome !== 'undefined') {
+ chrome.webNavigation.getFrame(getFrameDetails, callback);
+ } else if (typeof browser !== 'undefined') {
+ const gettingFrame = browser.webNavigation.getFrame(getFrameDetails);
+ gettingFrame.then(callback);
+ }
+ }
+
+ function ifBrowserValid(callback, elseCallback) {
+ if (isChrome()) {
+ callback();
+ } else if (typeof browser !== 'undefined') {
+ try {
+ const gettingInfo = browser.runtime.getBrowserInfo();
+ gettingInfo.then((browserInfo) => {
+ const browserVersion = parseInt(browserInfo.version.split('.')[0]);
+
+ if (browserInfo.name === 'Firefox'
+ && browserVersion >= MIN_FF_MAJOR_VERSION) {
+ callback();
+ } else {
+ elseCallback();
+ }
+ });
+ } catch (err) {
+ elseCallback();
+ }
+ } else {
+ elseCallback();
+ }
+ }
+
+ function ifTrackingEnabled(details, ifCallback, elseCallback) {
+ const fullIfCallback = function () {
+ allowedByRobotsTxt(details, ifCallback, elseCallback);
+ };
+
+ browser.storage.local.get('tracking').then((item) => {
+ if (item.hasOwnProperty('tracking')) {
+ if (item.tracking) {
+ fullIfCallback();
+ } else {
+ elseCallback();
+ }
+ } else {
+ fullIfCallback();
+ }
+ });
+ }
+
+ function allowedByRobotsTxt(details, ifCallback, elseCallback) {
+ if (details.url && !details.url.startsWith('chrome://')) {
+ robotsTxtAllows(details.url).then(ifCallback, elseCallback);
+ } else {
+ elseCallback();
+ }
+ }
+
+ function isPixelRequest(request) {
+ return (request.type === 'image' || request.responseStatus === 204)
+ && request.size <= 1000;
+ }
+
+ function isVpaidOrVastRequest(request) {
+ const lowerCaseUrl = request.url.toLowerCase();
+ return lowerCaseUrl.indexOf('vpaid') !== -1 || lowerCaseUrl.indexOf('vast') !== -1;
+ }
+
+ function hasValidRequestType(request) {
+ return whitelistReqTypes.indexOf(request.type) >= 0;
+ }
+
+ function stripQueryParams(url) {
+ return url.split('?', 1)[0];
+ }
+
+ function parseHostnameFromUrl(url) {
+ const parser = document.createElement('a');
+ parser.href = url;
+ return parser.hostname;
+ }
+
+ function hasDomain(url, domain) {
+ return parseHostnameFromUrl(url).endsWith(domain);
+ }
+
+ function findHeader(headers, key) {
+ let header;
+ for (let i = 0; i < headers.length; i += 1) {
+ header = headers[i];
+ if (header.name.toLowerCase() === key) {
+ return header;
+ }
+ }
+ return null;
+ }
+
+ function validVideoType(vtype) {
+ const goodType = videoContentTypesPrefixes.some(prefix => vtype.indexOf(prefix) === 0);
+ return goodType;
+ }
+
+ function assetMsgKey(assetReq) {
+ const url = stripQueryParams(assetReq.url);
+ const key = `${assetReq.frameId}-${url}`;
+ return key;
+ }
+
+ const PageNetworkTrafficCollector = function (tabId) {
+ this.tabId = tabId;
+ this.displayAdFound = false;
+ this.requests = {};
+ this.msgsBeingSent = {};
+ this.assetsSeen = {};
+ this.allRedirects = {};
+ };
+
+ var globalPageContainer = {
+ collectors: {},
+ dyingCollectors: {},
+
+ cleanupCollector(tabId) {
+ if (tabId in this.collectors) {
+ delete globalPageContainer.collectors[tabId];
+ }
+ },
+
+ onNewNavigation(details) {
+ const tabId = details.tabId;
+ this.cleanupCollector(tabId);
+
+ ifTrackingEnabled(
+ details,
+ () => {
+ if (!areListenersRegistered) {
+ registerListeners();
+ }
+ this.collectors[tabId] = new PageNetworkTrafficCollector(tabId);
+ },
+ () => {
+ if (areListenersRegistered) {
+ unregisterListeners();
+ }
+ },
+ );
+ },
+
+ onNavigationCommitted(details) {
+
+ },
+
+ onNavigationCompleted(details) {
+
+ },
+
+ onTabClose(tabId, closeInfo) {
+ this.cleanupCollector(tabId);
+ delete this.collectors[tabId];
+ },
+
+ onDisplayAdFound(tabId) {
+ this.collectors[tabId].displayAdFound = true;
+ },
+
+ getRandId() {
+ return String(Math.floor(Math.random() * 1e9));
+ },
+
+ getCollector(tabId) {
+ if (this.collectors.hasOwnProperty(tabId)) {
+ return this.collectors[tabId];
+ }
+ return null;
+ },
+
+ forwardCall(details, collectorMemberFunction) {
+ const collector = this.getCollector(details.tabId);
+ if (collector !== null) {
+ collectorMemberFunction.apply(collector, [details]);
+ }
+ },
+ };
+
+ PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function () {
+ const logMessage = Array.from(arguments).join(' ');
+ const message = { message: logMessage, event: 'console-log-message' };
+ browserProxy.tabs.sendMessage(this.tabId, message);
+ };
+
+ PageNetworkTrafficCollector.prototype.sendToTab = function (assetReq, reqs, curPageUrl, adTrackingEvent) {
+ const msg = {};
+ msg.assets = [];
+ msg.requests = [];
+ msg.event_data = {};
+ msg.event = adTrackingEvent;
+ if (adTrackingEvent === 'new-video-ad') {
+ msg.requests = reqs;
+ msg.requests.sort((reqA, reqB) => reqA.requestTimestamp - reqB.requestTimestamp);
+ if (assetReq) {
+ msg.assets = [assetReq];
+ }
+ } else if (adTrackingEvent === 'new-invalid-video-ad') {
+ msg.requests = reqs.map(request => parseHostnameFromUrl(request.url));
+ msg.assets = [{
+
+ url: parseHostnameFromUrl(assetReq.url),
+
+ contentType: assetReq.contentType,
+ size: assetReq.size,
+ }];
+ }
+ msg.origUrl = curPageUrl;
+ msg.displayAdFound = this.displayAdFound;
+
+ browserProxy.tabs.sendMessage(this.tabId, msg);
+ };
+
+ PageNetworkTrafficCollector.prototype.getRedirKey = function (url, frameId) {
+ return `${url}:${frameId}`;
+ };
+
+ PageNetworkTrafficCollector.prototype.seenBefore = function (request) {
+ const oldTime = this.assetsSeen[assetMsgKey(request)];
+ if (oldTime && (request.requestTimestamp - oldTime < secBetweenDupAssets)) {
+ return true;
+ }
+ return false;
+ };
+
+ PageNetworkTrafficCollector.prototype.recordSeenAsset = function (request) {
+ this.assetsSeen[assetMsgKey(request)] = request.requestTimestamp;
+ };
+
+ PageNetworkTrafficCollector.prototype.onBeforeRequest = function (details) {
+ const req = {
+ url: details.url,
+ type: details.type,
+ httpMethod: details.method,
+ frameId: details.frameId,
+ parentFrameId: details.parentFrameId,
+ requestTimestamp: details.timeStamp,
+ };
+ this.requests[details.requestId] = req;
+ };
+
+ PageNetworkTrafficCollector.prototype.onSendHeaders = function (details) {
+ let request,
+ header;
+ request = this.requests[details.requestId];
+ header = request && findHeader(details.requestHeaders, 'x-requested-with');
+ if (header && header.value.toLowerCase().indexOf('flash') > -1) {
+ request.from_flash = true;
+ }
+ };
+
+ PageNetworkTrafficCollector.prototype.onHeadersReceived = function (details) {
+ const getFrameDetails = {
+ tabId: details.tabId,
+ processId: null,
+ frameId: details.frameId,
+ };
+ const pageNetworkTrafficController = this;
+ getFrame(getFrameDetails, (frameDetails) => {
+ if (frameDetails && frameDetails.url) {
+ pageNetworkTrafficController._onHeadersReceived(details, frameDetails);
+ }
+ });
+ };
+
+ PageNetworkTrafficCollector.prototype._onHeadersReceived = function (details, frameDetails) {
+ let contentSize,
+ contentRange;
+
+ const request = this.requests[details.requestId];
+ if (request) {
+ const redirParent = this.allRedirects[this.getRedirKey(details.url, details.frameId)];
+ let header = request && findHeader(details.responseHeaders, 'content-type');
+ const contentType = header && header.value.toLowerCase();
+
+ if (contentType) {
+ request.contentType = contentType;
+ }
+ header = request && findHeader(details.responseHeaders, 'content-length');
+ contentSize = header && header.value;
+ if (contentSize) {
+ request.size = request.size || 0;
+ request.size += parseInt(contentSize);
+ }
+ header = request && findHeader(details.responseHeaders, 'content-range');
+ contentRange = header && header.value;
+ if (contentRange) {
+ request.contentRange = parseInt(contentRange.split('/')[1]);
+ }
+
+ let frameUrl = null;
+ if (frameDetails && frameDetails.url) {
+ frameUrl = frameDetails.url;
+ }
+ if (!this.bannedRequest(request)
+ && (this.isVideoReq(frameUrl, request) || (redirParent && redirParent.isVideo))) {
+ request.isVideo = true;
+ }
+ }
+ };
+
+ PageNetworkTrafficCollector.prototype.onBeforeRedirect = function (details) {
+ const request = this.requests[details.requestId];
+ if (request) {
+ if (request.redirects) {
+ request.redirects.push(details.redirectUrl);
+ } else {
+ request.redirects = [details.redirectUrl];
+ }
+ this.allRedirects[this.getRedirKey(details.redirectUrl, details.frameId)] = request;
+ }
+ };
+
+ PageNetworkTrafficCollector.prototype.isYoutubeMastheadRequest = function (url) {
+ const re = /video_masthead/;
+ return this.hasYoutubeDomain(url) && re.test(url);
+ };
+ PageNetworkTrafficCollector.prototype.isYoutubeVideoRequest = function (srcUrl, destUrl) {
+ if (!this.hasYoutubeDomain(srcUrl)) {
+ return false;
+ }
+
+ const re = /https?:\/\/r.*?\.googlevideo\.com\/videoplayback\?/;
+ return re.test(destUrl);
+ };
+ PageNetworkTrafficCollector.prototype.processResponse = function (requestDetails, frameDetails) {
+ let request;
+ if (requestDetails) {
+ request = this.requests[requestDetails.requestId];
+ if (request) {
+ request.responseStatus = requestDetails.statusCode;
+ request.responseTimestamp = requestDetails.timeStamp;
+
+ let frameUrl = null;
+ if (frameDetails && frameDetails.url) {
+ frameUrl = frameDetails.url;
+ }
+
+ let requestUrl = null;
+ if (request.url) {
+ requestUrl = request.url;
+ }
+
+ if (this.isYoutubeAdReq(frameUrl, requestUrl)) {
+ const destVideoId = this.parseYoutubeVideoIdFromUrl(requestUrl);
+ const srcVideoId = this.parseYoutubeVideoIdFromUrl(frameUrl);
+ if (srcVideoId && destVideoId) {
+ request.isYoutubeAd = true;
+ request.isVideo = true;
+ request.rawSrcUrl = frameUrl;
+ request.rawDestUrl = requestUrl;
+ request.url = `https://www.youtube.com/watch?v=${this.parseYoutubeVideoIdFromUrl(requestUrl)}`;
+ }
+ } else if (!this.bannedRequest(request)
+ && (this.isVideo || this.isVideoReq(frameUrl, request))) {
+ request.isVideo = true;
+ }
+
+ if (request.isVideo) {
+ const msgKey = assetMsgKey(request);
+ this.msgsBeingSent[msgKey] = request;
+ if (!this.seenBefore(request)) {
+ this.sendMsgWhenQuiet(msgKey);
+ }
+ this.recordSeenAsset(request);
+ }
+ }
+ }
+ };
+
+ PageNetworkTrafficCollector.prototype.onResponseStarted = function (responseDetails) {
+ if (responseDetails.frameId < 0) {
+ responseDetails.frameId = 99999;
+ }
+ const getFrameDetails = {
+ tabId: responseDetails.tabId,
+ processId: null,
+ frameId: responseDetails.frameId,
+ };
+ const pageNetworkTrafficController = this;
+ getFrame(getFrameDetails, (frameDetails) => {
+ if (frameDetails && frameDetails.url) {
+ pageNetworkTrafficController.processResponse(responseDetails, frameDetails);
+ }
+ });
+ };
+
+ PageNetworkTrafficCollector.prototype.hasBannedFiletype = function (request) {
+ const url = stripQueryParams(request.url);
+ if (bannedFiletypesReg.exec(url)) {
+ return true;
+ }
+ return false;
+ };
+
+ PageNetworkTrafficCollector.prototype.checkContentHeaders = function (request) {
+ if (request.contentType && validVideoType(request.contentType)) {
+ return true;
+ }
+ return false;
+ };
+
+ PageNetworkTrafficCollector.prototype.checkUrlExtension = function (request) {
+ const url = stripQueryParams(request.url);
+ if (extensionsReg.exec(url)) {
+ return true;
+ }
+ return false;
+ };
+
+ PageNetworkTrafficCollector.prototype.isVideoReq = function (srcUrl, request) {
+ if (this.isYoutubeVideoRequest(srcUrl, request.url)) {
+ return false;
+ }
+ return this.checkUrlExtension(request) || this.checkContentHeaders(request);
+ };
+ PageNetworkTrafficCollector.prototype.hasYoutubeDomain = function (url) {
+ const hostname = parseHostnameFromUrl(url);
+ if (hostname === 'www.youtube.com') {
+ return true;
+ }
+ return false;
+ };
+ PageNetworkTrafficCollector.prototype.parseYoutubeVideoIdFromUrl = function (url) {
+ let re = /^https?:\/\/www\.youtube\.com\/get_video_info.*(?:\?|&)video_id=(.*?)(?:$|&)/;
+ let match = re.exec(url);
+ if (match && match.length > 1) {
+ return match[1];
+ }
+
+ re = /^https?:\/\/www\.youtube\.com\/embed\/(.*?)(?:$|\?)/;
+ match = re.exec(url);
+ if (match && match.length > 1) {
+ return match[1];
+ }
+
+ re = /^https?:\/\/www\.youtube\.com\/watch.*(\?|&)v=([^&]*)/;
+ match = re.exec(url);
+ if (match && match.length > 1) {
+ return match[1];
+ }
+ return null;
+ };
+
+ PageNetworkTrafficCollector.prototype.isYoutubeGetVideoInfoReq = function (url) {
+ const re = /^https?:\/\/www\.youtube\.com\/get_video_info\?/;
+ return re.test(url);
+ };
+ PageNetworkTrafficCollector.prototype.isYoutubeAdReq = function (srcUrl, destUrl) {
+ if (!this.hasYoutubeDomain(srcUrl)
+ || !this.isYoutubeGetVideoInfoReq(destUrl)) {
+ return false;
+ }
+ if (this.parseYoutubeVideoIdFromUrl(srcUrl)
+ === this.parseYoutubeVideoIdFromUrl(destUrl)
+ && !this.isYoutubeMastheadRequest(destUrl)) {
+ return false;
+ }
+ return true;
+ };
+
+ PageNetworkTrafficCollector.prototype.bannedRequest = function (request) {
+ return this.bannedVideoType(request) || this.hasBannedFiletype(request) || this.bannedVideoSize(request);
+ };
+
+ PageNetworkTrafficCollector.prototype.bannedVideoType = function (request) {
+ let badType = false;
+ if (request.contentType) {
+ badType = bannedContentTypes.some(prefix => request.contentType.indexOf(prefix) >= 0);
+ }
+ return badType;
+ };
+
+ PageNetworkTrafficCollector.prototype.bannedVideoSize = function (request) {
+ if (request.size !== null) {
+ if (request.size < minVidSize || request.size > maxVidSize || request.contentRange > maxContentRange) {
+ return true;
+ }
+ }
+ return false;
+ };
+
+ PageNetworkTrafficCollector.prototype.grabTagReqs = function (tabRequests, assetRequest) {
+ let minTimestamp,
+ maxTimestamp;
+ minTimestamp = assetRequest.requestTimestamp - secBefore;
+ maxTimestamp = assetRequest.requestTimestamp + secAfter;
+
+ const filteredRequests = tabRequests.filter(request => (request.requestTimestamp > minTimestamp
+ && request.requestTimestamp < maxTimestamp
+ && request.frameId === assetRequest.frameId
+ && request.url !== assetRequest.url
+ && (hasValidRequestType(request)
+ || isPixelRequest(request))));
+
+ return filteredRequests;
+ };
+
+ PageNetworkTrafficCollector.prototype.isValidVideoAd = function (assetRequest, tagRequests) {
+ const hasVpaidOrVastRequest = tagRequests.some(tagRequest => isVpaidOrVastRequest(tagRequest));
+
+ if (assetRequest.isYoutubeAd) {
+ return true;
+ }
+ if (hasVpaidOrVastRequest) {
+ return true;
+ }
+ const hasTopVideoAssetDomain = topVideoAssetDomains.some(assetDomain => hasDomain(assetRequest.url, assetDomain));
+
+ return hasTopVideoAssetDomain;
+ };
+
+ PageNetworkTrafficCollector.prototype.sendMsgWhenQuiet = function (msgKey) {
+ let _this = this,
+ origPageUrl,
+ msgAssetReq;
+ msgAssetReq = this.msgsBeingSent[msgKey];
+ browserProxy.tabs.get(this.tabId, (tab) => { origPageUrl = tab.url; });
+
+ setTimeout(() => {
+ const rawRequests = [];
+ if (globalPageContainer.collectors[_this.tabId] === _this) {
+ for (const reqId in _this.requests) {
+ rawRequests.push(_this.requests[reqId]);
+ }
+ const tagReqs = _this.grabTagReqs(rawRequests, msgAssetReq);
+
+ if (_this.isValidVideoAd(msgAssetReq, tagReqs)) {
+ _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, 'new-video-ad');
+ } else {
+ _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, 'new-invalid-video-ad');
+ }
+ } else {
+
+ }
+ delete _this.msgsBeingSent[msgKey];
+ }, secAfter + secBefore);
+ };
+
+ PageNetworkTrafficCollector.prototype.existingMessage = function (candidateRequest) {
+ const frameMsg = this.msgsBeingSent[candidateRequest.frameId];
+ if (frameMsg) {
+ return frameMsg;
+ }
+ return null;
+ };
+
+ function onBeforeRequestListener(details) {
+ globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRequest);
+ }
+
+ function onSendHeadersListener(details) {
+ globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onSendHeaders);
+ }
+
+ function onHeadersReceivedListener(details) {
+ globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onHeadersReceived);
+ }
+
+ function onBeforeRedirectListener(details) {
+ globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRedirect);
+ }
+
+ function onResponseStartedListener(details) {
+ globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onResponseStarted);
+ }
+
+ function onCommittedListener(details) {
+ if (details.frameId === 0) {
+ globalPageContainer.onNavigationCommitted(details);
+ }
+ }
+
+ function onCompletedListener(details) {
+ if (details.frameId === 0) {
+ globalPageContainer.onNavigationCompleted(details);
+ }
+ }
+
+ function onRemovedListener(tabId, closeInfo) {
+ globalPageContainer.onTabClose(tabId, closeInfo);
+ }
+
+ function onMessageListener(message, sender, sendResponse) {
+ if (message.event === 'new-ad' && message.data.event === 'ad') {
+ const tabId = sender.tab.id;
+ if (tabId) {
+ globalPageContainer.onDisplayAdFound(tabId);
+ }
+ }
+ }
+
+ function registerListeners() {
+ browserProxy.webRequest.onBeforeRequest.addListener(
+ onBeforeRequestListener,
+ { urls: ['http://*/*', 'https://*/*'] },
+ [],
+ );
+
+ browserProxy.webRequest.onSendHeaders.addListener(
+ onSendHeadersListener,
+ { urls: ['http://*/*', 'https://*/*'] },
+ ['requestHeaders'],
+ );
+
+ browserProxy.webRequest.onHeadersReceived.addListener(
+ onHeadersReceivedListener,
+ { urls: ['http://*/*', 'https://*/*'] },
+ ['responseHeaders'],
+ );
+
+ browserProxy.webRequest.onBeforeRedirect.addListener(
+ onBeforeRedirectListener,
+ { urls: ['http://*/*', 'https://*/*'] },
+ [],
+ );
+
+ browserProxy.webRequest.onResponseStarted.addListener(
+ onResponseStartedListener,
+ { urls: ['http://*/*', 'https://*/*'] },
+ ['responseHeaders'],
+ );
+
+ browserProxy.webNavigation.onCommitted.addListener(onCommittedListener);
+ browserProxy.webNavigation.onCompleted.addListener(onCompletedListener);
+ browserProxy.tabs.onRemoved.addListener(onRemovedListener);
+ browserProxy.runtime.onMessage.addListener(onMessageListener);
+
+ areListenersRegistered = true;
+ }
+
+ function unregisterListeners() {
+ browserProxy.webRequest.onBeforeRequest.removeListener(
+ onBeforeRequestListener,
+ );
+
+ browserProxy.webRequest.onSendHeaders.removeListener(
+ onSendHeadersListener,
+ );
+
+ browserProxy.webRequest.onHeadersReceived.removeListener(
+ onHeadersReceivedListener,
+ );
+
+ browserProxy.webRequest.onBeforeRedirect.removeListener(
+ onBeforeRedirectListener,
+ );
+
+ browserProxy.webRequest.onResponseStarted.removeListener(
+ onResponseStartedListener,
+ );
+
+ browserProxy.webNavigation.onCommitted.removeListener(onCommittedListener);
+ browserProxy.webNavigation.onCompleted.removeListener(onCompletedListener);
+ browserProxy.tabs.onRemoved.removeListener(onRemovedListener);
+ browserProxy.runtime.onMessage.removeListener(onMessageListener);
+
+ areListenersRegistered = false;
+ }
+
+ function areRequiredBrowserApisAvailable() {
+ return requiredBrowserApis.every(api => typeof api !== 'undefined');
+ }
+
+ if (areRequiredBrowserApisAvailable()) {
+ ifBrowserValid(
+ () => {
+ browserProxy.webNavigation.onBeforeNavigate.addListener(
+ (details) => {
+ if (details.frameId === 0) {
+ globalPageContainer.onNewNavigation(details);
+ }
+ },
+ {
+ url: [{ urlMatches: 'http://*/*' }, { urlMatches: 'https://*/*' }],
+ },
+ );
+ }, () => {
+
+ },
+ );
+ }
+
+ browserProxy.runtime.onMessage.addListener((request, sender, sendResponse) => {
+ if (request === 'is_browser_valid') {
+ ifBrowserValid(
+ sendResponse({ browser_valid: true }),
+ sendResponse({ browser_valid: false }),
+ );
+ }
+ });
+
+ browserProxy.runtime.onMessage.addListener((request, sender, sendResponse) => {
+ if (request === 'is_tracking_enabled') {
+ ifTrackingEnabled(
+ sender.tab,
+ () => {
+ try { sendResponse({ tracking_enabled: true }); } catch (err) {}
+ },
+ () => {
+ try { sendResponse({ tracking_enabled: false }); } catch (err) {}
+ },
+ );
+ }
+ return true;
+ });
+}());
diff --git a/src/drivers/webextension/js/network.js b/src/drivers/webextension/js/network.js
deleted file mode 100644
index 6df8237ca..000000000
--- a/src/drivers/webextension/js/network.js
+++ /dev/null
@@ -1,824 +0,0 @@
-'use strict';
-(function() {
-
- function isChrome() {
- return (typeof chrome !== 'undefined' &&
- window.navigator.userAgent.match(/Chrom(?:e|ium)\/([0-9\.]+)/));
- }
-
- var browserProxy;
- if ( isChrome() ) {
- browserProxy = chrome;
- } else {
- browserProxy = browser;
- }
-
- var MIN_FF_MAJOR_VERSION = 51;
-
- var requiredBrowserApis = [
- browserProxy.webNavigation,
- browserProxy.tabs,
- browserProxy.webRequest,
- browserProxy.runtime
- ];
- var areListenersRegistered = false;
- var secBefore = 2000;
- var secAfter = 5000;
- var secBetweenDupAssets = 10e3;
- var minVidSize = 500e3;
- var maxVidSize = 25e6;
- var maxContentRange = 25e6;
- var videoExtensions = [
- 'af', '3gp', 'asf', 'avchd', 'avi', 'cam', 'dsh', 'flv', 'm1v', 'm2v',
- 'fla', 'flr', 'sol', 'm4v', 'mkv', 'wrap', 'mng', 'mov', 'mpeg', 'mpg',
- 'mpe', 'mp4', 'mxf', 'nsv', 'ogg', 'rm', 'svi', 'smi', 'wmv', 'webm'
- ];
- var extensionsReg = new RegExp('\\.' + videoExtensions.join('$|\\.') + '$');
- var videoContentTypesPrefixes = ['binary/octet-stream', 'video/', 'flv-application/', 'media'];
-
- var bannedContentTypes = ['video/mp2t','video/f4m','video/f4f'];
- var bannedFiletypes = ['ts'];
- var bannedFiletypesReg = new RegExp('\\.' + bannedFiletypes.join('$|\\.') + '$');
- var whitelistReqTypes = ['object', 'xmlhttprequest', 'other'];
-
- var topVideoAssetDomains = [
- '2mdn.net',
- 'adap.tv',
- 'adnxs.com',
- 'adsrvr.org',
- 'btrll.com',
- 'celtra.com',
- 'flashtalking.com',
- 'flite.com',
- 'innovid.com',
- 'jivox.com',
- 'mixpo.com',
- 'nytimes.com',
- 'playwire.com',
- 'selectmedia.asia',
- 'serving-sys.com',
- 'solvemedia.com',
- 'spotible.com',
- 'teads.tv',
- 'tribalfusion.com',
- 'tubemogul.com',
- 'videologygroup.com',
- 'washingtonpost.com'
- ];
-
- var robotsTxtAllows = wappalyzer.robotsTxtAllows.bind(wappalyzer);
- if ( !String.prototype.endsWith ) {
- String.prototype.endsWith = function(searchString, position) {
- var subjectString = this.toString();
- if ( typeof position !== 'number' || !isFinite(position) ||
- Math.floor(position) !== position || position > subjectString.length) {
- position = subjectString.length;
- }
- position -= searchString.length;
- var lastIndex = subjectString.indexOf(searchString, position);
- return lastIndex !== -1 && lastIndex === position;
- };
- }
-
- function getFrame(getFrameDetails, callback) {
- if ( typeof chrome !== 'undefined' ) {
- chrome.webNavigation.getFrame(getFrameDetails, callback);
- } else if ( typeof browser !== 'undefined' ) {
- var gettingFrame = browser.webNavigation.getFrame(getFrameDetails);
- gettingFrame.then(callback);
- }
- }
-
- function ifBrowserValid(callback, elseCallback) {
- if ( isChrome() ) {
-
- callback();
- } else if ( typeof browser !== 'undefined' ) {
- try {
- var gettingInfo = browser.runtime.getBrowserInfo();
- gettingInfo.then(function(browserInfo) {
- var browserVersion = parseInt(browserInfo.version.split('.')[0]);
-
- if ( browserInfo.name === 'Firefox' &&
- browserVersion >= MIN_FF_MAJOR_VERSION) {
- callback();
- } else {
- elseCallback();
- }
- });
- } catch (err) {
-
- elseCallback();
- }
- } else {
- elseCallback();
- }
- }
-
- function ifTrackingEnabled(details, ifCallback, elseCallback) {
-
- var fullIfCallback = function() {
- allowedByRobotsTxt(details, ifCallback, elseCallback);
- };
-
- browser.storage.local.get('tracking').then(function(item) {
-
- if ( item.hasOwnProperty('tracking') ) {
- if ( item.tracking ) {
- fullIfCallback();
- } else {
- elseCallback();
- }
- } else {
- fullIfCallback();
- }
- });
-
- }
-
- function allowedByRobotsTxt(details, ifCallback, elseCallback) {
- if ( details.url && !details.url.startsWith('chrome://') ) {
- robotsTxtAllows(details.url).then(ifCallback, elseCallback);
- } else {
- elseCallback();
- }
- }
-
- function isPixelRequest(request) {
- return (request.type === 'image' || request.responseStatus === 204) &&
- request.size <= 1000;
- }
-
- function isVpaidOrVastRequest(request) {
- var lowerCaseUrl = request.url.toLowerCase();
- return lowerCaseUrl.indexOf('vpaid') !== -1 || lowerCaseUrl.indexOf('vast') !== -1;
- }
-
- function hasValidRequestType(request) {
- return whitelistReqTypes.indexOf(request.type) >= 0;
- }
-
- function stripQueryParams(url) {
- return url.split('?', 1)[0];
- }
-
- function parseHostnameFromUrl(url) {
- var parser = document.createElement('a');
- parser.href = url;
- return parser.hostname;
- }
-
- function hasDomain(url, domain) {
- return parseHostnameFromUrl(url).endsWith(domain);
- }
-
- function findHeader(headers, key) {
- var header;
- for ( var i = 0; i < headers.length; i += 1 ) {
- header = headers[i];
- if ( header.name.toLowerCase() === key ) {
- return header;
- }
- }
- return null;
- }
-
- function validVideoType(vtype) {
- var goodType = videoContentTypesPrefixes.some(function(prefix) {
- return vtype.indexOf(prefix) === 0;
- });
- return goodType;
- }
-
- function assetMsgKey(assetReq) {
- var url = stripQueryParams(assetReq.url);
- var key = assetReq.frameId + '-' + url;
- return key;
- }
-
- var PageNetworkTrafficCollector = function(tabId) {
- this.tabId = tabId;
- this.displayAdFound = false;
- this.requests = {};
- this.msgsBeingSent = {};
- this.assetsSeen = {};
- this.allRedirects = {};
- };
-
- var globalPageContainer = {
- collectors: {},
- dyingCollectors: {},
-
- cleanupCollector: function(tabId) {
- if ( tabId in this.collectors ) {
- delete globalPageContainer.collectors[tabId];
- }
- },
-
- onNewNavigation: function(details) {
- var tabId = details.tabId;
- this.cleanupCollector(tabId);
-
- ifTrackingEnabled(
- details,
- function() {
- if ( !areListenersRegistered ) {
-
- registerListeners();
- }
- this.collectors[tabId] = new PageNetworkTrafficCollector(tabId);
- }.bind(this),
- function() {
- if ( areListenersRegistered ) {
-
- unregisterListeners();
- }
- }
- );
- },
-
- onNavigationCommitted: function(details) {
-
- },
-
- onNavigationCompleted: function(details) {
-
- },
-
- onTabClose: function(tabId, closeInfo) {
-
- this.cleanupCollector(tabId);
- delete this.collectors[tabId];
- },
-
- onDisplayAdFound: function(tabId) {
- this.collectors[tabId].displayAdFound = true;
- },
-
- getRandId: function() {
- return String(Math.floor(Math.random() * 1e9));
- },
-
- getCollector: function(tabId) {
- if ( this.collectors.hasOwnProperty(tabId) ) {
- return this.collectors[tabId];
- }
- return null;
- },
-
- forwardCall: function(details, collectorMemberFunction) {
- var collector = this.getCollector(details.tabId);
- if ( collector !== null ) {
- collectorMemberFunction.apply(collector, [details]);
- }
- }
- };
-
- PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() {
- var logMessage = Array.from(arguments).join(' ');
- var message = {message: logMessage, event: 'console-log-message'};
- browserProxy.tabs.sendMessage(this.tabId, message);
- };
-
- PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, adTrackingEvent) {
- var msg = {};
- msg.assets = [];
- msg.requests = [];
- msg.event_data = {};
- msg.event = adTrackingEvent;
- if ( adTrackingEvent === 'new-video-ad' ) {
- msg.requests = reqs;
- msg.requests.sort(function(reqA, reqB) {return reqA.requestTimestamp - reqB.requestTimestamp;});
- if ( assetReq ) {
- msg.assets = [assetReq];
- }
- } else if ( adTrackingEvent === 'new-invalid-video-ad' ) {
- msg.requests = reqs.map(function(request) {
- return parseHostnameFromUrl(request.url);
- });
- msg.assets = [{
-
- url: parseHostnameFromUrl(assetReq.url),
-
- contentType: assetReq.contentType,
- size: assetReq.size
- }];
- }
- msg.origUrl = curPageUrl;
- msg.displayAdFound = this.displayAdFound;
-
- browserProxy.tabs.sendMessage(this.tabId, msg);
- };
-
- PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) {
- return url + ':' + frameId;
- };
-
- PageNetworkTrafficCollector.prototype.seenBefore = function(request) {
- var oldTime = this.assetsSeen[assetMsgKey(request)];
- if ( oldTime && (request.requestTimestamp-oldTime < secBetweenDupAssets)){
-
- return true;
- }
- return false;
- };
-
- PageNetworkTrafficCollector.prototype.recordSeenAsset = function(request) {
- this.assetsSeen[assetMsgKey(request)] = request.requestTimestamp;
- };
-
- PageNetworkTrafficCollector.prototype.onBeforeRequest = function(details) {
- var req = {
- url: details.url,
- type: details.type,
- httpMethod: details.method,
- frameId: details.frameId,
- parentFrameId: details.parentFrameId,
- requestTimestamp: details.timeStamp,
- };
- this.requests[details.requestId] = req;
- };
-
- PageNetworkTrafficCollector.prototype.onSendHeaders = function(details) {
- var request, header;
- request = this.requests[details.requestId];
- header = request && findHeader(details.requestHeaders, 'x-requested-with');
- if ( header && header.value.toLowerCase().indexOf('flash') > -1 ) {
- request.from_flash = true;
- }
- };
-
- PageNetworkTrafficCollector.prototype.onHeadersReceived = function(details) {
- var getFrameDetails = {
- tabId: details.tabId,
- processId: null,
- frameId: details.frameId
- };
- var pageNetworkTrafficController = this;
- getFrame(getFrameDetails, function(frameDetails) {
- if ( frameDetails && frameDetails.url ) {
- pageNetworkTrafficController._onHeadersReceived(details, frameDetails);
- }
- });
- };
-
- PageNetworkTrafficCollector.prototype._onHeadersReceived = function(details, frameDetails) {
- var contentSize, contentRange;
-
- var request = this.requests[details.requestId];
- if ( request ) {
- var redirParent = this.allRedirects[this.getRedirKey(details.url, details.frameId)];
- var header = request && findHeader(details.responseHeaders, 'content-type');
- var contentType = header && header.value.toLowerCase();
-
- if ( contentType){
- request.contentType = contentType;
- }
- header = request && findHeader(details.responseHeaders, 'content-length');
- contentSize = header && header.value;
- if ( contentSize ) {
- request.size = request.size || 0;
- request.size += parseInt(contentSize);
- }
- header = request && findHeader(details.responseHeaders, 'content-range');
- contentRange = header && header.value;
- if ( contentRange ) {
- request.contentRange = parseInt(contentRange.split('/')[1]);
- }
-
- var frameUrl = null;
- if ( frameDetails && frameDetails.url ) {
- frameUrl = frameDetails.url;
- }
- if ( !this.bannedRequest(request) &&
- (this.isVideoReq(frameUrl, request) || (redirParent && redirParent.isVideo))) {
- request.isVideo = true;
- }
- }
- };
-
- PageNetworkTrafficCollector.prototype.onBeforeRedirect = function(details) {
- var request = this.requests[details.requestId];
- if ( request ) {
- if ( request.redirects ) {
- request.redirects.push(details.redirectUrl);
- } else {
- request.redirects = [details.redirectUrl];
- }
- this.allRedirects[this.getRedirKey(details.redirectUrl, details.frameId)] = request;
- }
- };
-
- PageNetworkTrafficCollector.prototype.isYoutubeMastheadRequest = function(url) {
- var re = /video_masthead/;
- return this.hasYoutubeDomain(url) && re.test(url);
- };
- PageNetworkTrafficCollector.prototype.isYoutubeVideoRequest = function(srcUrl, destUrl) {
- if ( !this.hasYoutubeDomain(srcUrl) ) {
- return false;
- }
-
- var re = /https?:\/\/r.*?\.googlevideo\.com\/videoplayback\?/;
- return re.test(destUrl);
- };
- PageNetworkTrafficCollector.prototype.processResponse = function(requestDetails, frameDetails) {
- var request;
- if ( requestDetails ) {
- request = this.requests[requestDetails.requestId];
- if ( request ) {
- request.responseStatus = requestDetails.statusCode;
- request.responseTimestamp = requestDetails.timeStamp;
-
- var frameUrl = null;
- if ( frameDetails && frameDetails.url ) {
- frameUrl = frameDetails.url;
- }
-
- var requestUrl = null;
- if ( request.url ) {
- requestUrl = request.url;
- }
-
- if ( this.isYoutubeAdReq(frameUrl, requestUrl) ) {
- var destVideoId = this.parseYoutubeVideoIdFromUrl(requestUrl);
- var srcVideoId = this.parseYoutubeVideoIdFromUrl(frameUrl);
- if ( srcVideoId && destVideoId ) {
- request.isYoutubeAd = true;
- request.isVideo = true;
- request.rawSrcUrl = frameUrl;
- request.rawDestUrl = requestUrl;
- request.url = 'https://www.youtube.com/watch?v=' + this.parseYoutubeVideoIdFromUrl(requestUrl);
- }
- } else if ( !this.bannedRequest(request) &&
- (this.isVideo || this.isVideoReq(frameUrl, request))) {
- request.isVideo = true;
- }
-
- if ( request.isVideo ) {
-
- var msgKey = assetMsgKey(request);
- this.msgsBeingSent[msgKey] = request;
- if ( !this.seenBefore(request) ) {
- this.sendMsgWhenQuiet(msgKey);
- }
- this.recordSeenAsset(request);
- }
- }
- }
- };
-
- PageNetworkTrafficCollector.prototype.onResponseStarted = function(responseDetails) {
- if ( responseDetails.frameId < 0 ) {
- responseDetails.frameId = 99999;
-
- }
- var getFrameDetails = {
- tabId: responseDetails.tabId,
- processId: null,
- frameId: responseDetails.frameId
- };
- var pageNetworkTrafficController = this;
- getFrame(getFrameDetails, function(frameDetails) {
- if ( frameDetails && frameDetails.url ) {
- pageNetworkTrafficController.processResponse(responseDetails, frameDetails);
- }
- });
- };
-
- PageNetworkTrafficCollector.prototype.hasBannedFiletype = function(request) {
- var url = stripQueryParams(request.url);
- if ( bannedFiletypesReg.exec(url) ) {
- return true;
- } else {
- return false;
- }
- };
-
- PageNetworkTrafficCollector.prototype.checkContentHeaders = function(request) {
- if ( request.contentType && validVideoType(request.contentType) ) {
- return true;
- }
- return false;
- };
-
- PageNetworkTrafficCollector.prototype.checkUrlExtension = function(request) {
- var url = stripQueryParams(request.url);
- if ( extensionsReg.exec(url) ) {
- return true;
- } else {
- return false;
- }
- };
-
- PageNetworkTrafficCollector.prototype.isVideoReq = function(srcUrl, request) {
- if ( this.isYoutubeVideoRequest(srcUrl, request.url) ) {
- return false;
- }
- return this.checkUrlExtension(request) || this.checkContentHeaders(request);
- };
- PageNetworkTrafficCollector.prototype.hasYoutubeDomain = function(url) {
- var hostname = parseHostnameFromUrl(url) ;
- if ( hostname === 'www.youtube.com' ) {
- return true;
- }
- return false;
- };
- PageNetworkTrafficCollector.prototype.parseYoutubeVideoIdFromUrl = function(url) {
- var re = /^https?:\/\/www\.youtube\.com\/get_video_info.*(?:\?|&)video_id=(.*?)(?:$|&)/;
- var match = re.exec(url);
- if ( match && match.length > 1 ) {
- return match[1];
- }
-
- re = /^https?:\/\/www\.youtube\.com\/embed\/(.*?)(?:$|\?)/;
- match = re.exec(url);
- if ( match && match.length > 1 ) {
- return match[1];
- }
-
- re = /^https?:\/\/www\.youtube\.com\/watch.*(\?|&)v=([^&]*)/;
- match = re.exec(url);
- if ( match && match.length > 1 ) {
- return match[1];
- }
- return null;
- };
-
- PageNetworkTrafficCollector.prototype.isYoutubeGetVideoInfoReq = function(url) {
- var re = /^https?:\/\/www\.youtube\.com\/get_video_info\?/;
- return re.test(url);
- };
- PageNetworkTrafficCollector.prototype.isYoutubeAdReq = function(srcUrl, destUrl) {
-
- if ( !this.hasYoutubeDomain(srcUrl) ||
- !this.isYoutubeGetVideoInfoReq(destUrl)) {
- return false;
- }
- if ( this.parseYoutubeVideoIdFromUrl(srcUrl) ===
- this.parseYoutubeVideoIdFromUrl(destUrl) &&
- !this.isYoutubeMastheadRequest(destUrl)) {
- return false;
- }
- return true;
- };
-
- PageNetworkTrafficCollector.prototype.bannedRequest = function(request) {
- return this.bannedVideoType(request) || this.hasBannedFiletype(request) || this.bannedVideoSize(request);
- };
-
- PageNetworkTrafficCollector.prototype.bannedVideoType = function(request) {
- var badType = false;
- if ( request.contentType ) {
- badType = bannedContentTypes.some(function(prefix) {
- return request.contentType.indexOf(prefix) >= 0;
- });
- }
- return badType;
- };
-
- PageNetworkTrafficCollector.prototype.bannedVideoSize = function(request) {
- if ( request.size !== null ) {
- if ( request.size < minVidSize || request.size > maxVidSize || request.contentRange > maxContentRange ) {
- return true;
- }
- }
- return false;
- };
-
- PageNetworkTrafficCollector.prototype.grabTagReqs = function(tabRequests, assetRequest) {
- var minTimestamp, maxTimestamp;
- minTimestamp = assetRequest.requestTimestamp - secBefore;
- maxTimestamp = assetRequest.requestTimestamp + secAfter;
-
- var filteredRequests = tabRequests.filter(function(request) {
- return (request.requestTimestamp > minTimestamp &&
- request.requestTimestamp < maxTimestamp &&
- request.frameId === assetRequest.frameId &&
- request.url !== assetRequest.url &&
- (hasValidRequestType(request) ||
- isPixelRequest(request)));
- });
-
- return filteredRequests;
- };
-
- PageNetworkTrafficCollector.prototype.isValidVideoAd = function(assetRequest, tagRequests) {
- var hasVpaidOrVastRequest = tagRequests.some(function(tagRequest) {
- return isVpaidOrVastRequest(tagRequest);
- });
-
- if ( assetRequest.isYoutubeAd ) {
- return true;
- }
- if ( hasVpaidOrVastRequest ) {
- return true;
- }
- var hasTopVideoAssetDomain = topVideoAssetDomains.some(function(assetDomain) {
- return hasDomain(assetRequest.url, assetDomain);
- });
-
- return hasTopVideoAssetDomain;
- };
-
- PageNetworkTrafficCollector.prototype.sendMsgWhenQuiet = function(msgKey) {
- var _this = this,
- origPageUrl, msgAssetReq;
- msgAssetReq = this.msgsBeingSent[msgKey];
- browserProxy.tabs.get(this.tabId, function(tab) {origPageUrl = tab.url;});
-
- setTimeout(function() {
- var rawRequests = [];
- if ( globalPageContainer.collectors[_this.tabId] === _this ) {
- for ( var reqId in _this.requests ) {
- rawRequests.push(_this.requests[reqId]);
- }
- var tagReqs = _this.grabTagReqs(rawRequests, msgAssetReq);
-
- if ( _this.isValidVideoAd(msgAssetReq, tagReqs) ) {
- _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, 'new-video-ad');
- } else {
-
- _this.sendToTab(msgAssetReq, tagReqs, origPageUrl, 'new-invalid-video-ad');
- }
-
- } else {
-
- }
- delete _this.msgsBeingSent[msgKey];
- }, secAfter+secBefore);
- };
-
- PageNetworkTrafficCollector.prototype.existingMessage = function(candidateRequest) {
- var frameMsg = this.msgsBeingSent[candidateRequest.frameId];
- if ( frameMsg ) {
- return frameMsg;
- } else {
- return null;
- }
- };
-
- function onBeforeRequestListener(details) {
- globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRequest);
- }
-
- function onSendHeadersListener(details) {
- globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onSendHeaders);
- }
-
- function onHeadersReceivedListener(details) {
- globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onHeadersReceived);
- }
-
- function onBeforeRedirectListener(details) {
- globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRedirect);
- }
-
- function onResponseStartedListener(details) {
- globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onResponseStarted);
- }
-
- function onCommittedListener(details) {
- if ( details.frameId === 0 ) {
- globalPageContainer.onNavigationCommitted(details);
- }
- }
-
- function onCompletedListener(details) {
- if ( details.frameId === 0 ) {
- globalPageContainer.onNavigationCompleted(details);
- }
- }
-
- function onRemovedListener(tabId, closeInfo) {
- globalPageContainer.onTabClose(tabId, closeInfo);
- }
-
- function onMessageListener(message, sender, sendResponse) {
- if ( message.event === 'new-ad' && message.data.event === 'ad' ) {
- var tabId = sender.tab.id;
- if ( tabId ) {
- globalPageContainer.onDisplayAdFound(tabId);
- }
- }
- }
-
- function registerListeners() {
-
- browserProxy.webRequest.onBeforeRequest.addListener(
- onBeforeRequestListener,
- {urls: ['http://*/*', 'https://*/*']},
- []
- );
-
- browserProxy.webRequest.onSendHeaders.addListener(
- onSendHeadersListener,
- {urls: ['http://*/*', 'https://*/*']},
- ['requestHeaders']
- );
-
- browserProxy.webRequest.onHeadersReceived.addListener(
- onHeadersReceivedListener,
- {urls: ['http://*/*', 'https://*/*']},
- ['responseHeaders']
- );
-
- browserProxy.webRequest.onBeforeRedirect.addListener(
- onBeforeRedirectListener,
- {urls: ['http://*/*', 'https://*/*']},
- []
- );
-
- browserProxy.webRequest.onResponseStarted.addListener(
- onResponseStartedListener,
- {urls: ['http://*/*', 'https://*/*']},
- ['responseHeaders']
- );
-
- browserProxy.webNavigation.onCommitted.addListener(onCommittedListener);
- browserProxy.webNavigation.onCompleted.addListener(onCompletedListener);
- browserProxy.tabs.onRemoved.addListener(onRemovedListener);
- browserProxy.runtime.onMessage.addListener(onMessageListener);
-
- areListenersRegistered = true;
- }
-
- function unregisterListeners() {
-
- browserProxy.webRequest.onBeforeRequest.removeListener(
- onBeforeRequestListener
- );
-
- browserProxy.webRequest.onSendHeaders.removeListener(
- onSendHeadersListener
- );
-
- browserProxy.webRequest.onHeadersReceived.removeListener(
- onHeadersReceivedListener
- );
-
- browserProxy.webRequest.onBeforeRedirect.removeListener(
- onBeforeRedirectListener
- );
-
- browserProxy.webRequest.onResponseStarted.removeListener(
- onResponseStartedListener
- );
-
- browserProxy.webNavigation.onCommitted.removeListener(onCommittedListener);
- browserProxy.webNavigation.onCompleted.removeListener(onCompletedListener);
- browserProxy.tabs.onRemoved.removeListener(onRemovedListener);
- browserProxy.runtime.onMessage.removeListener(onMessageListener);
-
- areListenersRegistered = false;
- }
-
- function areRequiredBrowserApisAvailable() {
- return requiredBrowserApis.every(function(api) {
- return typeof api !== 'undefined';
- });
- }
-
- if ( areRequiredBrowserApisAvailable() ) {
- ifBrowserValid(
- function() {
- browserProxy.webNavigation.onBeforeNavigate.addListener(
- function(details) {
- if ( details.frameId === 0 ) {
- globalPageContainer.onNewNavigation(details);
- }
- },
- {
- url: [{urlMatches: 'http://*/*'}, {urlMatches: 'https://*/*'}]
- }
- );
- }, function() {
-
- }
- );
- }
-
- browserProxy.runtime.onMessage.addListener(function(request, sender, sendResponse) {
- if ( request === 'is_browser_valid' ) {
- ifBrowserValid(
- sendResponse({'browser_valid': true}),
- sendResponse({'browser_valid': false})
- );
- }
- });
-
- browserProxy.runtime.onMessage.addListener(function(request, sender, sendResponse) {
- if ( request === 'is_tracking_enabled' ) {
- ifTrackingEnabled(
- sender.tab,
- function() {
- try {sendResponse({'tracking_enabled': true});}
- catch(err) {} },
- function() {
- try {sendResponse({'tracking_enabled': false});}
- catch(err) {}}
- );
- }
- return true;
- });
-
-})();
diff --git a/src/drivers/webextension/js/options.js b/src/drivers/webextension/js/options.js
index 4c5d7cc67..935d11318 100644
--- a/src/drivers/webextension/js/options.js
+++ b/src/drivers/webextension/js/options.js
@@ -5,23 +5,23 @@ const wappalyzer = new Wappalyzer();
function getOption(name, defaultValue, callback) {
browser.storage.local.get(name)
- .then(item => {
+ .then((item) => {
callback(item.hasOwnProperty(name) ? item[name] : defaultValue);
});
}
function setOption(name, value) {
- ( chrome || browser ).runtime.sendMessage({
+ (chrome || browser).runtime.sendMessage({
id: 'set_option',
key: name,
- value: value
+ value,
});
}
document.addEventListener('DOMContentLoaded', () => {
- var nodes = document.querySelectorAll('[data-i18n]');
+ const nodes = document.querySelectorAll('[data-i18n]');
- Array.prototype.forEach.call(nodes, node => {
+ Array.prototype.forEach.call(nodes, (node) => {
node.childNodes[0].nodeValue = browser.i18n.getMessage(node.dataset.i18n);
});
@@ -37,7 +37,7 @@ document.addEventListener('DOMContentLoaded', () => {
open(wappalyzer.config.websiteURL);
});
- getOption('upgradeMessage', true, value => {
+ getOption('upgradeMessage', true, (value) => {
const el = document.querySelector('#option-upgrade-message');
el.checked = value;
@@ -47,7 +47,7 @@ document.addEventListener('DOMContentLoaded', () => {
});
});
- getOption('dynamicIcon', true, value => {
+ getOption('dynamicIcon', true, (value) => {
const el = document.querySelector('#option-dynamic-icon');
el.checked = value;
@@ -57,7 +57,7 @@ document.addEventListener('DOMContentLoaded', () => {
});
});
- getOption('tracking', true, value => {
+ getOption('tracking', true, (value) => {
const el = document.querySelector('#option-tracking');
el.checked = value;
diff --git a/src/drivers/webextension/js/popup.js b/src/drivers/webextension/js/popup.js
index e846dd0f5..e560162fd 100644
--- a/src/drivers/webextension/js/popup.js
+++ b/src/drivers/webextension/js/popup.js
@@ -1,14 +1,14 @@
/** global: chrome */
/** global: browser */
-var pinnedCategory = null;
+let pinnedCategory = null;
-var func = tabs => {
- ( chrome || browser ).runtime.sendMessage({
+const func = (tabs) => {
+ (chrome || browser).runtime.sendMessage({
id: 'get_apps',
tab: tabs[0],
- source: 'popup.js'
- }, response => {
+ source: 'popup.js',
+ }, (response) => {
pinnedCategory = response.pinnedCategory;
replaceDomWhenReady(appsToDomTemplate(response));
@@ -20,7 +20,7 @@ browser.tabs.query({ active: true, currentWindow: true })
.catch(console.error);
function replaceDomWhenReady(dom) {
- if ( /complete|interactive|loaded/.test(document.readyState) ) {
+ if (/complete|interactive|loaded/.test(document.readyState)) {
replaceDom(dom);
} else {
document.addEventListener('DOMContentLoaded', () => {
@@ -30,32 +30,32 @@ function replaceDomWhenReady(dom) {
}
function replaceDom(domTemplate) {
- var container = document.getElementsByClassName('container')[0];
+ const container = document.getElementsByClassName('container')[0];
- while ( container.firstChild ) {
+ while (container.firstChild) {
container.removeChild(container.firstChild);
}
container.appendChild(jsonToDOM(domTemplate, document, {}));
- var nodes = document.querySelectorAll('[data-i18n]');
+ const nodes = document.querySelectorAll('[data-i18n]');
- Array.prototype.forEach.call(nodes, node => {
+ Array.prototype.forEach.call(nodes, (node) => {
node.childNodes[0].nodeValue = browser.i18n.getMessage(node.dataset.i18n);
});
- Array.from(document.querySelectorAll('.detected__category-pin-wrapper')).forEach(pin => {
+ Array.from(document.querySelectorAll('.detected__category-pin-wrapper')).forEach((pin) => {
pin.addEventListener('click', () => {
const categoryId = parseInt(pin.dataset.categoryId, 10);
- if ( categoryId === pinnedCategory ) {
+ if (categoryId === pinnedCategory) {
pin.className = 'detected__category-pin-wrapper';
pinnedCategory = null;
} else {
const active = document.querySelector('.detected__category-pin-wrapper--active');
- if ( active ) {
+ if (active) {
active.className = 'detected__category-pin-wrapper';
}
@@ -64,7 +64,7 @@ function replaceDom(domTemplate) {
pinnedCategory = categoryId;
}
- ( chrome || browser ).runtime.sendMessage({
+ (chrome || browser).runtime.sendMessage({
id: 'set_option',
key: 'pinnedCategory',
value: pinnedCategory,
@@ -74,115 +74,115 @@ function replaceDom(domTemplate) {
}
function appsToDomTemplate(response) {
- let template = [];
+ let template = [];
- if ( response.tabCache && Object.keys(response.tabCache.detected).length > 0 ) {
+ if (response.tabCache && Object.keys(response.tabCache.detected).length > 0) {
const categories = {};
// Group apps by category
- for ( let appName in response.tabCache.detected ) {
- response.apps[appName].cats.forEach(cat => {
+ for (const appName in response.tabCache.detected) {
+ response.apps[appName].cats.forEach((cat) => {
categories[cat] = categories[cat] || { apps: [] };
categories[cat].apps[appName] = appName;
});
}
- for ( let cat in categories ) {
+ for (const cat in categories) {
const apps = [];
- for ( let appName in categories[cat].apps ) {
- let confidence = response.tabCache.detected[appName].confidenceTotal;
- let version = response.tabCache.detected[appName].version;
+ for (const appName in categories[cat].apps) {
+ const confidence = response.tabCache.detected[appName].confidenceTotal;
+ const version = response.tabCache.detected[appName].version;
apps.push(
[
'a', {
class: 'detected__app',
target: '_blank',
- href: 'https://www.wappalyzer.com/technologies/' + slugify(appName)
+ href: `https://www.wappalyzer.com/technologies/${slugify(appName)}`,
}, [
'img', {
class: 'detected__app-icon',
- src: '../images/icons/' + (response.apps[appName].icon || 'default.svg')
+ src: `../images/icons/${response.apps[appName].icon || 'default.svg'}`,
},
], [
'span', {
- class: 'detected__app-name'
+ class: 'detected__app-name',
},
appName,
], version ? [
'span', {
- class: 'detected__app-version'
+ class: 'detected__app-version',
},
- version
+ version,
] : null, confidence < 100 ? [
'span', {
- class: 'detected__app-confidence'
+ class: 'detected__app-confidence',
},
- confidence + '% sure'
- ] : null
- ]
+ `${confidence}% sure`,
+ ] : null,
+ ],
);
}
template.push(
[
'div', {
- class: 'detected__category'
+ class: 'detected__category',
}, [
'div', {
- class: 'detected__category-name'
+ class: 'detected__category-name',
}, [
'a', {
class: 'detected__category-link',
target: '_blank',
- href: 'https://www.wappalyzer.com/categories/' + slugify(response.categories[cat].name)
+ href: `https://www.wappalyzer.com/categories/${slugify(response.categories[cat].name)}`,
},
- browser.i18n.getMessage('categoryName' + cat),
+ browser.i18n.getMessage(`categoryName${cat}`),
], [
'span', {
- class: 'detected__category-pin-wrapper' + ( pinnedCategory == cat ? ' detected__category-pin-wrapper--active' : '' ),
+ class: `detected__category-pin-wrapper${pinnedCategory == cat ? ' detected__category-pin-wrapper--active' : ''}`,
'data-category-id': cat,
- 'title': browser.i18n.getMessage('categoryPin'),
+ title: browser.i18n.getMessage('categoryPin'),
}, [
'img', {
class: 'detected__category-pin detected__category-pin--active',
- src: '../images/pin-active.svg'
+ src: '../images/pin-active.svg',
},
], [
'img', {
class: 'detected__category-pin detected__category-pin--inactive',
- src: '../images/pin.svg'
- }
- ]
- ]
+ src: '../images/pin.svg',
+ },
+ ],
+ ],
], [
'div', {
- class: 'detected__apps'
+ class: 'detected__apps',
},
- apps
- ]
- ]
+ apps,
+ ],
+ ],
);
}
template = [
'div', {
- class: 'detected'
+ class: 'detected',
},
- template
+ template,
];
} else {
template = [
'div', {
- class: 'empty'
+ class: 'empty',
},
[
'span', {
- class: 'empty__text'
+ class: 'empty__text',
},
- browser.i18n.getMessage('noAppsDetected')
+ browser.i18n.getMessage('noAppsDetected'),
],
];
}
diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json
index 2880e3aea..4164e1195 100644
--- a/src/drivers/webextension/manifest.json
+++ b/src/drivers/webextension/manifest.json
@@ -51,7 +51,7 @@
"https://www.alphaecommerce.gr/*"
],
"js": [
- "js/iframe.js"
+ "lib/js/iframe.js"
],
"run_at": "document_start",
"all_frames": true
diff --git a/src/wappalyzer.js b/src/wappalyzer.js
index e117c6121..b83240533 100644
--- a/src/wappalyzer.js
+++ b/src/wappalyzer.js
@@ -6,13 +6,116 @@
* License: GPLv3 http://www.gnu.org/licenses/gpl-3.0.txt
*/
-'use strict';
-
const validation = {
hostname: /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/,
- hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/
+ hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/,
};
+/**
+ * Enclose string in array
+ */
+function asArray(value) {
+ return value instanceof Array ? value : [value];
+}
+
+/**
+ *
+ */
+function asyncForEach(iterable, iterator) {
+ return Promise.all((iterable || [])
+ .map(item => new Promise(resolve => setTimeout(() => resolve(iterator(item)), 1))));
+}
+
+/**
+ * Mark application as detected, set confidence and version
+ */
+function addDetected(app, pattern, type, value, key) {
+ app.detected = true;
+
+ // Set confidence level
+ app.confidence[`${type} ${key ? `${key} ` : ''}${pattern.regex}`] = pattern.confidence === undefined ? 100 : parseInt(pattern.confidence, 10);
+
+ // Detect version number
+ if (pattern.version) {
+ const versions = [];
+ const matches = pattern.regex.exec(value);
+
+ let { version } = pattern;
+
+ if (matches) {
+ matches.forEach((match, i) => {
+ // Parse ternary operator
+ const ternary = new RegExp(`\\\\${i}\\?([^:]+):(.*)$`).exec(version);
+
+ if (ternary && ternary.length === 3) {
+ version = version.replace(ternary[0], match ? ternary[1] : ternary[2]);
+ }
+
+ // Replace back references
+ version = version.trim().replace(new RegExp(`\\\\${i}`, 'g'), match || '');
+ });
+
+ if (version && versions.indexOf(version) === -1) {
+ versions.push(version);
+ }
+
+ if (versions.length) {
+ // Use the longest detected version number
+ app.version = versions.reduce((a, b) => (a.length > b.length ? a : b));
+ }
+ }
+ }
+}
+
+function resolveExcludes(apps, detected) {
+ const excludes = [];
+
+ // Exclude app in detected apps only
+ Object.keys(Object.assign({}, apps, detected)).forEach((appName) => {
+ const app = apps[appName];
+
+ if (app.props.excludes) {
+ asArray(app.props.excludes).forEach((excluded) => {
+ excludes.push(excluded);
+ });
+ }
+ });
+
+ // Remove excluded applications
+ Object.keys(apps).forEach((appName) => {
+ if (excludes.indexOf(appName) > -1) {
+ delete apps[appName];
+ }
+ });
+}
+
+class Application {
+ constructor(name, props, detected) {
+ this.confidence = {};
+ this.confidenceTotal = 0;
+ this.detected = Boolean(detected);
+ this.excludes = [];
+ this.name = name;
+ this.props = props;
+ this.version = '';
+ }
+
+ /**
+ * Calculate confidence total
+ */
+ getConfidence() {
+ let total = 0;
+
+ Object.values(this.confidence).forEach((id) => {
+ total += this.confidence[id];
+ });
+
+ this.confidenceTotal = Math.min(total, 100);
+
+ return this.confidenceTotal;
+ }
+}
+
class Wappalyzer {
constructor() {
this.apps = {};
@@ -37,27 +140,29 @@ class Wappalyzer {
this.driver.log(message, source || '', type || 'debug');
}
- asyncForEach(iterable, iterator) {
- return Promise.all(( iterable || [] ).map(item => new Promise(resolve => setTimeout(() => resolve(iterator(item)), 1))));
- }
-
analyze(url, data, context) {
- const startTime = new Date();
-
+ const apps = {};
const promises = [];
+ const startTime = new Date();
+ const {
+ scripts,
+ cookies,
+ headers,
+ js,
+ } = data;
- var apps = {};
+ let { html } = data;
- if ( this.detected[url.canonical] === undefined ) {
+ if (this.detected[url.canonical] === undefined) {
this.detected[url.canonical] = {};
}
// Additional information
- var language = null;
+ let language = null;
- if ( data.html ) {
- if ( typeof data.html !== 'string' ) {
- data.html = '';
+ if (html) {
+ if (typeof html !== 'string') {
+ html = '';
}
const matches = data.html.match(/ ${app
+ }${
+ version ? ` ${version}` : ''}${confidence < 100 ? ` (${confidence}% sure)` : ''
+ }`;
+
+ for (const i in wappalyzer.apps[app].cats) {
+ if (!hasOwn.call(wappalyzer.apps[app].cats, i)) {
continue;
}
category = wappalyzer.categories[wappalyzer.apps[app].cats[i]].name;
- html += '' + category + '';
+ html += `${category}`;
}
html += '