Elbert Alias
8 years ago
@ -1,3 +0,0 @@
|
|||||||
# Contributing
|
|
||||||
|
|
||||||
You can find our contribution guide [in our wiki](https://github.com/AliasIO/Wappalyzer/wiki/Contributing).
|
|
||||||
@ -0,0 +1,32 @@
|
|||||||
|
FROM alpine
|
||||||
|
|
||||||
|
MAINTAINER Elbert Alias <elbert@alias.io>
|
||||||
|
|
||||||
|
ENV WAPPALYZER_DIR=/opt/wappalyzer
|
||||||
|
|
||||||
|
RUN apk add --no-cache \
|
||||||
|
bash \
|
||||||
|
curl \
|
||||||
|
fontconfig \
|
||||||
|
nodejs \
|
||||||
|
optipng \
|
||||||
|
zip
|
||||||
|
|
||||||
|
RUN mkdir -p /usr/share && \
|
||||||
|
cd /usr/share \
|
||||||
|
&& curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 | tar xj \
|
||||||
|
&& ln -s /usr/share/phantomjs/phantomjs /usr/bin/phantomjs
|
||||||
|
|
||||||
|
RUN apk del \
|
||||||
|
curl
|
||||||
|
|
||||||
|
RUN npm i -g \
|
||||||
|
jsonlint-cli \
|
||||||
|
manifoldjs \
|
||||||
|
svg2png-many
|
||||||
|
|
||||||
|
RUN mkdir -p $WAPPALYZER_DIR
|
||||||
|
|
||||||
|
WORKDIR $WAPPALYZER_DIR
|
||||||
|
|
||||||
|
CMD [ "./bin/run" ]
|
||||||
@ -1,7 +1,7 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
cat << 'EOF'
|
cat << 'EOF'
|
||||||
usage: wappalyzer <command> [<args>]
|
usage: ./run <command> [<args>]
|
||||||
|
|
||||||
The following commands are available:
|
The following commands are available:
|
||||||
build Package drivers
|
build Package drivers
|
||||||
@ -0,0 +1,14 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
cd "$(dirname $0)/../src"
|
||||||
|
|
||||||
|
echo "Creating links..."
|
||||||
|
|
||||||
|
ln -nsf ../../apps.json drivers/npm
|
||||||
|
ln -nsf ../../wappalyzer.js drivers/npm
|
||||||
|
|
||||||
|
ln -nsf ../../apps.json drivers/webextension
|
||||||
|
ln -nsf ../../../wappalyzer.js drivers/webextension/js
|
||||||
|
ln -nsf ../../../icons/ drivers/webextension/images/icons
|
||||||
|
|
||||||
|
echo "Done."
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
cd "$(dirname $0)"
|
||||||
|
|
||||||
|
if [[ -f "$1" ]]; then
|
||||||
|
./$1 $@
|
||||||
|
else
|
||||||
|
./help
|
||||||
|
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
@ -0,0 +1,17 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
cd "$(dirname $0)/.."
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
echo "Validating apps.json..."
|
||||||
|
|
||||||
|
jsonlint-cli -s schema.json src/apps.json
|
||||||
|
|
||||||
|
echo "Validating regular expressions..."
|
||||||
|
|
||||||
|
./bin/validate-regex
|
||||||
|
|
||||||
|
echo "Validating icons..."
|
||||||
|
|
||||||
|
./bin/validate-icons
|
||||||
@ -0,0 +1,43 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
var
|
||||||
|
app,
|
||||||
|
fs = require('fs'),
|
||||||
|
fileType = require('../node_modules/file-type'),
|
||||||
|
readChunk = require('../node_modules/read-chunk')
|
||||||
|
isSvg = require('../node_modules/is-svg')
|
||||||
|
json = require('../src/apps.json');
|
||||||
|
|
||||||
|
for (app in json.apps) {
|
||||||
|
(function(app) {
|
||||||
|
var
|
||||||
|
basePath = 'src/icons/';
|
||||||
|
iconPath = json.apps[app].icon || 'default.svg';
|
||||||
|
path = basePath + iconPath,
|
||||||
|
ext = iconPath.substr(iconPath.length - 4);
|
||||||
|
|
||||||
|
if ( ext !== '.png' && ext !== '.svg' ) {
|
||||||
|
throw err = new Error('Icon file extension specified for app "' + app + '" is not ".png" or ".svg": src/icons/' + iconPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.exists(path, function(exists) {
|
||||||
|
if ( exists ) {
|
||||||
|
if ( ext === '.png' ) {
|
||||||
|
var buffer = fileType(readChunk.sync(path, 0, 262));
|
||||||
|
|
||||||
|
if ( buffer === null ) {
|
||||||
|
throw new Error('Unknown mimetype or bad file for "' + app + '": src/icons/' + iconPath);
|
||||||
|
} else if ( buffer.mime !== 'image/png' ) {
|
||||||
|
throw new Error('Incorrect mimetype "' + buffer.mime + '" when expected PNG for app "' + app + '": src/icons/' + iconPath);
|
||||||
|
}
|
||||||
|
} else if ( type === 'svg' ) {
|
||||||
|
if ( !isSvg(fs.readFileSync(path)) ) {
|
||||||
|
throw new Error('Incorrect mimetype when expected SVG for app "' + app + '": src/icons/' + iconPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
throw Error('Missing file for app "' + app + '": src/icons/' + iconPath);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}(app));
|
||||||
|
}
|
||||||
@ -1,12 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
if [ -f "$0-$1" ]
|
|
||||||
then
|
|
||||||
$0-$1 ${*:2}
|
|
||||||
else
|
|
||||||
$0-help
|
|
||||||
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
exit
|
|
||||||
@ -1,45 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
path=$1
|
|
||||||
|
|
||||||
if [ -z $path ]
|
|
||||||
then
|
|
||||||
if [ -z $WAPPALYZER_ROOT ]
|
|
||||||
then
|
|
||||||
echo "-$(basename $0): No path specified"
|
|
||||||
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
path=$WAPPALYZER_ROOT
|
|
||||||
fi
|
|
||||||
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
if [ ! -d $path/src ]
|
|
||||||
then
|
|
||||||
echo "-$(basename $0): Incorrect path"
|
|
||||||
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
path="$path/src"
|
|
||||||
|
|
||||||
echo "Creating hard links..."
|
|
||||||
|
|
||||||
ln -f $path/wappalyzer.js $path/drivers/phantomjs
|
|
||||||
ln -f $path/apps.json $path/drivers/phantomjs
|
|
||||||
|
|
||||||
ln -f $path/wappalyzer.js $path/drivers/webextension/js
|
|
||||||
ln -f $path/apps.json $path/drivers/webextension
|
|
||||||
ln -f $path/icons/*.png $path/drivers/webextension/images/icons
|
|
||||||
ln -f $path/icons/*.svg $path/drivers/webextension/images/icons
|
|
||||||
ln -f $path/utils/*.js $path/drivers/webextension/js
|
|
||||||
|
|
||||||
if [ "$(compgen -G "$path/icons/converted/*.png" | head -n1)" ]; then
|
|
||||||
ln -f $path/icons/converted/*.png $path/drivers/webextension/images/icons/converted
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "OK"
|
|
||||||
|
|
||||||
exit 0
|
|
||||||
@ -1,35 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
path="$1"
|
|
||||||
|
|
||||||
if [ -z "$path" ]; then
|
|
||||||
if [ -z "$WAPPALYZER_ROOT" ]; then
|
|
||||||
echo "-$(basename $0): No path specified"
|
|
||||||
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
path="$WAPPALYZER_ROOT"
|
|
||||||
fi
|
|
||||||
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
if [ ! -d "$path/src" ]; then
|
|
||||||
echo "-$(basename $0): Incorrect path"
|
|
||||||
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
path="$path/src"
|
|
||||||
|
|
||||||
echo "Validating apps.json..."
|
|
||||||
|
|
||||||
node "$WAPPALYZER_NODE_PATH/node_modules/jsonlint/lib/cli.js" --quiet -V "$WAPPALYZER_ROOT/schema.json" "$path/apps.json"
|
|
||||||
|
|
||||||
echo "Validating regular expressions..."
|
|
||||||
|
|
||||||
wappalyzer validate-regex
|
|
||||||
|
|
||||||
echo "Validating icons..."
|
|
||||||
|
|
||||||
wappalyzer validate-icons
|
|
||||||
@ -1,55 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
|
|
||||||
var
|
|
||||||
app,
|
|
||||||
modulesPath = process.env.WAPPALYZER_NODE_PATH !== undefined ? process.env.WAPPALYZER_NODE_PATH + '/node_modules/' : '',
|
|
||||||
fs = require('fs'),
|
|
||||||
fileType = require(modulesPath + 'file-type'),
|
|
||||||
readChunk = require(modulesPath + 'read-chunk')
|
|
||||||
isSvg = require(modulesPath + 'is-svg')
|
|
||||||
json = require(process.env.WAPPALYZER_ROOT + '/src/apps.json');
|
|
||||||
|
|
||||||
for (app in json.apps) {
|
|
||||||
(function(app) {
|
|
||||||
var basePath = process.env.WAPPALYZER_ROOT + '/src/icons/';
|
|
||||||
var iconPath = json.apps[app].icon || 'default.svg';
|
|
||||||
var path = basePath + iconPath;
|
|
||||||
|
|
||||||
var type;
|
|
||||||
if (path.substr(path.length - 4) === '.png') {
|
|
||||||
type = "PNG";
|
|
||||||
}
|
|
||||||
else if (path.substr(path.length - 4) === '.svg') {
|
|
||||||
type = "SVG";
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
var err = new Error('Icon file extension specified for app "' + app + '" is not ".png" or ".svg": src/icons/' + iconPath);
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
|
|
||||||
fs.exists(path, function(exists) {
|
|
||||||
if (exists) {
|
|
||||||
if (type === "PNG") {
|
|
||||||
var buffer = fileType(readChunk.sync(path, 0, 262));
|
|
||||||
if (buffer === null) {
|
|
||||||
var err = new Error('Unknown mimetype or bad file for "' + app + '": src/icons/' + iconPath);
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
else if (buffer.mime !== 'image/png') {
|
|
||||||
var err = new Error('Incorrect mimetype "' + buffer.mime + '" when expected PNG for app "' + app + '": src/icons/' + iconPath);
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (type === "SVG") {
|
|
||||||
if (!isSvg(fs.readFileSync(path))) {
|
|
||||||
var err = new Error('Incorrect mimetype when expected SVG for app "' + app + '": src/icons/' + iconPath);
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
var err = new Error('Missing file for app "' + app + '": src/icons/' + iconPath);
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}(app));
|
|
||||||
};
|
|
||||||
@ -1,72 +0,0 @@
|
|||||||
FROM phusion/baseimage
|
|
||||||
|
|
||||||
MAINTAINER Elbert Alias <elbert@alias.io>
|
|
||||||
|
|
||||||
ENV DEBIAN_FRONTEND noninteractive
|
|
||||||
|
|
||||||
ENV WAPPALYZER_ROOT /home/wappalyzer/synced
|
|
||||||
ENV WAPPALYZER_NODE_PATH /home/wappalyzer/node
|
|
||||||
|
|
||||||
# Install packages
|
|
||||||
RUN sed -i 's/^deb-src\ /\#deb-src\ /g' /etc/apt/sources.list
|
|
||||||
RUN sed -i 's/archive\.ubuntu\.com/au.archive.ubuntu.com/g' /etc/apt/sources.list
|
|
||||||
RUN sed -i 's/security\.ubuntu\.com/au.archive.ubuntu.com/g' /etc/apt/sources.list
|
|
||||||
RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - && \
|
|
||||||
apt-get install -y \
|
|
||||||
nodejs \
|
|
||||||
bzip2 \
|
|
||||||
zip \
|
|
||||||
libfreetype6 \
|
|
||||||
libfontconfig \
|
|
||||||
rsync && \
|
|
||||||
apt-get clean && \
|
|
||||||
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
|
||||||
|
|
||||||
|
|
||||||
# Add user
|
|
||||||
RUN useradd -ms /bin/bash wappalyzer && usermod -a -G docker_env wappalyzer && echo "wappalyzer:wappalyzer" | chpasswd
|
|
||||||
RUN echo 'wappalyzer ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
|
|
||||||
|
|
||||||
WORKDIR /home/wappalyzer
|
|
||||||
|
|
||||||
RUN su wappalyzer -c "mkdir bin synced"
|
|
||||||
RUN su wappalyzer -c "echo \"export PATH=\$PATH:/home/wappalyzer/bin:\\$WAPPALYZER_ROOT/bin\" | cat - .profile > /tmp/profile && mv /tmp/profile .profile"
|
|
||||||
|
|
||||||
RUN echo "cd \$WAPPALYZER_ROOT" >> .bashrc
|
|
||||||
RUN echo "wappalyzer" >> .bashrc
|
|
||||||
|
|
||||||
|
|
||||||
# PhantomJS
|
|
||||||
# RUN su wappalyzer -c "\
|
|
||||||
# mkdir phantomjs && \
|
|
||||||
# curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 | tar xvjC phantomjs && \
|
|
||||||
# ln -s /home/wappalyzer/phantomjs/phantomjs-*-linux-x86_64/bin/phantomjs /usr/bin/phantomjs"
|
|
||||||
|
|
||||||
|
|
||||||
# Node JS
|
|
||||||
RUN su wappalyzer -c "mkdir $WAPPALYZER_NODE_PATH"
|
|
||||||
|
|
||||||
ADD node/package.json $WAPPALYZER_NODE_PATH/package.json
|
|
||||||
|
|
||||||
RUN su wappalyzer -c "cd $WAPPALYZER_NODE_PATH && npm i"
|
|
||||||
|
|
||||||
RUN npm i -g manifoldjs
|
|
||||||
|
|
||||||
|
|
||||||
# SSH
|
|
||||||
RUN rm -f /etc/service/sshd/down
|
|
||||||
RUN su wappalyzer -c "mkdir .ssh && chmod 700 .ssh"
|
|
||||||
|
|
||||||
ADD ssh/insecure.key.pub /tmp/insecure.key.pub
|
|
||||||
|
|
||||||
RUN su wappalyzer -c "cat /tmp/insecure.key.pub >> .ssh/authorized_keys && chmod 600 .ssh/authorized_keys" && rm -f /tmp/insecure.key.pub
|
|
||||||
|
|
||||||
|
|
||||||
# Fix the `stdin: is not a tty` error in Vagrant
|
|
||||||
RUN sed -i 's/^mesg n$/tty -s \&\& mesg n/g' /root/.profile
|
|
||||||
|
|
||||||
EXPOSE 22
|
|
||||||
|
|
||||||
RUN echo "/usr/sbin/sshd -D" > /etc/my_init.d/sshd.sh
|
|
||||||
|
|
||||||
CMD ["/sbin/my_init"]
|
|
||||||
@ -1,27 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "wappalyzer-test",
|
|
||||||
"private": true,
|
|
||||||
"license": "GPLv3",
|
|
||||||
"bugs": {
|
|
||||||
"url": "https://github.com/AliasIO/Wappalyzer/issues"
|
|
||||||
},
|
|
||||||
"homepage": "https://github.com/AliasIO/Wappalyzer",
|
|
||||||
"description": "Build test for Wappalyzer",
|
|
||||||
"repository": "AliasIO/Wappalyzer",
|
|
||||||
"scripts": {
|
|
||||||
"test": "./bin/wappalyzer build"
|
|
||||||
},
|
|
||||||
"devDependencies": {
|
|
||||||
"file-type": "3.8.*",
|
|
||||||
"is-svg": "2.0.*",
|
|
||||||
"read-chunk": "2.0.*",
|
|
||||||
"jsonlint": "*",
|
|
||||||
"jpm": "*",
|
|
||||||
"imagemin-cli": "2.*.*",
|
|
||||||
"phantomjs-prebuilt": "*",
|
|
||||||
"svg2png": "3.*.*"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": ">= 4"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,27 +0,0 @@
|
|||||||
-----BEGIN RSA PRIVATE KEY-----
|
|
||||||
MIIEowIBAAKCAQEAqVo54bv3J4K62IDCUIVY7gF2iBOR8fub5c9t6j4MbsRQDmU0
|
|
||||||
yxhN1+7fMCERcoMSelAb/SYSzKE8azLugP7hsq4t+/xJijCB22uriFhKMpu4ySoA
|
|
||||||
6PVNbfbIptaVo6holVIMRKPXTg1hU6lr2DWMKB5AlGmz7ZHmQgX9iBoV8PmMYOF5
|
|
||||||
UpiwfokcNnIwDiCCpd19G/lT6FlCVZHZdeZh41FWke8oQgWevM8fdGJkZ+a7Lgdm
|
|
||||||
6A3WlvI0AIz9KWfXlXAOvthi4QlB4SvSCwRQbKdHbW5UBRShucjlce3hZO9Q6ROC
|
|
||||||
Ta1zgvscleKd63yHSQ18Z2+j1t5T3HJzhOn1ywIDAQABAoIBAFdbcWwd15oL93Hi
|
|
||||||
yVlbz8nF+2yqG6ilT3+2Bk7EfwlMPreOQTNacB0o3PwosqVNM2Tgme91ilEAAviR
|
|
||||||
JzJnR+kxrArF0vFzoCidW7hTQsLkTOeHwTxp/OzyVgKpwjrPjlMoJafqlA9xG3la
|
|
||||||
dYTr6kZvhISKGgQ8NHhuOp/QRO0uFaQD8l0kVCEufgqbs6gBY7aAkNf2iogSms9w
|
|
||||||
139zwsXeSg2Vm7Img1PVrRsRM5qbTNBMIim+9q4fSrqP+rYF9lU1p68IzmG/wY2o
|
|
||||||
0ElwRoatVVV9Rvk00a1qBGNaT39oVEtryIiAlXPno+KW5dyVx53o9vjWHDz44yBN
|
|
||||||
G8/EgEECgYEA3UJtj5ikEdhquzy1WfbZwZVmTCJLMefiVui00RLV4Q/r/4Uojdsb
|
|
||||||
L/QCAplpbHUrj7O8Lp9CuXSnUNa4CkrTidj2wkDWDFl1EBLJNz1zhJjs6Aaw2u1t
|
|
||||||
SV6ULgx9Y25oiStygkJhon+wlNF9HRSCFeNkBNNjpkD2gEifxnHIDLsCgYEAw/Fi
|
|
||||||
OHFvVPx0/DS3FD/MEJAAZaGdg8Zb2TCmCIVjoj0tZOTDvwKIEYCQ41vADDLlCSGg
|
|
||||||
XOgNXvosSZtezmJxnkzO3NYXE4XZ7bPVMamuqbKXXzaee3tPmiFaeZfiB4tj7uxh
|
|
||||||
tWQdUicSMQ8WPqyTB5wmcF7k3L4TzrjSLOi5MjECgYEAnsLYfI3pREjGpN01v3zp
|
|
||||||
2QbLZVvCGugpN3U4sUBTqW9URn+Ujt0mD+2FN7o+tC0K+czc/ZMXPJ1gpRe87N2u
|
|
||||||
Fh5RSSder3PNFKB2MyXr6rwUWZTbd+ufXGL57mTV3+/MIIjFXIQnLIN1AAOlQ6WA
|
|
||||||
hhYooXHQ8e1xMrFpu+nOwpsCgYAanYEf9ZiPzLL/RxHPOFM5tLW5GFjS+3Cicc87
|
|
||||||
vRKpiPHPkJteKLPe3L3RhDbz8T3XHbAwrnySeBzn6qjYE9snC3vV12bPChFuDk1Z
|
|
||||||
L5lAB7g0ALFKSJepcfpeJdlX+QEcBRPIlc69x/zamRgGb16RXA6EmP8xVMhP/h7e
|
|
||||||
ZsYxIQKBgBqOaZNHwaG6PqP2HCjc1keH4wQVx9eTV5y4/B5FJbMeK05vV2rBHoxA
|
|
||||||
P/dXE85olj8D/9E88xJ1V716kBeONcKFdbd3VfY5bo2VBz+piOaG3EBQOLIbd4/C
|
|
||||||
KVtgOWqI7WTLPIgpXlG5MA801RwhVyk1phmwhvW/Kb3lWsK1vkOe
|
|
||||||
-----END RSA PRIVATE KEY-----
|
|
||||||
@ -1 +0,0 @@
|
|||||||
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCpWjnhu/cngrrYgMJQhVjuAXaIE5Hx+5vlz23qPgxuxFAOZTTLGE3X7t8wIRFygxJ6UBv9JhLMoTxrMu6A/uGyri37/EmKMIHba6uIWEoym7jJKgDo9U1t9sim1pWjqGiVUgxEo9dODWFTqWvYNYwoHkCUabPtkeZCBf2IGhXw+Yxg4XlSmLB+iRw2cjAOIIKl3X0b+VPoWUJVkdl15mHjUVaR7yhCBZ68zx90YmRn5rsuB2boDdaW8jQAjP0pZ9eVcA6+2GLhCUHhK9ILBFBsp0dtblQFFKG5yOVx7eFk71DpE4JNrXOC+xyV4p3rfIdJDXxnb6PW3lPccnOE6fXL elbert@home
|
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"dependencies": {
|
||||||
|
"file-type": "3.8.*",
|
||||||
|
"is-svg": "2.0.*",
|
||||||
|
"read-chunk": "2.0.*"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@ -0,0 +1,19 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
cd "$(dirname $0)"
|
||||||
|
|
||||||
|
if [[ -z "$(which docker)" ]]; then
|
||||||
|
echo "Please install Docker from https://www.docker.com"
|
||||||
|
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker="docker run --rm -v "$(pwd):/opt/wappalyzer" -it wappalyzer/dev"
|
||||||
|
|
||||||
|
if [[ ! -d "node_modules" ]]; then
|
||||||
|
$docker ./bin/run links
|
||||||
|
|
||||||
|
$docker npm i
|
||||||
|
fi
|
||||||
|
|
||||||
|
$docker ./bin/run $@
|
||||||
@ -1,7 +1,6 @@
|
|||||||
apps.json
|
/apps.json
|
||||||
images/icons/converted/*.png
|
/images/icons/converted/*
|
||||||
images/icons/*.png
|
/images/icons/*
|
||||||
images/icons/*.svg
|
/js/wappalyzer.js
|
||||||
js/wappalyzer.js
|
/js/iframe.js
|
||||||
js/iframe.js
|
/js/network.js
|
||||||
js/network.js
|
|
||||||
|
|||||||
@ -0,0 +1 @@
|
|||||||
|
../../../icons/
|
||||||
|
Before Width: | Height: | Size: 1.8 KiB After Width: | Height: | Size: 1.7 KiB |
|
Before Width: | Height: | Size: 609 B After Width: | Height: | Size: 608 B |
|
Before Width: | Height: | Size: 1.8 KiB After Width: | Height: | Size: 1.4 KiB |
|
Before Width: | Height: | Size: 3.3 KiB After Width: | Height: | Size: 703 B |
|
Before Width: | Height: | Size: 8.1 KiB After Width: | Height: | Size: 5.7 KiB |
|
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 9.7 KiB |
|
Before Width: | Height: | Size: 6.5 KiB After Width: | Height: | Size: 3.6 KiB |
|
Before Width: | Height: | Size: 319 B After Width: | Height: | Size: 285 B |
|
Before Width: | Height: | Size: 534 B After Width: | Height: | Size: 524 B |
|
Before Width: | Height: | Size: 751 B After Width: | Height: | Size: 493 B |
|
Before Width: | Height: | Size: 4.4 KiB After Width: | Height: | Size: 2.3 KiB |
|
Before Width: | Height: | Size: 2.8 KiB After Width: | Height: | Size: 2.0 KiB |
|
Before Width: | Height: | Size: 1.7 KiB After Width: | Height: | Size: 1.7 KiB |
|
Before Width: | Height: | Size: 754 B After Width: | Height: | Size: 740 B |
|
Before Width: | Height: | Size: 2.0 KiB After Width: | Height: | Size: 1.8 KiB |
|
Before Width: | Height: | Size: 2.7 KiB After Width: | Height: | Size: 2.2 KiB |
|
Before Width: | Height: | Size: 1.5 KiB After Width: | Height: | Size: 1.5 KiB |
|
Before Width: | Height: | Size: 19 KiB After Width: | Height: | Size: 12 KiB |
|
Before Width: | Height: | Size: 16 KiB After Width: | Height: | Size: 9.8 KiB |
|
Before Width: | Height: | Size: 1.4 KiB After Width: | Height: | Size: 649 B |
|
Before Width: | Height: | Size: 1.7 KiB After Width: | Height: | Size: 855 B |
|
Before Width: | Height: | Size: 9.7 KiB After Width: | Height: | Size: 4.0 KiB |
|
Before Width: | Height: | Size: 3.2 KiB After Width: | Height: | Size: 2.5 KiB |
|
Before Width: | Height: | Size: 2.5 KiB After Width: | Height: | Size: 1.4 KiB |
|
Before Width: | Height: | Size: 540 B After Width: | Height: | Size: 493 B |
|
Before Width: | Height: | Size: 713 B After Width: | Height: | Size: 646 B |
|
Before Width: | Height: | Size: 6.3 KiB After Width: | Height: | Size: 5.4 KiB |
|
Before Width: | Height: | Size: 1.5 KiB After Width: | Height: | Size: 1.3 KiB |
|
Before Width: | Height: | Size: 2.6 KiB After Width: | Height: | Size: 1.4 KiB |
|
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 8.3 KiB |
|
Before Width: | Height: | Size: 2.7 KiB After Width: | Height: | Size: 1.2 KiB |
|
Before Width: | Height: | Size: 1.5 KiB After Width: | Height: | Size: 923 B |
|
Before Width: | Height: | Size: 368 B After Width: | Height: | Size: 234 B |
|
Before Width: | Height: | Size: 6.8 KiB After Width: | Height: | Size: 3.7 KiB |
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 3.4 KiB After Width: | Height: | Size: 3.4 KiB |
|
Before Width: | Height: | Size: 1.7 KiB After Width: | Height: | Size: 1.6 KiB |
|
Before Width: | Height: | Size: 1.6 KiB After Width: | Height: | Size: 1.5 KiB |
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 742 B |
|
Before Width: | Height: | Size: 2.4 KiB After Width: | Height: | Size: 1.0 KiB |
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 641 B |
|
Before Width: | Height: | Size: 1.7 KiB After Width: | Height: | Size: 1.2 KiB |
|
Before Width: | Height: | Size: 1.7 KiB After Width: | Height: | Size: 1.2 KiB |
|
Before Width: | Height: | Size: 1.2 KiB After Width: | Height: | Size: 326 B |
|
Before Width: | Height: | Size: 721 B After Width: | Height: | Size: 717 B |
|
Before Width: | Height: | Size: 54 KiB After Width: | Height: | Size: 53 KiB |
|
Before Width: | Height: | Size: 52 KiB After Width: | Height: | Size: 52 KiB |
|
Before Width: | Height: | Size: 1009 B After Width: | Height: | Size: 468 B |
|
Before Width: | Height: | Size: 505 B After Width: | Height: | Size: 351 B |
|
Before Width: | Height: | Size: 4.1 KiB After Width: | Height: | Size: 3.9 KiB |
|
Before Width: | Height: | Size: 2.7 KiB After Width: | Height: | Size: 2.6 KiB |
|
Before Width: | Height: | Size: 22 KiB After Width: | Height: | Size: 22 KiB |
|
Before Width: | Height: | Size: 375 B After Width: | Height: | Size: 341 B |
|
Before Width: | Height: | Size: 947 B After Width: | Height: | Size: 561 B |
|
Before Width: | Height: | Size: 8.9 KiB After Width: | Height: | Size: 8.6 KiB |
|
Before Width: | Height: | Size: 1.3 KiB After Width: | Height: | Size: 1.2 KiB |
|
Before Width: | Height: | Size: 4.5 KiB After Width: | Height: | Size: 4.3 KiB |
|
Before Width: | Height: | Size: 2.0 KiB After Width: | Height: | Size: 1.4 KiB |
|
Before Width: | Height: | Size: 400 B After Width: | Height: | Size: 267 B |
|
Before Width: | Height: | Size: 1.9 KiB After Width: | Height: | Size: 1.7 KiB |
|
Before Width: | Height: | Size: 1.9 KiB After Width: | Height: | Size: 1.9 KiB |
|
Before Width: | Height: | Size: 2.4 KiB After Width: | Height: | Size: 1.4 KiB |
|
Before Width: | Height: | Size: 4.2 KiB After Width: | Height: | Size: 2.5 KiB |
|
Before Width: | Height: | Size: 925 B After Width: | Height: | Size: 785 B |
@ -1,784 +0,0 @@
|
|||||||
'use strict';
|
|
||||||
(function() {
|
|
||||||
|
|
||||||
function isChrome() {
|
|
||||||
return (typeof chrome !== 'undefined' &&
|
|
||||||
window.navigator.userAgent.match(/Chrom(?:e|ium)\/([0-9\.]+)/));
|
|
||||||
}
|
|
||||||
|
|
||||||
var browserProxy;
|
|
||||||
if ( isChrome() ) {
|
|
||||||
browserProxy = chrome;
|
|
||||||
} else {
|
|
||||||
browserProxy = browser;
|
|
||||||
}
|
|
||||||
|
|
||||||
var MIN_FF_MAJOR_VERSION = 51;
|
|
||||||
|
|
||||||
var requiredBrowserApis = [
|
|
||||||
browserProxy.webNavigation,
|
|
||||||
browserProxy.tabs,
|
|
||||||
browserProxy.webRequest,
|
|
||||||
browserProxy.runtime
|
|
||||||
];
|
|
||||||
var areListenersRegistered = false;
|
|
||||||
var secBefore = 2000;
|
|
||||||
var secAfter = 5000;
|
|
||||||
var secBetweenDupAssets = 10e3;
|
|
||||||
var minVidSize = 500e3;
|
|
||||||
var maxVidSize = 25e6;
|
|
||||||
var maxContentRange = 25e6;
|
|
||||||
var videoExtensions = [
|
|
||||||
'af', '3gp', 'asf', 'avchd', 'avi', 'cam', 'dsh', 'flv', 'm1v', 'm2v',
|
|
||||||
'fla', 'flr', 'sol', 'm4v', 'mkv', 'wrap', 'mng', 'mov', 'mpeg', 'mpg',
|
|
||||||
'mpe', 'mp4', 'mxf', 'nsv', 'ogg', 'rm', 'svi', 'smi', 'wmv', 'webm'
|
|
||||||
];
|
|
||||||
var extensionsReg = new RegExp('\\.' + videoExtensions.join('$|\\.') + '$');
|
|
||||||
var videoContentTypesPrefixes = ['binary/octet-stream', 'video/', 'flv-application/', 'media'];
|
|
||||||
|
|
||||||
var bannedContentTypes = ['video/mp2t','video/f4m','video/f4f'];
|
|
||||||
var bannedFiletypes = ['ts'];
|
|
||||||
var bannedFiletypesReg = new RegExp('\\.' + bannedFiletypes.join('$|\\.') + '$');
|
|
||||||
var whitelistReqTypes = ['object', 'xmlhttprequest', 'other'];
|
|
||||||
|
|
||||||
var topVideoAssetDomains = [
|
|
||||||
'2mdn.net',
|
|
||||||
'adap.tv',
|
|
||||||
'adnxs.com',
|
|
||||||
'adsrvr.org',
|
|
||||||
'btrll.com',
|
|
||||||
'celtra.com',
|
|
||||||
'flashtalking.com',
|
|
||||||
'flite.com',
|
|
||||||
'innovid.com',
|
|
||||||
'jivox.com',
|
|
||||||
'mixpo.com',
|
|
||||||
'nytimes.com',
|
|
||||||
'playwire.com',
|
|
||||||
'selectmedia.asia',
|
|
||||||
'serving-sys.com',
|
|
||||||
'solvemedia.com',
|
|
||||||
'spotible.com',
|
|
||||||
'teads.tv',
|
|
||||||
'tribalfusion.com',
|
|
||||||
'tubemogul.com',
|
|
||||||
'videologygroup.com',
|
|
||||||
'washingtonpost.com'
|
|
||||||
];
|
|
||||||
|
|
||||||
if ( !String.prototype.endsWith ) {
|
|
||||||
String.prototype.endsWith = function(searchString, position) {
|
|
||||||
var subjectString = this.toString();
|
|
||||||
if ( typeof position !== 'number' || !isFinite(position) ||
|
|
||||||
Math.floor(position) !== position || position > subjectString.length) {
|
|
||||||
position = subjectString.length;
|
|
||||||
}
|
|
||||||
position -= searchString.length;
|
|
||||||
var lastIndex = subjectString.indexOf(searchString, position);
|
|
||||||
return lastIndex !== -1 && lastIndex === position;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function getFrame(getFrameDetails, callback) {
|
|
||||||
if ( typeof chrome !== 'undefined' ) {
|
|
||||||
chrome.webNavigation.getFrame(getFrameDetails, callback);
|
|
||||||
} else if ( typeof browser !== 'undefined' ) {
|
|
||||||
var gettingFrame = browser.webNavigation.getFrame(getFrameDetails);
|
|
||||||
gettingFrame.then(callback);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function ifBrowserValid(callback, elseCallback) {
|
|
||||||
if ( isChrome() ) {
|
|
||||||
|
|
||||||
callback();
|
|
||||||
} else if ( typeof browser !== 'undefined' ) {
|
|
||||||
try {
|
|
||||||
var gettingInfo = browser.runtime.getBrowserInfo();
|
|
||||||
gettingInfo.then(function(browserInfo) {
|
|
||||||
var browserVersion = parseInt(browserInfo.version.split('.')[0]);
|
|
||||||
|
|
||||||
if ( browserInfo.name === 'Firefox' &&
|
|
||||||
browserVersion >= MIN_FF_MAJOR_VERSION) {
|
|
||||||
callback();
|
|
||||||
} else {
|
|
||||||
elseCallback();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (err) {
|
|
||||||
|
|
||||||
elseCallback();
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
elseCallback();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function isTrackingEnabled() {
|
|
||||||
|
|
||||||
return parseInt(localStorage.tracking, 10);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
function isPixelRequest(request) {
|
|
||||||
return (request.type === 'image' || request.responseStatus === 204) &&
|
|
||||||
request.size <= 1000;
|
|
||||||
}
|
|
||||||
|
|
||||||
function isVpaidOrVastRequest(request) {
|
|
||||||
var lowerCaseUrl = request.url.toLowerCase();
|
|
||||||
return lowerCaseUrl.indexOf('vpaid') !== -1 || lowerCaseUrl.indexOf('vast') !== -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
function hasValidRequestType(request) {
|
|
||||||
return whitelistReqTypes.indexOf(request.type) >= 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
function stripQueryParams(url) {
|
|
||||||
return url.split('?', 1)[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseHostnameFromUrl(url) {
|
|
||||||
var parser = document.createElement('a');
|
|
||||||
parser.href = url;
|
|
||||||
return parser.hostname;
|
|
||||||
}
|
|
||||||
|
|
||||||
function hasDomain(url, domain) {
|
|
||||||
return parseHostnameFromUrl(url).endsWith(domain);
|
|
||||||
}
|
|
||||||
|
|
||||||
function findHeader(headers, key) {
|
|
||||||
var header;
|
|
||||||
for ( var i = 0; i < headers.length; i += 1 ) {
|
|
||||||
header = headers[i];
|
|
||||||
if ( header.name.toLowerCase() === key ) {
|
|
||||||
return header;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function validVideoType(vtype) {
|
|
||||||
var goodType = videoContentTypesPrefixes.some(function(prefix) {
|
|
||||||
return vtype.indexOf(prefix) === 0;
|
|
||||||
});
|
|
||||||
return goodType;
|
|
||||||
}
|
|
||||||
|
|
||||||
function assetMsgKey(assetReq) {
|
|
||||||
var url = stripQueryParams(assetReq.url);
|
|
||||||
var key = assetReq.frameId + '-' + url;
|
|
||||||
return key;
|
|
||||||
}
|
|
||||||
|
|
||||||
var PageNetworkTrafficCollector = function(tabId) {
|
|
||||||
this.tabId = tabId;
|
|
||||||
this.displayAdFound = false;
|
|
||||||
this.requests = {};
|
|
||||||
this.msgsBeingSent = {};
|
|
||||||
this.assetsSeen = {};
|
|
||||||
this.allRedirects = {};
|
|
||||||
};
|
|
||||||
|
|
||||||
var globalPageContainer = {
|
|
||||||
collectors: {},
|
|
||||||
dyingCollectors: {},
|
|
||||||
|
|
||||||
cleanupCollector: function(tabId) {
|
|
||||||
if ( tabId in this.collectors ) {
|
|
||||||
delete globalPageContainer.collectors[tabId];
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
onNewNavigation: function(details) {
|
|
||||||
var tabId = details.tabId;
|
|
||||||
this.cleanupCollector(tabId);
|
|
||||||
|
|
||||||
if ( isTrackingEnabled() ) {
|
|
||||||
if ( !areListenersRegistered ) {
|
|
||||||
|
|
||||||
registerListeners();
|
|
||||||
}
|
|
||||||
this.collectors[tabId] = new PageNetworkTrafficCollector(tabId);
|
|
||||||
} else {
|
|
||||||
if ( areListenersRegistered ) {
|
|
||||||
|
|
||||||
unregisterListeners();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
onNavigationCommitted: function(details) {
|
|
||||||
|
|
||||||
},
|
|
||||||
|
|
||||||
onNavigationCompleted: function(details) {
|
|
||||||
|
|
||||||
},
|
|
||||||
|
|
||||||
onTabClose: function(tabId, closeInfo) {
|
|
||||||
|
|
||||||
this.cleanupCollector(tabId);
|
|
||||||
delete this.collectors[tabId];
|
|
||||||
},
|
|
||||||
|
|
||||||
onDisplayAdFound: function(tabId) {
|
|
||||||
this.collectors[tabId].displayAdFound = true;
|
|
||||||
},
|
|
||||||
|
|
||||||
getRandId: function() {
|
|
||||||
return String(Math.floor(Math.random() * 1e9));
|
|
||||||
},
|
|
||||||
|
|
||||||
getCollector: function(tabId) {
|
|
||||||
if ( this.collectors.hasOwnProperty(tabId) ) {
|
|
||||||
return this.collectors[tabId];
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
},
|
|
||||||
|
|
||||||
forwardCall: function(details, collectorMemberFunction) {
|
|
||||||
var collector = this.getCollector(details.tabId);
|
|
||||||
if ( collector !== null ) {
|
|
||||||
collectorMemberFunction.apply(collector, [details]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() {
|
|
||||||
var logMessage = Array.from(arguments).join(' ');
|
|
||||||
var message = {message: logMessage, event: 'console-log-message'};
|
|
||||||
browserProxy.tabs.sendMessage(this.tabId, message);
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, isValidAd) {
|
|
||||||
var msg = {};
|
|
||||||
msg.assets = [];
|
|
||||||
msg.event_data = {};
|
|
||||||
if ( isValidAd ) {
|
|
||||||
msg.event = 'new-video-ad';
|
|
||||||
msg.requests = reqs;
|
|
||||||
msg.requests.sort(function(reqA, reqB) {return reqA.requestTimestamp - reqB.requestTimestamp;});
|
|
||||||
if ( assetReq ) {
|
|
||||||
msg.assets = [assetReq];
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
msg.requests = reqs.map(function(request) {
|
|
||||||
return parseHostnameFromUrl(request.url);
|
|
||||||
});
|
|
||||||
msg.assets = [{
|
|
||||||
|
|
||||||
url: parseHostnameFromUrl(assetReq.url),
|
|
||||||
|
|
||||||
contentType: assetReq.contentType,
|
|
||||||
size: assetReq.size
|
|
||||||
}];
|
|
||||||
msg.event = 'new-invalid-video-ad';
|
|
||||||
}
|
|
||||||
msg.origUrl = curPageUrl;
|
|
||||||
msg.displayAdFound = this.displayAdFound;
|
|
||||||
|
|
||||||
browserProxy.tabs.sendMessage(this.tabId, msg);
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) {
|
|
||||||
return url + ':' + frameId;
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.seenBefore = function(request) {
|
|
||||||
var oldTime = this.assetsSeen[assetMsgKey(request)];
|
|
||||||
if ( oldTime && (request.requestTimestamp-oldTime < secBetweenDupAssets)){
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.recordSeenAsset = function(request) {
|
|
||||||
this.assetsSeen[assetMsgKey(request)] = request.requestTimestamp;
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.onBeforeRequest = function(details) {
|
|
||||||
var req = {
|
|
||||||
url: details.url,
|
|
||||||
type: details.type,
|
|
||||||
httpMethod: details.method,
|
|
||||||
frameId: details.frameId,
|
|
||||||
parentFrameId: details.parentFrameId,
|
|
||||||
requestTimestamp: details.timeStamp,
|
|
||||||
};
|
|
||||||
this.requests[details.requestId] = req;
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.onSendHeaders = function(details) {
|
|
||||||
var request, header;
|
|
||||||
request = this.requests[details.requestId];
|
|
||||||
header = request && findHeader(details.requestHeaders, 'x-requested-with');
|
|
||||||
if ( header && header.value.toLowerCase().indexOf('flash') > -1 ) {
|
|
||||||
request.from_flash = true;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.onHeadersReceived = function(details) {
|
|
||||||
var getFrameDetails = {
|
|
||||||
tabId: details.tabId,
|
|
||||||
processId: null,
|
|
||||||
frameId: details.frameId
|
|
||||||
};
|
|
||||||
var pageNetworkTrafficController = this;
|
|
||||||
getFrame(getFrameDetails, function(frameDetails) {
|
|
||||||
if ( frameDetails && frameDetails.url ) {
|
|
||||||
pageNetworkTrafficController._onHeadersReceived(details, frameDetails);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype._onHeadersReceived = function(details, frameDetails) {
|
|
||||||
var contentSize, contentRange;
|
|
||||||
|
|
||||||
var request = this.requests[details.requestId];
|
|
||||||
if ( request ) {
|
|
||||||
var redirParent = this.allRedirects[this.getRedirKey(details.url, details.frameId)];
|
|
||||||
var header = request && findHeader(details.responseHeaders, 'content-type');
|
|
||||||
var contentType = header && header.value.toLowerCase();
|
|
||||||
|
|
||||||
if ( contentType){
|
|
||||||
request.contentType = contentType;
|
|
||||||
}
|
|
||||||
header = request && findHeader(details.responseHeaders, 'content-length');
|
|
||||||
contentSize = header && header.value;
|
|
||||||
if ( contentSize ) {
|
|
||||||
request.size = request.size || 0;
|
|
||||||
request.size += parseInt(contentSize);
|
|
||||||
}
|
|
||||||
header = request && findHeader(details.responseHeaders, 'content-range');
|
|
||||||
contentRange = header && header.value;
|
|
||||||
if ( contentRange ) {
|
|
||||||
request.contentRange = parseInt(contentRange.split('/')[1]);
|
|
||||||
}
|
|
||||||
|
|
||||||
var frameUrl = null;
|
|
||||||
if ( frameDetails && frameDetails.url ) {
|
|
||||||
frameUrl = frameDetails.url;
|
|
||||||
}
|
|
||||||
if ( !this.bannedRequest(request) &&
|
|
||||||
(this.isVideoReq(frameUrl, request) || (redirParent && redirParent.isVideo))) {
|
|
||||||
request.isVideo = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.onBeforeRedirect = function(details) {
|
|
||||||
var request = this.requests[details.requestId];
|
|
||||||
if ( request ) {
|
|
||||||
if ( request.redirects ) {
|
|
||||||
request.redirects.push(details.redirectUrl);
|
|
||||||
} else {
|
|
||||||
request.redirects = [details.redirectUrl];
|
|
||||||
}
|
|
||||||
this.allRedirects[this.getRedirKey(details.redirectUrl, details.frameId)] = request;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.isYoutubeMastheadRequest = function(url) {
|
|
||||||
var re = /video_masthead/;
|
|
||||||
return this.hasYoutubeDomain(url) && re.test(url);
|
|
||||||
};
|
|
||||||
PageNetworkTrafficCollector.prototype.isYoutubeVideoRequest = function(srcUrl, destUrl) {
|
|
||||||
if ( !this.hasYoutubeDomain(srcUrl) ) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
var re = /https?:\/\/r.*?\.googlevideo\.com\/videoplayback\?/;
|
|
||||||
return re.test(destUrl);
|
|
||||||
};
|
|
||||||
PageNetworkTrafficCollector.prototype.processResponse = function(requestDetails, frameDetails) {
|
|
||||||
var request;
|
|
||||||
if ( requestDetails ) {
|
|
||||||
request = this.requests[requestDetails.requestId];
|
|
||||||
if ( request ) {
|
|
||||||
request.responseStatus = requestDetails.statusCode;
|
|
||||||
request.responseTimestamp = requestDetails.timeStamp;
|
|
||||||
|
|
||||||
var frameUrl = null;
|
|
||||||
if ( frameDetails && frameDetails.url ) {
|
|
||||||
frameUrl = frameDetails.url;
|
|
||||||
}
|
|
||||||
|
|
||||||
var requestUrl = null;
|
|
||||||
if ( request.url ) {
|
|
||||||
requestUrl = request.url;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( this.isYoutubeAdReq(frameUrl, requestUrl) ) {
|
|
||||||
var videoId = this.parseYoutubeVideoIdFromUrl(requestUrl);
|
|
||||||
if ( videoId ) {
|
|
||||||
request.isYoutubeAd = true;
|
|
||||||
request.isVideo = true;
|
|
||||||
request.url = 'https://www.youtube.com/watch?v=' + this.parseYoutubeVideoIdFromUrl(requestUrl);
|
|
||||||
}
|
|
||||||
} else if ( !this.bannedRequest(request) &&
|
|
||||||
(this.isVideo || this.isVideoReq(frameUrl, request))) {
|
|
||||||
request.isVideo = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( request.isVideo ) {
|
|
||||||
|
|
||||||
var msgKey = assetMsgKey(request);
|
|
||||||
this.msgsBeingSent[msgKey] = request;
|
|
||||||
if ( !this.seenBefore(request) ) {
|
|
||||||
this.sendMsgWhenQuiet(msgKey);
|
|
||||||
}
|
|
||||||
this.recordSeenAsset(request);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.onResponseStarted = function(responseDetails) {
|
|
||||||
if ( responseDetails.frameId < 0 ) {
|
|
||||||
responseDetails.frameId = 99999;
|
|
||||||
|
|
||||||
}
|
|
||||||
var getFrameDetails = {
|
|
||||||
tabId: responseDetails.tabId,
|
|
||||||
processId: null,
|
|
||||||
frameId: responseDetails.frameId
|
|
||||||
};
|
|
||||||
var pageNetworkTrafficController = this;
|
|
||||||
getFrame(getFrameDetails, function(frameDetails) {
|
|
||||||
if ( frameDetails && frameDetails.url ) {
|
|
||||||
pageNetworkTrafficController.processResponse(responseDetails, frameDetails);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.hasBannedFiletype = function(request) {
|
|
||||||
var url = stripQueryParams(request.url);
|
|
||||||
if ( bannedFiletypesReg.exec(url) ) {
|
|
||||||
return true;
|
|
||||||
} else {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.checkContentHeaders = function(request) {
|
|
||||||
if ( request.contentType && validVideoType(request.contentType) ) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.checkUrlExtension = function(request) {
|
|
||||||
var url = stripQueryParams(request.url);
|
|
||||||
if ( extensionsReg.exec(url) ) {
|
|
||||||
return true;
|
|
||||||
} else {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.isVideoReq = function(srcUrl, request) {
|
|
||||||
if ( this.isYoutubeVideoRequest(srcUrl, request.url) ) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return this.checkUrlExtension(request) || this.checkContentHeaders(request);
|
|
||||||
};
|
|
||||||
PageNetworkTrafficCollector.prototype.hasYoutubeDomain = function(url) {
|
|
||||||
var hostname = parseHostnameFromUrl(url) ;
|
|
||||||
if ( hostname === 'www.youtube.com' ) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
PageNetworkTrafficCollector.prototype.parseYoutubeVideoIdFromUrl = function(url) {
|
|
||||||
var re = /^https?:\/\/www\.youtube\.com\/get_video_info.*(?:\?|&)video_id=(.*?)(?:$|&)/;
|
|
||||||
var match = re.exec(url);
|
|
||||||
if ( match && match.length > 1 ) {
|
|
||||||
return match[1];
|
|
||||||
}
|
|
||||||
|
|
||||||
re = /^https?:\/\/www\.youtube\.com\/embed\/(.*?)(?:$|\?)/;
|
|
||||||
match = re.exec(url);
|
|
||||||
if ( match && match.length > 1 ) {
|
|
||||||
return match[1];
|
|
||||||
}
|
|
||||||
|
|
||||||
re = /^https?:\/\/www\.youtube\.com\/watch\?v=(.*$)/;
|
|
||||||
match = re.exec(url);
|
|
||||||
if ( match && match.length > 1 ) {
|
|
||||||
return match[1];
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.isYoutubeGetVideoInfoReq = function(url) {
|
|
||||||
var re = /^https?:\/\/www\.youtube\.com\/get_video_info\?/;
|
|
||||||
return re.test(url);
|
|
||||||
};
|
|
||||||
PageNetworkTrafficCollector.prototype.isYoutubeAdReq = function(srcUrl, destUrl) {
|
|
||||||
|
|
||||||
if ( !this.hasYoutubeDomain(srcUrl) ||
|
|
||||||
!this.isYoutubeGetVideoInfoReq(destUrl)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if ( this.parseYoutubeVideoIdFromUrl(srcUrl) ===
|
|
||||||
this.parseYoutubeVideoIdFromUrl(destUrl) &&
|
|
||||||
!this.isYoutubeMastheadRequest(destUrl)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.bannedRequest = function(request) {
|
|
||||||
return this.bannedVideoType(request) || this.hasBannedFiletype(request) || this.bannedVideoSize(request);
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.bannedVideoType = function(request) {
|
|
||||||
var badType = false;
|
|
||||||
if ( request.contentType ) {
|
|
||||||
badType = bannedContentTypes.some(function(prefix) {
|
|
||||||
return request.contentType.indexOf(prefix) >= 0;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return badType;
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.bannedVideoSize = function(request) {
|
|
||||||
if ( request.size !== null ) {
|
|
||||||
if ( request.size < minVidSize || request.size > maxVidSize || request.contentRange > maxContentRange ) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.grabTagReqs = function(tabRequests, assetRequest) {
|
|
||||||
var minTimestamp, maxTimestamp;
|
|
||||||
minTimestamp = assetRequest.requestTimestamp - secBefore;
|
|
||||||
maxTimestamp = assetRequest.requestTimestamp + secAfter;
|
|
||||||
|
|
||||||
var filteredRequests = tabRequests.filter(function(request) {
|
|
||||||
return (request.requestTimestamp > minTimestamp &&
|
|
||||||
request.requestTimestamp < maxTimestamp &&
|
|
||||||
request.frameId === assetRequest.frameId &&
|
|
||||||
request.url !== assetRequest.url &&
|
|
||||||
(hasValidRequestType(request) ||
|
|
||||||
isPixelRequest(request)));
|
|
||||||
});
|
|
||||||
|
|
||||||
return filteredRequests;
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.isValidVideoAd = function(assetRequest, tagRequests) {
|
|
||||||
var hasVpaidOrVastRequest = tagRequests.some(function(tagRequest) {
|
|
||||||
return isVpaidOrVastRequest(tagRequest);
|
|
||||||
});
|
|
||||||
|
|
||||||
if ( assetRequest.isYoutubeAd ) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if ( hasVpaidOrVastRequest ) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
var hasTopVideoAssetDomain = topVideoAssetDomains.some(function(assetDomain) {
|
|
||||||
return hasDomain(assetRequest.url, assetDomain);
|
|
||||||
});
|
|
||||||
|
|
||||||
return hasTopVideoAssetDomain;
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.sendMsgWhenQuiet = function(msgKey) {
|
|
||||||
var _this = this,
|
|
||||||
origPageUrl, msgAssetReq;
|
|
||||||
msgAssetReq = this.msgsBeingSent[msgKey];
|
|
||||||
browserProxy.tabs.get(this.tabId, function(tab) {origPageUrl = tab.url;});
|
|
||||||
|
|
||||||
setTimeout(function() {
|
|
||||||
var rawRequests = [];
|
|
||||||
if ( globalPageContainer.collectors[_this.tabId] === _this ) {
|
|
||||||
for ( var reqId in _this.requests ) {
|
|
||||||
rawRequests.push(_this.requests[reqId]);
|
|
||||||
}
|
|
||||||
var tagReqs = _this.grabTagReqs(rawRequests, msgAssetReq);
|
|
||||||
|
|
||||||
if ( _this.isValidVideoAd(msgAssetReq, tagReqs) ) {
|
|
||||||
_this.sendToTab(msgAssetReq, tagReqs, origPageUrl, true);
|
|
||||||
} else {
|
|
||||||
|
|
||||||
_this.sendToTab(msgAssetReq, tagReqs, origPageUrl, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
|
||||||
|
|
||||||
}
|
|
||||||
delete _this.msgsBeingSent[msgKey];
|
|
||||||
}, secAfter+secBefore);
|
|
||||||
};
|
|
||||||
|
|
||||||
PageNetworkTrafficCollector.prototype.existingMessage = function(candidateRequest) {
|
|
||||||
var frameMsg = this.msgsBeingSent[candidateRequest.frameId];
|
|
||||||
if ( frameMsg ) {
|
|
||||||
return frameMsg;
|
|
||||||
} else {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
function onBeforeRequestListener(details) {
|
|
||||||
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRequest);
|
|
||||||
}
|
|
||||||
|
|
||||||
function onSendHeadersListener(details) {
|
|
||||||
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onSendHeaders);
|
|
||||||
}
|
|
||||||
|
|
||||||
function onHeadersReceivedListener(details) {
|
|
||||||
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onHeadersReceived);
|
|
||||||
}
|
|
||||||
|
|
||||||
function onBeforeRedirectListener(details) {
|
|
||||||
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onBeforeRedirect);
|
|
||||||
}
|
|
||||||
|
|
||||||
function onResponseStartedListener(details) {
|
|
||||||
globalPageContainer.forwardCall(details, PageNetworkTrafficCollector.prototype.onResponseStarted);
|
|
||||||
}
|
|
||||||
|
|
||||||
function onCommittedListener(details) {
|
|
||||||
if ( details.frameId === 0 ) {
|
|
||||||
globalPageContainer.onNavigationCommitted(details);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function onCompletedListener(details) {
|
|
||||||
if ( details.frameId === 0 ) {
|
|
||||||
globalPageContainer.onNavigationCompleted(details);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function onRemovedListener(tabId, closeInfo) {
|
|
||||||
globalPageContainer.onTabClose(tabId, closeInfo);
|
|
||||||
}
|
|
||||||
|
|
||||||
function onMessageListener(message, sender, sendResponse) {
|
|
||||||
if ( message.event === 'new-ad' && message.data.event === 'ad' ) {
|
|
||||||
var tabId = sender.tab.id;
|
|
||||||
if ( tabId ) {
|
|
||||||
globalPageContainer.onDisplayAdFound(tabId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function registerListeners() {
|
|
||||||
|
|
||||||
browserProxy.webRequest.onBeforeRequest.addListener(
|
|
||||||
onBeforeRequestListener,
|
|
||||||
{urls: ['http://*/*', 'https://*/*']},
|
|
||||||
[]
|
|
||||||
);
|
|
||||||
|
|
||||||
browserProxy.webRequest.onSendHeaders.addListener(
|
|
||||||
onSendHeadersListener,
|
|
||||||
{urls: ['http://*/*', 'https://*/*']},
|
|
||||||
['requestHeaders']
|
|
||||||
);
|
|
||||||
|
|
||||||
browserProxy.webRequest.onHeadersReceived.addListener(
|
|
||||||
onHeadersReceivedListener,
|
|
||||||
{urls: ['http://*/*', 'https://*/*']},
|
|
||||||
['responseHeaders']
|
|
||||||
);
|
|
||||||
|
|
||||||
browserProxy.webRequest.onBeforeRedirect.addListener(
|
|
||||||
onBeforeRedirectListener,
|
|
||||||
{urls: ['http://*/*', 'https://*/*']},
|
|
||||||
[]
|
|
||||||
);
|
|
||||||
|
|
||||||
browserProxy.webRequest.onResponseStarted.addListener(
|
|
||||||
onResponseStartedListener,
|
|
||||||
{urls: ['http://*/*', 'https://*/*']},
|
|
||||||
['responseHeaders']
|
|
||||||
);
|
|
||||||
|
|
||||||
browserProxy.webNavigation.onCommitted.addListener(onCommittedListener);
|
|
||||||
browserProxy.webNavigation.onCompleted.addListener(onCompletedListener);
|
|
||||||
browserProxy.tabs.onRemoved.addListener(onRemovedListener);
|
|
||||||
browserProxy.runtime.onMessage.addListener(onMessageListener);
|
|
||||||
|
|
||||||
areListenersRegistered = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
function unregisterListeners() {
|
|
||||||
|
|
||||||
browserProxy.webRequest.onBeforeRequest.removeListener(
|
|
||||||
onBeforeRequestListener
|
|
||||||
);
|
|
||||||
|
|
||||||
browserProxy.webRequest.onSendHeaders.removeListener(
|
|
||||||
onSendHeadersListener
|
|
||||||
);
|
|
||||||
|
|
||||||
browserProxy.webRequest.onHeadersReceived.removeListener(
|
|
||||||
onHeadersReceivedListener
|
|
||||||
);
|
|
||||||
|
|
||||||
browserProxy.webRequest.onBeforeRedirect.removeListener(
|
|
||||||
onBeforeRedirectListener
|
|
||||||
);
|
|
||||||
|
|
||||||
browserProxy.webRequest.onResponseStarted.removeListener(
|
|
||||||
onResponseStartedListener
|
|
||||||
);
|
|
||||||
|
|
||||||
browserProxy.webNavigation.onCommitted.removeListener(onCommittedListener);
|
|
||||||
browserProxy.webNavigation.onCompleted.removeListener(onCompletedListener);
|
|
||||||
browserProxy.tabs.onRemoved.removeListener(onRemovedListener);
|
|
||||||
browserProxy.runtime.onMessage.removeListener(onMessageListener);
|
|
||||||
|
|
||||||
areListenersRegistered = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
function areRequiredBrowserApisAvailable() {
|
|
||||||
return requiredBrowserApis.every(function(api) {
|
|
||||||
return typeof api !== 'undefined';
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( areRequiredBrowserApisAvailable() ) {
|
|
||||||
ifBrowserValid(
|
|
||||||
function() {
|
|
||||||
browserProxy.webNavigation.onBeforeNavigate.addListener(
|
|
||||||
function(details) {
|
|
||||||
if ( details.frameId === 0 ) {
|
|
||||||
globalPageContainer.onNewNavigation(details);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
url: [{urlMatches: 'http://*/*'}, {urlMatches: 'https://*/*'}]
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}, function() {
|
|
||||||
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
browserProxy.runtime.onMessage.addListener(function(request, sender, sendResponse) {
|
|
||||||
if ( request === 'is_browser_valid' ) {
|
|
||||||
ifBrowserValid(
|
|
||||||
sendResponse({'browser_valid': true}),
|
|
||||||
sendResponse({'browser_valid': false})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
browserProxy.runtime.onMessage.addListener(function(request, sender, sendResponse) {
|
|
||||||
if ( request === 'is_tracking_enabled' ) {
|
|
||||||
sendResponse({'tracking_enabled': isTrackingEnabled()});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
})();
|
|
||||||