Migrations
/
wappalyzer
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactoring of wappalyzer validate-icons

Browse Source
main
Elbert Alias 10 years ago
parent 5602b9223b
commit d6debd16bb
11 changed files with 204 additions and 252 deletions
Show Stats Download Patch File Download Diff File

15
.gitignore vendored
Unescape View File

@ -3,18 +3,9 @@
build/*
npm-debug.log
node_modules/*
bin/node_modules/
bin/npm/npm-debug.log
drivers/npm/node_modules/
drivers/npm/node_modules
drivers/npm/npm-debug.log
!.gitkeep
Thumbs.db
!.gitkeep

3
.travis.yml
Unescape View File

@ -3,13 +3,14 @@ node_js:
- "0.10"
before_install:
- export WAPPALYZER_ROOT=$TRAVIS_BUILD_DIR
- export WAPPALYZER_NODE_PATH=$TRAVIS_BUILD_DIR
- export PATH=$PATH:$TRAVIS_BUILD_DIR/bin
install:
- sudo apt-get update -y
- sudo apt-get install -y curl zip sudo -y --force-yes
- sudo apt-get clean
- npm install jsonlint -g
- ln -s bin/package.json package.json && npm install
- ln -s docker/node/package.json package.json && npm install
- mkdir mozilla && curl -L https://ftp.mozilla.org/pub/mozilla.org/labs/jetpack/jetpack-sdk-latest.tar.gz | tar xvzC mozilla && ln -s $WAPPALYZER_ROOT/mozilla/addon-sdk-*/bin/cfx bin/cfx
- mkdir phantomjs && curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2 | tar xvjC phantomjs && ln -s $WAPPALYZER_ROOT/phantomjs/phantomjs-*-linux-x86_64/bin/phantomjs bin/phantomjs
script: wappalyzer build

2
Vagrantfile vendored
Unescape View File

@ -13,7 +13,7 @@ Vagrant.configure("2") do |config|
config.ssh.port = "22"
config.ssh.username = "wappalyzer"
config.ssh.private_key_path = "docker/insecure.key"
config.ssh.private_key_path = "docker/ssh/insecure.key"
config.vm.provision "shell", inline: "su - wappalyzer -c 'wappalyzer links'"
config.vm.provision "shell", inline: "echo Finished. Run \\`vagrant ssh\\` to access the environment."

2
bin/wappalyzer-validate
Unescape View File

@ -31,4 +31,4 @@ jsonlint --quiet -V $WAPPALYZER_ROOT/schema.json $path/apps.json
echo "Validating icons..."
node $WAPPALYZER_ROOT/bin/wappalyzer-validate-icons.js $path/apps.json $path/icons
wappalyzer validate-icons

26
bin/wappalyzer-validate-icons
Unescape View File

@ -0,0 +1,26 @@
#!/usr/bin/env node
var
modulesPath = process.env.WAPPALYZER_NODE_PATH !== undefined ? process.env.WAPPALYZER_NODE_PATH + '/node_modules/' : '',
fileType = require(modulesPath + 'file-type'),
fs = require(modulesPath + 'fs-extra'),
readChunk = require(modulesPath + 'read-chunk')
json = require(process.env.WAPPALYZER_ROOT + '/src/apps.json');
Object.keys(json.apps).forEach(function(app) {
var path = process.env.WAPPALYZER_ROOT + '/src/icons/' + app + '.png';
fs.exists(path, function(exists) {
var buffer;
if ( exists ) {
buffer = fileType(readChunk.sync(path, 0, 262));
if ( buffer.mime !== 'image/png' ) {
throw new Error('Incorrect mimetype "' + buffer.mime + '": src/icons/' + app + '.png');
}
} else {
throw new Error('Missing file: src/icons/' + app + '.png');
}
});
});

68
bin/wappalyzer-validate-icons.js
Unescape View File

@ -1,68 +0,0 @@
var readChunk = require('read-chunk');
var fileType = require('file-type');
var path = require('path');
var fs = require('fs-extra');
var async = require('async');
var glob = require('glob');
var appsJSON = require(process.argv[2]);
var iconsDir = process.argv[3];
var appsIconPaths = [];
function arrayDiff(a1, a2) {
var o1={}, o2={}, diff=[], i, len, k;
for (i=0, len=a1.length; i<len; i++) { o1[a1[i]] = true; }
for (i=0, len=a2.length; i<len; i++) { o2[a2[i]] = true; }
for (k in o1) { if (!(k in o2)) { diff.push(k); } }
for (k in o2) { if (!(k in o1)) { diff.push(k); } }
return diff;
}
function removeA(arr) {
var what, a = arguments, L = a.length, ax;
while (L > 1 && arr.length) {
what = a[--L];
while ((ax= arr.indexOf(what)) !== -1) {
arr.splice(ax, 1);
}
}
return arr;
}
async.each(Object.keys(appsJSON.apps), function (app, callback) {
glob(iconsDir + "/" + app + ".+(png|gif|jpg|jpeg|ico|icon|icns|tiff|tif|svg|bmp|psd|pspimage|thm|yuv|ai|drw|eps|ps)", function (err, files) {
if (err) throw err;
if (files.length < 1) {
var err = new Error("There is no icon for '" + app + "'!");
throw err;
} else if (files.length > 1) {
var err = new Error("There is more than one icon for '" + app + "'!");
throw err;
} else {
if (files[0].split('.').pop() !== 'png') {
var err = new Error("The icon at " + files[0] + " does not have a '.png' extension!");
throw err;
} else {
var buffer = fileType(readChunk.sync(files[0], 0, 262));
if (buffer.mime !== 'image/png' || buffer.ext !== 'png') {
var err = new Error("The icon at " + files[0] + " has a '.png' extension, but it is not actually a PNG file! It is actually a " + buffer.mime + " which usually has an extension of '" + buffer.ext + "'.");
throw err;
} else {
appsIconPaths.push(path.basename(files[0]));
callback();
}
}
}
});
}, function(err) {
if (err) throw err;
fs.readdir(iconsDir, function(err, iconsList) {
if (err) throw err;
iconsList = removeA(iconsList, 'Thumbs.db'); // While Thumbs.db is excluded from git, Windows still adds it and it messes up tests
appsIconPaths.push("default.png");
if (appsIconPaths.length < iconsList.length) {
var err = new Error("There are " + (iconsList.length - appsIconPaths.length) + " more files in the icons directory (" + iconsDir + ") than there are apps! There are " + appsIconPaths.length + " verified icons (one is the default), but there are " + iconsList.length + " total files." + "\n" + "The extra files are: " + arrayDiff(iconsList, appsIconPaths));
throw err;
}
});
});

28
docker/Dockerfile
Unescape View File

@ -1,35 +1,40 @@
FROM phusion/baseimage
MAINTAINER Elbert Alias <elbert@alias.io>
ENV DEBIAN_FRONTEND noninteractive
ENV WAPPALYZER_ROOT /home/wappalyzer/synced
ENV WAPPALYZER_NODE_PATH /home/wappalyzer/node
# Install packages
RUN apt-get update && apt-get install -y curl nodejs npm zip
RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
RUN ln -s /usr/bin/nodejs /usr/bin/node
RUN npm install jsonlint -g
# Add user
RUN useradd -ms /bin/bash wappalyzer && echo "wappalyzer:wappalyzer" | chpasswd
RUN useradd -ms /bin/bash wappalyzer && usermod -a -G docker_env wappalyzer && echo "wappalyzer:wappalyzer" | chpasswd
RUN echo 'wappalyzer ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
WORKDIR /home/wappalyzer
RUN su wappalyzer -c "mkdir bin synced"
RUN echo "export WAPPALYZER_ROOT='/home/wappalyzer/synced'" >> /tmp/profile
RUN echo "export PATH=$PATH:/home/wappalyzer/bin:\$WAPPALYZER_ROOT/bin" >> /tmp/profile
RUN cat .profile >> /tmp/profile && mv /tmp/profile .profile
RUN su wappalyzer -c "echo \"export PATH=\$PATH:/home/wappalyzer/bin:\\$WAPPALYZER_ROOT/bin\" | cat - .profile > /tmp/profile && mv /tmp/profile .profile"
RUN echo "cd \$WAPPALYZER_ROOT" >> .bashrc
RUN echo "wappalyzer" >> .bashrc
RUN su wappalyzer -c "\
ln -s bin/package.json package.json && \
npm install"
# Node JS
RUN su wappalyzer -c "mkdir $WAPPALYZER_NODE_PATH"
ADD node/package.json $WAPPALYZER_NODE_PATH/package.json
RUN npm install jsonlint -g && su wappalyzer -c "cd $WAPPALYZER_NODE_PATH && npm install"
# Mozilla Add-on SDK
RUN su wappalyzer -c "\
@ -49,7 +54,7 @@ RUN su wappalyzer -c "\
RUN rm -f /etc/service/sshd/down
RUN su wappalyzer -c "mkdir .ssh && chmod 700 .ssh"
ADD insecure.key.pub /tmp/insecure.key.pub
ADD ssh/insecure.key.pub /tmp/insecure.key.pub
RUN su wappalyzer -c "cat /tmp/insecure.key.pub >> .ssh/authorized_keys && chmod 600 .ssh/authorized_keys" && rm -f /tmp/insecure.key.pub
@ -57,7 +62,6 @@ RUN su wappalyzer -c "cat /tmp/insecure.key.pub >> .ssh/authorized_keys && chmod
# Fix the `stdin: is not a tty` error in Vagrant
RUN sed -i 's/^mesg n$/tty -s \&\& mesg n/g' /root/.profile
EXPOSE 22
RUN echo "/usr/sbin/sshd -D" > /etc/my_init.d/sshd.sh

2
bin/package.json → docker/node/package.json
Unescape View File

@ -1,9 +1,7 @@
{
"devDependencies": {
"async": "0.9.*",
"file-type": "2.2.*",
"fs-extra": "0.16.*",
"glob": "4.4.*",
"read-chunk": "1.0.*"
}
}

0
docker/insecure.key → docker/ssh/insecure.key
Unescape View File

0
docker/insecure.key.pub → docker/ssh/insecure.key.pub
Unescape View File

310
src/apps.json
Unescape View File

@ -661,6 +661,14 @@
},
"website": "www.boa.org"
},
"Boba.js": {
"cats": [
12
],
"implies": "Google Analytics",
"script": "boba(\\.min)?\\.js",
"website": "http://boba.space150.com/"
},
"Bolt": {
"cats": [
1
@ -1710,6 +1718,15 @@
"script": "enyo\\.js",
"website": "enyojs.com"
},
"Epoch": {
"cats": [
25
],
"html": "<link.+?href=\"[^\"]+epoch(?:\\.min)?\\.css",
"implies": "D3",
"script": "epoch(\\.min)?\\.js",
"website": "https://fastly.github.io/epoch/"
},
"Epom": {
"cats": [
36
@ -2039,6 +2056,20 @@
],
"website": "about.gitlab.com"
},
"GitLab CI": {
"cats": [
44,
47
],
"implies": [
"Ruby",
"Ruby on Rails"
],
"meta": {
"description": "GitLab Continuous Integration"
},
"website": "https://about.gitlab.com/gitlab-ci/"
},
"GlassFish": {
"cats": [
22
@ -2307,6 +2338,14 @@
},
"website": "???"
},
"Hammer.js": {
"cats": [
12
],
"env": "^Hammer$",
"script": "hammer(\\.min)?\\.js",
"website": "http://hammerjs.github.io"
},
"Handlebars": {
"cats": [
12
@ -2929,28 +2968,28 @@
"implies": "PHP",
"website": "kohanaframework.org"
},
"Koken": {
"cats": [
1
],
"env": "^\\$K$",
"headers": {
"Set-Cookie": "koken_referrer="
},
"html": [
"<html lang=\"en\" class=\"k-source-essays k-lens-essays\">",
"<!--\\s+KOKEN DEBUGGING"
],
"implies": [
"PHP",
"MySQL"
],
"meta": {
"generator": "Koken ([\\d.]+);version:\\1"
},
"script": "koken(?:\\.js\\?([\\d.]+)|/storage);version:\\1",
"website": "koken.me"
},
"Koken": {
"cats": [
1
],
"env": "^\\$K$",
"headers": {
"Set-Cookie": "koken_referrer="
},
"html": [
"<html lang=\"en\" class=\"k-source-essays k-lens-essays\">",
"<!--\\s+KOKEN DEBUGGING"
],
"implies": [
"PHP",
"MySQL"
],
"meta": {
"generator": "Koken ([\\d.]+);version:\\1"
},
"script": "koken(?:\\.js\\?([\\d.]+)|/storage);version:\\1",
"website": "koken.me"
},
"Kolibri CMS": {
"cats": [
1
@ -3026,6 +3065,13 @@
"implies": "PHP",
"website": "laravel.com"
},
"Lazy.js": {
"cats": [
12
],
"script": "lazy(\\.browser)?(\\.min)?\\.js",
"website": "http://danieltao.com/lazy.js/"
},
"Leaflet": {
"cats": [
35
@ -3818,6 +3864,16 @@
"env": "^OAS_AD$",
"website": "xaxis.com"
},
"Open Classifieds": {
"cats": [
6
],
"meta": {
"author": "open-classifieds\\.com",
"copyright": "Open Classifieds ?([0-9.]+)?\\;version:\\1"
},
"website": "http://open-classifieds.com"
},
"Open Journal Systems": {
"cats": [
50
@ -3841,6 +3897,16 @@
"html": "<!-- (?:Start|End) Open Web Analytics Tracker -->",
"website": "openwebanalytics.com"
},
"Open eShop": {
"cats": [
6
],
"meta": {
"author": "open-eshop\\.com",
"copyright": "Open eShop ?([0-9.]+)?\\;version:\\1"
},
"website": "http://open-eshop.com/"
},
"OpenCart": {
"cats": [
6
@ -4079,6 +4145,13 @@
"env": "^PARSELY$",
"website": "parse.ly"
},
"Paths.js": {
"cats": [
25
],
"script": "paths(\\.min)?\\.js",
"website": "https://github.com/andreaferretti/paths-js"
},
"PayPal": {
"cats": [
41
@ -4508,6 +4581,15 @@
"script": "reveal(?:\\.min)?\\.js",
"website": "lab.hakim.se/reveal-js"
},
"Rickshaw": {
"cats": [
25
],
"env": "^Rickshaw$",
"implies": "D3",
"script": "rickshaw(\\.min)?\\.js",
"website": "http://code.shutterstock.com/rickshaw/"
},
"RiteCMS": {
"cats": [
1
@ -5187,6 +5269,15 @@
"script": "supersized(?:\\.([\\d.]*[\\d]))?.*\\.js\\;version:\\1",
"website": "buildinternet.com/project/supersized"
},
"SweetAlert": {
"cats": [
12
],
"env": "^swal$",
"html": "<link.+?href=\"[^\"]+sweet-alert(?:\\.min)?\\.css",
"script": "sweet-alert(\\.min)?\\.js",
"website": "http://tristanedwards.me/sweetalert"
},
"Swiftlet": {
"cats": [
18
@ -5487,6 +5578,31 @@
"script": "(?:twitter\\.github\\.com/bootstrap|bootstrap(?:\\.js|\\.min\\.js))",
"website": "getbootstrap.com"
},
"Twitter Emoji (Twemoji)": {
"cats": [
25
],
"env": "^twemoji$",
"script": "twemoji(\\.min)?\\.js",
"website": "https://twitter.github.io/twemoji/"
},
"Twitter Flight": {
"cats": [
12
],
"env": "^flight$",
"implies": "jQuery",
"website": "https://flightjs.github.io/"
},
"Twitter typeahead.js": {
"cats": [
12
],
"env": "^typeahead$",
"implies": "jQuery\\;confidence:50",
"script": "(typeahead|bloodhound)(\\.jquery|\\.bundle)?(\\.min)?\\.js",
"website": "https://twitter.github.io/typeahead.js/"
},
"TypePad": {
"cats": [
11
@ -6918,6 +7034,14 @@
"script": "(?:scriptaculous|protoaculous)\\.js",
"website": "script.aculo.us"
},
"shine.js": {
"cats": [
25
],
"env": "^Shine$",
"script": "shine(\\.min)?\\.js",
"website": "http://bigspaceship.github.io/shine.js/"
},
"spin.js": {
"cats": [
12,
@ -6995,6 +7119,15 @@
"html": "powered by <a href=\"[^>]+viennacms",
"website": "www.viennacms.nl"
},
"vis.js": {
"cats": [
25
],
"env": "^vis$",
"html": "<link.+?href=\"[^\"]+vis(?:\\.min)?\\.css",
"script": "vis(\\.min)?\\.js",
"website": "http://visjs.org"
},
"webEdition": {
"cats": [
1
@ -7045,139 +7178,6 @@
"yepnope.*\\.js"
],
"website": "yepnopejs.com"
},
"Open Classifieds": {
"cats": [
6
],
"meta": {
"author": "open-classifieds\\.com",
"copyright": "Open Classifieds ?([0-9.]+)?\\;version:\\1"
},
"website": "http://open-classifieds.com"
},
"Open eShop": {
"cats": [
6
],
"meta": {
"author": "open-eshop\\.com",
"copyright": "Open eShop ?([0-9.]+)?\\;version:\\1"
},
"website": "http://open-eshop.com/"
},
"Hammer.js": {
"cats": [
12
],
"script": "hammer(\\.min)?\\.js",
"env": "^Hammer$",
"website": "http://hammerjs.github.io"
},
"Epoch": {
"cats": [
25
],
"script": "epoch(\\.min)?\\.js",
"html": "<link.+?href=\"[^\"]+epoch(?:\\.min)?\\.css",
"implies": "D3",
"website": "https://fastly.github.io/epoch/"
},
"Boba.js": {
"cats": [
12
],
"script": "boba(\\.min)?\\.js",
"implies": "Google Analytics",
"website": "http://boba.space150.com/"
},
"Lazy.js": {
"cats": [
12
],
"script": "lazy(\\.browser)?(\\.min)?\\.js",
"website": "http://danieltao.com/lazy.js/"
},
"SweetAlert": {
"cats": [
12
],
"env": "^swal$",
"script": "sweet-alert(\\.min)?\\.js",
"html": "<link.+?href=\"[^\"]+sweet-alert(?:\\.min)?\\.css",
"website": "http://tristanedwards.me/sweetalert"
},
"vis.js": {
"cats": [
25
],
"env": "^vis$",
"script": "vis(\\.min)?\\.js",
"html": "<link.+?href=\"[^\"]+vis(?:\\.min)?\\.css",
"website": "http://visjs.org"
},
"shine.js": {
"cats": [
25
],
"env": "^Shine$",
"script": "shine(\\.min)?\\.js",
"website": "http://bigspaceship.github.io/shine.js/"
},
"Paths.js": {
"cats": [
25
],
"script": "paths(\\.min)?\\.js",
"website": "https://github.com/andreaferretti/paths-js"
},
"Twitter Flight": {
"cats": [
12
],
"env": "^flight$",
"implies": "jQuery",
"website": "https://flightjs.github.io/"
},
"Twitter Emoji (Twemoji)": {
"cats": [
25
],
"env": "^twemoji$",
"script": "twemoji(\\.min)?\\.js",
"website": "https://twitter.github.io/twemoji/"
},
"Twitter typeahead.js": {
"cats": [
12
],
"env": "^typeahead$",
"script": "(typeahead|bloodhound)(\\.jquery|\\.bundle)?(\\.min)?\\.js",
"implies": "jQuery\\;confidence:50",
"website": "https://twitter.github.io/typeahead.js/"
},
"Rickshaw": {
"cats": [
25
],
"env": "^Rickshaw$",
"script": "rickshaw(\\.min)?\\.js",
"implies": "D3",
"website": "http://code.shutterstock.com/rickshaw/"
},
"GitLab CI": {
"cats": [
44,
47
],
"meta": {
"description": "GitLab Continuous Integration"
},
"implies": [
"Ruby",
"Ruby on Rails"
],
"website": "https://about.gitlab.com/gitlab-ci/"
}
},
"categories": {