diff --git a/.gitignore b/.gitignore index 2f0d797d8..36d3e091e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,18 +3,9 @@ build/* -npm-debug.log - -node_modules/* - -bin/node_modules/ - -bin/npm/npm-debug.log - -drivers/npm/node_modules/ - +drivers/npm/node_modules drivers/npm/npm-debug.log -!.gitkeep - Thumbs.db + +!.gitkeep diff --git a/.travis.yml b/.travis.yml index 4e740fe58..75f2fdc82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,13 +3,14 @@ node_js: - "0.10" before_install: - export WAPPALYZER_ROOT=$TRAVIS_BUILD_DIR + - export WAPPALYZER_NODE_PATH=$TRAVIS_BUILD_DIR - export PATH=$PATH:$TRAVIS_BUILD_DIR/bin install: - sudo apt-get update -y - sudo apt-get install -y curl zip sudo -y --force-yes - sudo apt-get clean - npm install jsonlint -g - - ln -s bin/package.json package.json && npm install + - ln -s docker/node/package.json package.json && npm install - mkdir mozilla && curl -L https://ftp.mozilla.org/pub/mozilla.org/labs/jetpack/jetpack-sdk-latest.tar.gz | tar xvzC mozilla && ln -s $WAPPALYZER_ROOT/mozilla/addon-sdk-*/bin/cfx bin/cfx - mkdir phantomjs && curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2 | tar xvjC phantomjs && ln -s $WAPPALYZER_ROOT/phantomjs/phantomjs-*-linux-x86_64/bin/phantomjs bin/phantomjs script: wappalyzer build diff --git a/Vagrantfile b/Vagrantfile index 42d1e32c9..b4d0f218c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,7 +13,7 @@ Vagrant.configure("2") do |config| config.ssh.port = "22" config.ssh.username = "wappalyzer" - config.ssh.private_key_path = "docker/insecure.key" + config.ssh.private_key_path = "docker/ssh/insecure.key" config.vm.provision "shell", inline: "su - wappalyzer -c 'wappalyzer links'" config.vm.provision "shell", inline: "echo Finished. Run \\`vagrant ssh\\` to access the environment." diff --git a/bin/wappalyzer-validate b/bin/wappalyzer-validate index deadd724b..41018c211 100755 --- a/bin/wappalyzer-validate +++ b/bin/wappalyzer-validate @@ -31,4 +31,4 @@ jsonlint --quiet -V $WAPPALYZER_ROOT/schema.json $path/apps.json echo "Validating icons..." -node $WAPPALYZER_ROOT/bin/wappalyzer-validate-icons.js $path/apps.json $path/icons \ No newline at end of file +wappalyzer validate-icons diff --git a/bin/wappalyzer-validate-icons b/bin/wappalyzer-validate-icons new file mode 100755 index 000000000..49676f0f0 --- /dev/null +++ b/bin/wappalyzer-validate-icons @@ -0,0 +1,26 @@ +#!/usr/bin/env node + +var + modulesPath = process.env.WAPPALYZER_NODE_PATH !== undefined ? process.env.WAPPALYZER_NODE_PATH + '/node_modules/' : '', + fileType = require(modulesPath + 'file-type'), + fs = require(modulesPath + 'fs-extra'), + readChunk = require(modulesPath + 'read-chunk') + json = require(process.env.WAPPALYZER_ROOT + '/src/apps.json'); + +Object.keys(json.apps).forEach(function(app) { + var path = process.env.WAPPALYZER_ROOT + '/src/icons/' + app + '.png'; + + fs.exists(path, function(exists) { + var buffer; + + if ( exists ) { + buffer = fileType(readChunk.sync(path, 0, 262)); + + if ( buffer.mime !== 'image/png' ) { + throw new Error('Incorrect mimetype "' + buffer.mime + '": src/icons/' + app + '.png'); + } + } else { + throw new Error('Missing file: src/icons/' + app + '.png'); + } + }); +}); diff --git a/bin/wappalyzer-validate-icons.js b/bin/wappalyzer-validate-icons.js deleted file mode 100644 index 9e984d96a..000000000 --- a/bin/wappalyzer-validate-icons.js +++ /dev/null @@ -1,68 +0,0 @@ -var readChunk = require('read-chunk'); -var fileType = require('file-type'); -var path = require('path'); -var fs = require('fs-extra'); -var async = require('async'); -var glob = require('glob'); - -var appsJSON = require(process.argv[2]); -var iconsDir = process.argv[3]; - -var appsIconPaths = []; - -function arrayDiff(a1, a2) { - var o1={}, o2={}, diff=[], i, len, k; - for (i=0, len=a1.length; i 1 && arr.length) { - what = a[--L]; - while ((ax= arr.indexOf(what)) !== -1) { - arr.splice(ax, 1); - } - } - return arr; -} - -async.each(Object.keys(appsJSON.apps), function (app, callback) { - glob(iconsDir + "/" + app + ".+(png|gif|jpg|jpeg|ico|icon|icns|tiff|tif|svg|bmp|psd|pspimage|thm|yuv|ai|drw|eps|ps)", function (err, files) { - if (err) throw err; - if (files.length < 1) { - var err = new Error("There is no icon for '" + app + "'!"); - throw err; - } else if (files.length > 1) { - var err = new Error("There is more than one icon for '" + app + "'!"); - throw err; - } else { - if (files[0].split('.').pop() !== 'png') { - var err = new Error("The icon at " + files[0] + " does not have a '.png' extension!"); - throw err; - } else { - var buffer = fileType(readChunk.sync(files[0], 0, 262)); - if (buffer.mime !== 'image/png' || buffer.ext !== 'png') { - var err = new Error("The icon at " + files[0] + " has a '.png' extension, but it is not actually a PNG file! It is actually a " + buffer.mime + " which usually has an extension of '" + buffer.ext + "'."); - throw err; - } else { - appsIconPaths.push(path.basename(files[0])); - callback(); - } - } - } - }); -}, function(err) { - if (err) throw err; - fs.readdir(iconsDir, function(err, iconsList) { - if (err) throw err; - iconsList = removeA(iconsList, 'Thumbs.db'); // While Thumbs.db is excluded from git, Windows still adds it and it messes up tests - appsIconPaths.push("default.png"); - if (appsIconPaths.length < iconsList.length) { - var err = new Error("There are " + (iconsList.length - appsIconPaths.length) + " more files in the icons directory (" + iconsDir + ") than there are apps! There are " + appsIconPaths.length + " verified icons (one is the default), but there are " + iconsList.length + " total files." + "\n" + "The extra files are: " + arrayDiff(iconsList, appsIconPaths)); - throw err; - } - }); -}); \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 5d60ae6c2..28a17858c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,35 +1,40 @@ FROM phusion/baseimage + MAINTAINER Elbert Alias ENV DEBIAN_FRONTEND noninteractive +ENV WAPPALYZER_ROOT /home/wappalyzer/synced +ENV WAPPALYZER_NODE_PATH /home/wappalyzer/node + + # Install packages RUN apt-get update && apt-get install -y curl nodejs npm zip RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN ln -s /usr/bin/nodejs /usr/bin/node -RUN npm install jsonlint -g # Add user -RUN useradd -ms /bin/bash wappalyzer && echo "wappalyzer:wappalyzer" | chpasswd +RUN useradd -ms /bin/bash wappalyzer && usermod -a -G docker_env wappalyzer && echo "wappalyzer:wappalyzer" | chpasswd RUN echo 'wappalyzer ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers WORKDIR /home/wappalyzer RUN su wappalyzer -c "mkdir bin synced" - -RUN echo "export WAPPALYZER_ROOT='/home/wappalyzer/synced'" >> /tmp/profile -RUN echo "export PATH=$PATH:/home/wappalyzer/bin:\$WAPPALYZER_ROOT/bin" >> /tmp/profile - -RUN cat .profile >> /tmp/profile && mv /tmp/profile .profile +RUN su wappalyzer -c "echo \"export PATH=\$PATH:/home/wappalyzer/bin:\\$WAPPALYZER_ROOT/bin\" | cat - .profile > /tmp/profile && mv /tmp/profile .profile" RUN echo "cd \$WAPPALYZER_ROOT" >> .bashrc RUN echo "wappalyzer" >> .bashrc -RUN su wappalyzer -c "\ - ln -s bin/package.json package.json && \ - npm install" + +# Node JS +RUN su wappalyzer -c "mkdir $WAPPALYZER_NODE_PATH" + +ADD node/package.json $WAPPALYZER_NODE_PATH/package.json + +RUN npm install jsonlint -g && su wappalyzer -c "cd $WAPPALYZER_NODE_PATH && npm install" + # Mozilla Add-on SDK RUN su wappalyzer -c "\ @@ -49,7 +54,7 @@ RUN su wappalyzer -c "\ RUN rm -f /etc/service/sshd/down RUN su wappalyzer -c "mkdir .ssh && chmod 700 .ssh" -ADD insecure.key.pub /tmp/insecure.key.pub +ADD ssh/insecure.key.pub /tmp/insecure.key.pub RUN su wappalyzer -c "cat /tmp/insecure.key.pub >> .ssh/authorized_keys && chmod 600 .ssh/authorized_keys" && rm -f /tmp/insecure.key.pub @@ -57,7 +62,6 @@ RUN su wappalyzer -c "cat /tmp/insecure.key.pub >> .ssh/authorized_keys && chmod # Fix the `stdin: is not a tty` error in Vagrant RUN sed -i 's/^mesg n$/tty -s \&\& mesg n/g' /root/.profile - EXPOSE 22 RUN echo "/usr/sbin/sshd -D" > /etc/my_init.d/sshd.sh diff --git a/bin/package.json b/docker/node/package.json similarity index 71% rename from bin/package.json rename to docker/node/package.json index 052b3ccf1..db55543bb 100644 --- a/bin/package.json +++ b/docker/node/package.json @@ -1,9 +1,7 @@ { "devDependencies": { - "async": "0.9.*", "file-type": "2.2.*", "fs-extra": "0.16.*", - "glob": "4.4.*", "read-chunk": "1.0.*" } } diff --git a/docker/insecure.key b/docker/ssh/insecure.key similarity index 100% rename from docker/insecure.key rename to docker/ssh/insecure.key diff --git a/docker/insecure.key.pub b/docker/ssh/insecure.key.pub similarity index 100% rename from docker/insecure.key.pub rename to docker/ssh/insecure.key.pub diff --git a/src/apps.json b/src/apps.json index bbc9e1c09..6448eb852 100755 --- a/src/apps.json +++ b/src/apps.json @@ -661,6 +661,14 @@ }, "website": "www.boa.org" }, + "Boba.js": { + "cats": [ + 12 + ], + "implies": "Google Analytics", + "script": "boba(\\.min)?\\.js", + "website": "http://boba.space150.com/" + }, "Bolt": { "cats": [ 1 @@ -1710,6 +1718,15 @@ "script": "enyo\\.js", "website": "enyojs.com" }, + "Epoch": { + "cats": [ + 25 + ], + "html": "", - "", "website": "openwebanalytics.com" }, + "Open eShop": { + "cats": [ + 6 + ], + "meta": { + "author": "open-eshop\\.com", + "copyright": "Open eShop ?([0-9.]+)?\\;version:\\1" + }, + "website": "http://open-eshop.com/" + }, "OpenCart": { "cats": [ 6 @@ -4079,6 +4145,13 @@ "env": "^PARSELY$", "website": "parse.ly" }, + "Paths.js": { + "cats": [ + 25 + ], + "script": "paths(\\.min)?\\.js", + "website": "https://github.com/andreaferretti/paths-js" + }, "PayPal": { "cats": [ 41 @@ -4508,6 +4581,15 @@ "script": "reveal(?:\\.min)?\\.js", "website": "lab.hakim.se/reveal-js" }, + "Rickshaw": { + "cats": [ + 25 + ], + "env": "^Rickshaw$", + "implies": "D3", + "script": "rickshaw(\\.min)?\\.js", + "website": "http://code.shutterstock.com/rickshaw/" + }, "RiteCMS": { "cats": [ 1 @@ -5187,6 +5269,15 @@ "script": "supersized(?:\\.([\\d.]*[\\d]))?.*\\.js\\;version:\\1", "website": "buildinternet.com/project/supersized" }, + "SweetAlert": { + "cats": [ + 12 + ], + "env": "^swal$", + "html": "]+viennacms", "website": "www.viennacms.nl" }, + "vis.js": { + "cats": [ + 25 + ], + "env": "^vis$", + "html": "