From 8229354d8a76a8f57340e4bdfc822f487e636fab Mon Sep 17 00:00:00 2001 From: Elbert Alias Date: Sun, 22 Feb 2015 14:43:03 +1100 Subject: [PATCH] Added utility scripts --- .gitignore | 2 ++ README.md | 11 ++++---- Vagrantfile | 4 +-- bin/links | 35 ------------------------- bin/package | 10 ------- bin/wappalyzer | 12 +++++++++ bin/wappalyzer-build | 26 ++++++++++++++++++ bin/wappalyzer-help | 11 ++++++++ bin/wappalyzer-links | 58 +++++++++++++++++++++++++++++++++++++++++ bin/wappalyzer-validate | 39 +++++++++++++++++++++++++++ docker/Dockerfile | 20 +++++++++++--- src/apps.json | 4 +-- 12 files changed, 173 insertions(+), 59 deletions(-) delete mode 100755 bin/links delete mode 100755 bin/package create mode 100755 bin/wappalyzer create mode 100755 bin/wappalyzer-build create mode 100755 bin/wappalyzer-help create mode 100755 bin/wappalyzer-links create mode 100755 bin/wappalyzer-validate diff --git a/.gitignore b/.gitignore index 84735cebe..45ba46273 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ .vagrant +build/* + node_modules/* drivers/npm/node_modules/ diff --git a/README.md b/README.md index 1f8962d73..965fce4fe 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,13 @@ Refer to the [wiki](https://github.com/ElbertF/Wappalyzer/wiki) for ## Getting Started This section describes how to set up a development environment. Everything you -need is contained in a Docker image which is managed by Vagrant. +need is contained in a [Docker image](https://registry.hub.docker.com/u/wappalyzer/dev/) +which is managed by Vagrant. Running this environment optional but recommended as it provides some helpful tools. -First, install Docker and Vagrant on your system. - -* Docker ([docker.com](https://www.docker.com/)) -* Vagrant ([vagrantup.com](https://www.vagrantup.com/)) +First, install [Docker](https://www.docker.com/) and [Vagrant](https://www.vagrantup.com/) +on your system. Clone the Wappalyzer repository and open the newly created directory in a terminal. Run `vagrant up` to start the environment. @@ -42,4 +41,4 @@ $ vagrant up To stop the environment, run `vagrant halt`. -In order to access the environment, run `vagrant ssh`. +Run `vagrant ssh` to access the environment. diff --git a/Vagrantfile b/Vagrantfile index 0346a4ba6..42d1e32c9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -15,6 +15,6 @@ Vagrant.configure("2") do |config| config.ssh.username = "wappalyzer" config.ssh.private_key_path = "docker/insecure.key" - config.vm.provision "shell", path: "bin/links" - config.vm.provision "shell", inline: "echo Finished" + config.vm.provision "shell", inline: "su - wappalyzer -c 'wappalyzer links'" + config.vm.provision "shell", inline: "echo Finished. Run \\`vagrant ssh\\` to access the environment." end diff --git a/bin/links b/bin/links deleted file mode 100755 index e18e64f65..000000000 --- a/bin/links +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh - -dir="/home/wappalyzer/synced/src" - -ln -f $dir/wappalyzer.js $dir/drivers/firefox/lib -ln -f $dir/apps.json $dir/drivers/firefox/data -ln -f $dir/icons/*.png $dir/drivers/firefox/data/images/icons - -ln -f $dir/wappalyzer.js $dir/drivers/chrome/js -ln -f $dir/apps.json $dir/drivers/chrome -ln -f $dir/icons/*.png $dir/drivers/chrome/images/icons - -ln -f $dir/wappalyzer.js $dir/drivers/bookmarklet/js -ln -f $dir/icons/*.png $dir/drivers/bookmarklet/images/icons - -ln -f $dir/wappalyzer.js $dir/drivers/html/js -ln -f $dir/apps.json $dir/drivers/html -ln -f $dir/icons/*.png $dir/drivers/html/images/icons - -ln -f $dir/wappalyzer.js $dir/drivers/php/js -ln -f $dir/apps.json $dir/drivers/php - -ln -f $dir/wappalyzer.js $dir/drivers/python/js -ln -f $dir/apps.json $dir/drivers/python - -ln -f $dir/wappalyzer.js $dir/drivers/phantomjs/js -ln -f $dir/apps.json $dir/drivers/phantomjs - -ln -f $dir/wappalyzer.js $dir/drivers/ruby/js -ln -f $dir/apps.json $dir/drivers/ruby - -ln -f $dir/wappalyzer.js $dir/drivers/npm -ln -f $dir/apps.json $dir/drivers/npm - -echo "Hard links created" diff --git a/bin/package b/bin/package deleted file mode 100755 index 1628df7fa..000000000 --- a/bin/package +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -path=`pwd` - -./links.sh - -rm -f wappalyzer-chrome.zip wappalyzer-firefox.xpi - -cd $path/drivers/chrome && zip -r $path/wappalyzer-chrome.zip . -#cd $path/drivers/firefox && zip -r $path/wappalyzer-firefox.xpi . diff --git a/bin/wappalyzer b/bin/wappalyzer new file mode 100755 index 000000000..5d42480ac --- /dev/null +++ b/bin/wappalyzer @@ -0,0 +1,12 @@ +#!/bin/bash + +if [ -f "$0-$1" ] +then + $0-$1 ${*:2} +else + $0-help + + exit 1 +fi + +exit 0 diff --git a/bin/wappalyzer-build b/bin/wappalyzer-build new file mode 100755 index 000000000..d544d05c4 --- /dev/null +++ b/bin/wappalyzer-build @@ -0,0 +1,26 @@ +#!/bin/bash + +if [ -z $WAPPALYZER_ROOT ] +then + echo "\$WAPPALYZER_ROOT not set" + + exit 1 +fi + +wappalyzer links + +# Mozilla Firefox +echo "Building Firefox driver..." + +cfx xpi --pkgdir=$WAPPALYZER_ROOT/src/drivers/firefox --output-file=$WAPPALYZER_ROOT/build/wappalyzer_firefox.xpi + +# Google Chrome +echo "Building Chrome driver..." + +pushd $WAPPALYZER_ROOT/src/drivers/chrome > /dev/null + +zip -qr $WAPPALYZER_ROOT/build/wappalyzer_chrome.zip . + +popd > /dev/null + +echo "Done. Builds have been created in $WAPPALYZER_ROOT/build." diff --git a/bin/wappalyzer-help b/bin/wappalyzer-help new file mode 100755 index 000000000..d832f88c1 --- /dev/null +++ b/bin/wappalyzer-help @@ -0,0 +1,11 @@ +#!/bin/bash + +cat << 'EOF' +usage: wappalyzer [] + +The following commands are available: + build Package drivers + help This help text + links Create hard links to shared files in each driver + validate Validate apps.json +EOF diff --git a/bin/wappalyzer-links b/bin/wappalyzer-links new file mode 100755 index 000000000..3e7fe5baf --- /dev/null +++ b/bin/wappalyzer-links @@ -0,0 +1,58 @@ +#!/bin/bash + +path=$1 + +if [ -z $path ] +then + if [ -z $WAPPALYZER_ROOT ] + then + echo "-$(basename $0): No path specified" + + exit 1 + fi + + path=$WAPPALYZER_ROOT +fi + +if [ ! -d $path/src ] +then + echo "-$(basename $0): Incorrect path" + + exit 1 +fi + +path="$path/src" + +ln -f $path/wappalyzer.js $path/drivers/firefox/lib +ln -f $path/apps.json $path/drivers/firefox/data +ln -f $path/icons/*.png $path/drivers/firefox/data/images/icons + +ln -f $path/wappalyzer.js $path/drivers/chrome/js +ln -f $path/apps.json $path/drivers/chrome +ln -f $path/icons/*.png $path/drivers/chrome/images/icons + +ln -f $path/wappalyzer.js $path/drivers/bookmarklet/js +ln -f $path/icons/*.png $path/drivers/bookmarklet/images/icons + +ln -f $path/wappalyzer.js $path/drivers/html/js +ln -f $path/apps.json $path/drivers/html +ln -f $path/icons/*.png $path/drivers/html/images/icons + +ln -f $path/wappalyzer.js $path/drivers/php/js +ln -f $path/apps.json $path/drivers/php + +ln -f $path/wappalyzer.js $path/drivers/python/js +ln -f $path/apps.json $path/drivers/python + +ln -f $path/wappalyzer.js $path/drivers/phantomjs/js +ln -f $path/apps.json $path/drivers/phantomjs + +ln -f $path/wappalyzer.js $path/drivers/ruby/js +ln -f $path/apps.json $path/drivers/ruby + +ln -f $path/wappalyzer.js $path/drivers/npm +ln -f $path/apps.json $path/drivers/npm + +echo "Hard links created" + +exit 0 diff --git a/bin/wappalyzer-validate b/bin/wappalyzer-validate new file mode 100755 index 000000000..8ef64f769 --- /dev/null +++ b/bin/wappalyzer-validate @@ -0,0 +1,39 @@ +#!/bin/bash + +color_ok='\033[1;32m' +color_err='\033[0;31m' +color_none='\033[0m' + +path=$1 + +if [ -z $path ] +then + if [ -z $WAPPALYZER_ROOT ] + then + echo "-$(basename $0): No path specified" + + exit 1 + fi + + path=$WAPPALYZER_ROOT +fi + +if [ ! -d $path/src ] +then + echo "-$(basename $0): Incorrect path" + + exit 1 +fi + +path="$path/src" + +echo -en "${color_err}" + +jsonlint --quiet $path/apps.json + +if [ $? -eq 0 ] +then + echo -e "${color_ok}No errors found" +fi + +echo -en "${color_none}" diff --git a/docker/Dockerfile b/docker/Dockerfile index 1b5f96428..45d92091f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,7 @@ MAINTAINER Elbert Alias ENV DEBIAN_FRONTEND noninteractive # Install packages -RUN apt-get update && apt-get install -y nodejs npm +RUN apt-get update && apt-get install -y curl nodejs npm zip RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN ln -s /usr/bin/nodejs /usr/bin/node @@ -18,10 +18,22 @@ RUN echo 'wappalyzer ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers WORKDIR /home/wappalyzer -RUN su wappalyzer -c "mkdir -p synced" +RUN su wappalyzer -c "mkdir bin synced" -RUN echo "cd /home/wappalyzer/synced" >> .bashrc -RUN echo "export PATH=$PATH:/home/wappalyzer/synced/bin" >> .bashrc +RUN echo "export WAPPALYZER_ROOT='/home/wappalyzer/synced'" >> /tmp/profile +RUN echo "export PATH=$PATH:/home/wappalyzer/bin:\$WAPPALYZER_ROOT/bin" >> /tmp/profile + +RUN cat .profile >> /tmp/profile && mv /tmp/profile .profile + +RUN echo "cd \$WAPPALYZER_ROOT" >> .bashrc +RUN echo "wappalyzer" >> .bashrc + + +# Mozilla Add-on SDK +RUN su wappalyzer -c "\ + mkdir mozilla && \ + curl -L https://ftp.mozilla.org/pub/mozilla.org/labs/jetpack/jetpack-sdk-latest.tar.gz | tar xvzC mozilla && \ + ln -s /home/wappalyzer/mozilla/addon-sdk-*/bin/cfx bin/cfx" # SSH diff --git a/src/apps.json b/src/apps.json index 07574e008..ca09d6598 100755 --- a/src/apps.json +++ b/src/apps.json @@ -3409,7 +3409,7 @@ "Snap": { "website": "snapframework.com", "cats": [ 18, 22 ], - "headers": { "Server": "Snap/\d+(\.\d+)+" }, + "headers": { "Server": "Snap/\\d+(\\.\\d+)+" }, "implies": "Haskell" }, "Snap.svg": { @@ -4280,7 +4280,7 @@ "Yesod": { "website": "www.yesodweb.com", "cats": [ 18, 22 ], - "headers": { "Server": "Warp/\d+(\.\d+)+" }, + "headers": { "Server": "Warp/\\d+(\\.\\d+)+" }, "implies": "Haskell" }, "Yieldlab": {