Refactoring of wappalyzer validate-icons

10 years ago · d6debd16bb
parent 5602b9223b
commit d6debd16bb
11 changed files with 204 additions and 252 deletions
--- a/.gitignore
+++ b/.gitignore
@ -3,18 +3,9 @@

 build/*

-npm-debug.log
-
-node_modules/*
-
-bin/node_modules/
-
-bin/npm/npm-debug.log
-
-drivers/npm/node_modules/
-
+drivers/npm/node_modules
 drivers/npm/npm-debug.log

-!.gitkeep
-
 Thumbs.db
+
+!.gitkeep
--- a/.travis.yml
+++ b/.travis.yml
@ -3,13 +3,14 @@ node_js:
  - "0.10"
 before_install:
  - export WAPPALYZER_ROOT=$TRAVIS_BUILD_DIR
+  - export WAPPALYZER_NODE_PATH=$TRAVIS_BUILD_DIR
  - export PATH=$PATH:$TRAVIS_BUILD_DIR/bin
 install:
  - sudo apt-get update -y
  - sudo apt-get install -y curl zip sudo -y --force-yes
  - sudo apt-get clean
  - npm install jsonlint -g
-  - ln -s bin/package.json package.json && npm install
+  - ln -s docker/node/package.json package.json && npm install
  - mkdir mozilla && curl -L https://ftp.mozilla.org/pub/mozilla.org/labs/jetpack/jetpack-sdk-latest.tar.gz | tar xvzC mozilla && ln -s $WAPPALYZER_ROOT/mozilla/addon-sdk-*/bin/cfx bin/cfx
  - mkdir phantomjs && curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2 | tar xvjC phantomjs && ln -s $WAPPALYZER_ROOT/phantomjs/phantomjs-*-linux-x86_64/bin/phantomjs bin/phantomjs
 script: wappalyzer build
--- a/2
+++ b/2
@ -13,7 +13,7 @@ Vagrant.configure("2") do |config|

  config.ssh.port             = "22"
  config.ssh.username         = "wappalyzer"
-  config.ssh.private_key_path = "docker/insecure.key"
+  config.ssh.private_key_path = "docker/ssh/insecure.key"

  config.vm.provision "shell", inline: "su - wappalyzer -c 'wappalyzer links'"
  config.vm.provision "shell", inline: "echo Finished. Run \\`vagrant ssh\\` to access the environment."
--- a/bin/wappalyzer-validate
+++ b/bin/wappalyzer-validate
@ -31,4 +31,4 @@ jsonlint --quiet -V $WAPPALYZER_ROOT/schema.json $path/apps.json

 echo "Validating icons..."

-node $WAPPALYZER_ROOT/bin/wappalyzer-validate-icons.js $path/apps.json $path/icons
+wappalyzer validate-icons
--- a/bin/wappalyzer-validate-icons
+++ b/bin/wappalyzer-validate-icons
@ -0,0 +1,26 @@
+#!/usr/bin/env node
+
+var
+	modulesPath = process.env.WAPPALYZER_NODE_PATH !== undefined ? process.env.WAPPALYZER_NODE_PATH + '/node_modules/' : '',
+	fileType    = require(modulesPath + 'file-type'),
+	fs          = require(modulesPath + 'fs-extra'),
+	readChunk   = require(modulesPath + 'read-chunk')
+	json        = require(process.env.WAPPALYZER_ROOT + '/src/apps.json');
+
+Object.keys(json.apps).forEach(function(app) {
+	var path = process.env.WAPPALYZER_ROOT + '/src/icons/' + app + '.png';
+
+	fs.exists(path, function(exists) {
+		var buffer;
+
+		if ( exists ) {
+			buffer = fileType(readChunk.sync(path, 0, 262));
+
+			if ( buffer.mime !== 'image/png' ) {
+				throw new Error('Incorrect mimetype "' + buffer.mime + '": src/icons/' + app + '.png');
+			}
+		} else {
+			throw new Error('Missing file: src/icons/' + app + '.png');
+		}
+	});
+});
--- a/bin/wappalyzer-validate-icons.js
+++ b/bin/wappalyzer-validate-icons.js
@ -1,68 +0,0 @@
-var readChunk = require('read-chunk');
-var fileType = require('file-type');
-var path = require('path');
-var fs = require('fs-extra');
-var async = require('async');
-var glob = require('glob');
-
-var appsJSON = require(process.argv[2]);
-var iconsDir = process.argv[3];
-
-var appsIconPaths = [];
-
-function arrayDiff(a1, a2) {
-  var o1={}, o2={}, diff=[], i, len, k;
-  for (i=0, len=a1.length; i<len; i++) { o1[a1[i]] = true; }
-  for (i=0, len=a2.length; i<len; i++) { o2[a2[i]] = true; }
-  for (k in o1) { if (!(k in o2)) { diff.push(k); } }
-  for (k in o2) { if (!(k in o1)) { diff.push(k); } }
-  return diff;
-}
-function removeA(arr) {
-    var what, a = arguments, L = a.length, ax;
-    while (L > 1 && arr.length) {
-        what = a[--L];
-        while ((ax= arr.indexOf(what)) !== -1) {
-            arr.splice(ax, 1);
-        }
-    }
-    return arr;
-}
-
-async.each(Object.keys(appsJSON.apps), function (app, callback) {
-	glob(iconsDir + "/" + app + ".+(png|gif|jpg|jpeg|ico|icon|icns|tiff|tif|svg|bmp|psd|pspimage|thm|yuv|ai|drw|eps|ps)", function (err, files) {
-		if (err) throw err;
-		if (files.length < 1) {
-			var err = new Error("There is no icon for '" + app + "'!");
-			throw err;
-		} else if (files.length > 1) {
-			var err = new Error("There is more than one icon for '" + app + "'!");
-			throw err;
-		} else {
-			if (files[0].split('.').pop() !== 'png') {
-				var err = new Error("The icon at " + files[0] + " does not have a '.png' extension!");
-				throw err;
-			} else {
-				var buffer = fileType(readChunk.sync(files[0], 0, 262));
-				if (buffer.mime !== 'image/png' || buffer.ext !== 'png') {
-					var err = new Error("The icon at " + files[0] + " has a '.png' extension, but it is not actually a PNG file! It is actually a " + buffer.mime + " which usually has an extension of '" + buffer.ext + "'.");
-					throw err;
-				} else {
-					appsIconPaths.push(path.basename(files[0]));
-					callback();
-				}
-			}
-		}
-	});
-}, function(err) {
-	if (err) throw err;
-	fs.readdir(iconsDir, function(err, iconsList) {
-		if (err) throw err;
-		iconsList = removeA(iconsList, 'Thumbs.db');				// While Thumbs.db is excluded from git, Windows still adds it and it messes up tests
-		appsIconPaths.push("default.png");
-		if (appsIconPaths.length < iconsList.length) {
-			var err = new Error("There are " + (iconsList.length - appsIconPaths.length) + " more files in the icons directory (" + iconsDir + ") than there are apps! There are " + appsIconPaths.length + " verified icons (one is the default), but there are " + iconsList.length + " total files." + "\n" + "The extra files are: " + arrayDiff(iconsList, appsIconPaths));
-			throw err;
-		}
-	});
-});
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -1,35 +1,40 @@
 FROM phusion/baseimage
+
 MAINTAINER Elbert Alias <elbert@alias.io>

 ENV DEBIAN_FRONTEND noninteractive

+ENV WAPPALYZER_ROOT /home/wappalyzer/synced
+ENV WAPPALYZER_NODE_PATH /home/wappalyzer/node
+
+
 # Install packages
 RUN apt-get update && apt-get install -y curl nodejs npm zip
 RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

 RUN ln -s /usr/bin/nodejs /usr/bin/node

-RUN npm install jsonlint -g

 # Add user
-RUN useradd -ms /bin/bash wappalyzer && echo "wappalyzer:wappalyzer" | chpasswd
+RUN useradd -ms /bin/bash wappalyzer && usermod -a -G docker_env wappalyzer && echo "wappalyzer:wappalyzer" | chpasswd
 RUN echo 'wappalyzer ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers

 WORKDIR /home/wappalyzer

 RUN su wappalyzer -c "mkdir bin synced"
-
-RUN echo "export WAPPALYZER_ROOT='/home/wappalyzer/synced'" >> /tmp/profile
-RUN echo "export PATH=$PATH:/home/wappalyzer/bin:\$WAPPALYZER_ROOT/bin" >> /tmp/profile
-
-RUN cat .profile >> /tmp/profile && mv /tmp/profile .profile
+RUN su wappalyzer -c "echo \"export PATH=\$PATH:/home/wappalyzer/bin:\\$WAPPALYZER_ROOT/bin\" | cat - .profile > /tmp/profile && mv /tmp/profile .profile"

 RUN echo "cd \$WAPPALYZER_ROOT" >> .bashrc
 RUN echo "wappalyzer" >> .bashrc

-RUN su wappalyzer -c "\
-	ln -s bin/package.json package.json && \
-	npm install"
+
+# Node JS
+RUN su wappalyzer -c "mkdir $WAPPALYZER_NODE_PATH"
+
+ADD node/package.json $WAPPALYZER_NODE_PATH/package.json
+
+RUN npm install jsonlint -g && su wappalyzer -c "cd $WAPPALYZER_NODE_PATH && npm install"
+

 # Mozilla Add-on SDK
 RUN su wappalyzer -c "\
@ -49,7 +54,7 @@ RUN su wappalyzer -c "\
 RUN rm -f /etc/service/sshd/down
 RUN su wappalyzer -c "mkdir .ssh && chmod 700 .ssh"

-ADD insecure.key.pub /tmp/insecure.key.pub
+ADD ssh/insecure.key.pub /tmp/insecure.key.pub

 RUN su wappalyzer -c "cat /tmp/insecure.key.pub >> .ssh/authorized_keys && chmod 600 .ssh/authorized_keys" && rm -f /tmp/insecure.key.pub

@ -57,7 +62,6 @@ RUN su wappalyzer -c "cat /tmp/insecure.key.pub >> .ssh/authorized_keys && chmod
 # Fix the `stdin: is not a tty` error in Vagrant
 RUN sed -i 's/^mesg n$/tty -s \&\& mesg n/g' /root/.profile

-
 EXPOSE 22

 RUN echo "/usr/sbin/sshd -D" > /etc/my_init.d/sshd.sh
--- a/docker/node/package.json
+++ b/docker/node/package.json
@ -1,9 +1,7 @@
 {
  "devDependencies": {
-    "async": "0.9.*",
    "file-type": "2.2.*",
    "fs-extra": "0.16.*",
-    "glob": "4.4.*",
    "read-chunk": "1.0.*"
  }
 }
--- a/docker/ssh/insecure.key
+++ b/docker/ssh/insecure.key
--- a/docker/ssh/insecure.key.pub
+++ b/docker/ssh/insecure.key.pub
--- a/src/apps.json
+++ b/src/apps.json
@ -661,6 +661,14 @@
 			},
 			"website": "www.boa.org"
 		},
+		"Boba.js": {
+			"cats": [
+				12
+			],
+			"implies": "Google Analytics",
+			"script": "boba(\\.min)?\\.js",
+			"website": "http://boba.space150.com/"
+		},
 		"Bolt": {
 			"cats": [
 				1
@ -1710,6 +1718,15 @@
 			"script": "enyo\\.js",
 			"website": "enyojs.com"
 		},
+		"Epoch": {
+			"cats": [
+				25
+			],
+			"html": "<link.+?href=\"[^\"]+epoch(?:\\.min)?\\.css",
+			"implies": "D3",
+			"script": "epoch(\\.min)?\\.js",
+			"website": "https://fastly.github.io/epoch/"
+		},
 		"Epom": {
 			"cats": [
 				36
@ -2039,6 +2056,20 @@
 			],
 			"website": "about.gitlab.com"
 		},
+		"GitLab CI": {
+			"cats": [
+				44,
+				47
+			],
+			"implies": [
+				"Ruby",
+				"Ruby on Rails"
+			],
+			"meta": {
+				"description": "GitLab Continuous Integration"
+			},
+			"website": "https://about.gitlab.com/gitlab-ci/"
+		},
 		"GlassFish": {
 			"cats": [
 				22
@ -2307,6 +2338,14 @@
 			},
 			"website": "???"
 		},
+		"Hammer.js": {
+			"cats": [
+				12
+			],
+			"env": "^Hammer$",
+			"script": "hammer(\\.min)?\\.js",
+			"website": "http://hammerjs.github.io"
+		},
 		"Handlebars": {
 			"cats": [
 				12
@ -3026,6 +3065,13 @@
 			"implies": "PHP",
 			"website": "laravel.com"
 		},
+		"Lazy.js": {
+			"cats": [
+				12
+			],
+			"script": "lazy(\\.browser)?(\\.min)?\\.js",
+			"website": "http://danieltao.com/lazy.js/"
+		},
 		"Leaflet": {
 			"cats": [
 				35
@ -3818,6 +3864,16 @@
 			"env": "^OAS_AD$",
 			"website": "xaxis.com"
 		},
+		"Open Classifieds": {
+			"cats": [
+				6
+			],
+			"meta": {
+				"author": "open-classifieds\\.com",
+				"copyright": "Open Classifieds ?([0-9.]+)?\\;version:\\1"
+			},
+			"website": "http://open-classifieds.com"
+		},
 		"Open Journal Systems": {
 			"cats": [
 				50
@ -3841,6 +3897,16 @@
 			"html": "<!-- (?:Start|End) Open Web Analytics Tracker -->",
 			"website": "openwebanalytics.com"
 		},
+		"Open eShop": {
+			"cats": [
+				6
+			],
+			"meta": {
+				"author": "open-eshop\\.com",
+				"copyright": "Open eShop ?([0-9.]+)?\\;version:\\1"
+			},
+			"website": "http://open-eshop.com/"
+		},
 		"OpenCart": {
 			"cats": [
 				6
@ -4079,6 +4145,13 @@
 			"env": "^PARSELY$",
 			"website": "parse.ly"
 		},
+		"Paths.js": {
+			"cats": [
+				25
+			],
+			"script": "paths(\\.min)?\\.js",
+			"website": "https://github.com/andreaferretti/paths-js"
+		},
 		"PayPal": {
 			"cats": [
 				41
@ -4508,6 +4581,15 @@
 			"script": "reveal(?:\\.min)?\\.js",
 			"website": "lab.hakim.se/reveal-js"
 		},
+		"Rickshaw": {
+			"cats": [
+				25
+			],
+			"env": "^Rickshaw$",
+			"implies": "D3",
+			"script": "rickshaw(\\.min)?\\.js",
+			"website": "http://code.shutterstock.com/rickshaw/"
+		},
 		"RiteCMS": {
 			"cats": [
 				1
@ -5187,6 +5269,15 @@
 			"script": "supersized(?:\\.([\\d.]*[\\d]))?.*\\.js\\;version:\\1",
 			"website": "buildinternet.com/project/supersized"
 		},
+		"SweetAlert": {
+			"cats": [
+				12
+			],
+			"env": "^swal$",
+			"html": "<link.+?href=\"[^\"]+sweet-alert(?:\\.min)?\\.css",
+			"script": "sweet-alert(\\.min)?\\.js",
+			"website": "http://tristanedwards.me/sweetalert"
+		},
 		"Swiftlet": {
 			"cats": [
 				18
@ -5487,6 +5578,31 @@
 			"script": "(?:twitter\\.github\\.com/bootstrap|bootstrap(?:\\.js|\\.min\\.js))",
 			"website": "getbootstrap.com"
 		},
+		"Twitter Emoji (Twemoji)": {
+			"cats": [
+				25
+			],
+			"env": "^twemoji$",
+			"script": "twemoji(\\.min)?\\.js",
+			"website": "https://twitter.github.io/twemoji/"
+		},
+		"Twitter Flight": {
+			"cats": [
+				12
+			],
+			"env": "^flight$",
+			"implies": "jQuery",
+			"website": "https://flightjs.github.io/"
+		},
+		"Twitter typeahead.js": {
+			"cats": [
+				12
+			],
+			"env": "^typeahead$",
+			"implies": "jQuery\\;confidence:50",
+			"script": "(typeahead|bloodhound)(\\.jquery|\\.bundle)?(\\.min)?\\.js",
+			"website": "https://twitter.github.io/typeahead.js/"
+		},
 		"TypePad": {
 			"cats": [
 				11
@ -6918,6 +7034,14 @@
 			"script": "(?:scriptaculous|protoaculous)\\.js",
 			"website": "script.aculo.us"
 		},
+		"shine.js": {
+			"cats": [
+				25
+			],
+			"env": "^Shine$",
+			"script": "shine(\\.min)?\\.js",
+			"website": "http://bigspaceship.github.io/shine.js/"
+		},
 		"spin.js": {
 			"cats": [
 				12,
@ -6995,6 +7119,15 @@
 			"html": "powered by <a href=\"[^>]+viennacms",
 			"website": "www.viennacms.nl"
 		},
+		"vis.js": {
+			"cats": [
+				25
+			],
+			"env": "^vis$",
+			"html": "<link.+?href=\"[^\"]+vis(?:\\.min)?\\.css",
+			"script": "vis(\\.min)?\\.js",
+			"website": "http://visjs.org"
+		},
 		"webEdition": {
 			"cats": [
 				1
@ -7045,139 +7178,6 @@
 				"yepnope.*\\.js"
 			],
 			"website": "yepnopejs.com"
-		},
-		"Open Classifieds": {
-			"cats": [
-				6
-			],
-			"meta": {
-				"author": "open-classifieds\\.com",
-				"copyright": "Open Classifieds ?([0-9.]+)?\\;version:\\1"
-			},
-			"website": "http://open-classifieds.com"
-		},
-		"Open eShop": {
-			"cats": [
-				6
-			],
-			"meta": {
-				"author": "open-eshop\\.com",
-				"copyright": "Open eShop ?([0-9.]+)?\\;version:\\1"
-			},
-			"website": "http://open-eshop.com/"
-		},
-		"Hammer.js": {
-			"cats": [
-				12
-			],
-			"script": "hammer(\\.min)?\\.js",
-			"env": "^Hammer$",
-			"website": "http://hammerjs.github.io"
-		},
-		"Epoch": {
-			"cats": [
-				25
-			],
-			"script": "epoch(\\.min)?\\.js",
-			"html": "<link.+?href=\"[^\"]+epoch(?:\\.min)?\\.css",
-			"implies": "D3",
-			"website": "https://fastly.github.io/epoch/"
-		},
-		"Boba.js": {
-			"cats": [
-				12
-			],
-			"script": "boba(\\.min)?\\.js",
-			"implies": "Google Analytics",
-			"website": "http://boba.space150.com/"
-		},
-		"Lazy.js": {
-			"cats": [
-				12
-			],
-			"script": "lazy(\\.browser)?(\\.min)?\\.js",
-			"website": "http://danieltao.com/lazy.js/"
-		},
-		"SweetAlert": {
-			"cats": [
-				12
-			],
-			"env": "^swal$",
-			"script": "sweet-alert(\\.min)?\\.js",
-			"html": "<link.+?href=\"[^\"]+sweet-alert(?:\\.min)?\\.css",
-			"website": "http://tristanedwards.me/sweetalert"
-		},
-		"vis.js": {
-			"cats": [
-				25
-			],
-			"env": "^vis$",
-			"script": "vis(\\.min)?\\.js",
-			"html": "<link.+?href=\"[^\"]+vis(?:\\.min)?\\.css",
-			"website": "http://visjs.org"
-		},
-		"shine.js": {
-			"cats": [
-				25
-			],
-			"env": "^Shine$",
-			"script": "shine(\\.min)?\\.js",
-			"website": "http://bigspaceship.github.io/shine.js/"
-		},
-		"Paths.js": {
-			"cats": [
-				25
-			],
-			"script": "paths(\\.min)?\\.js",
-			"website": "https://github.com/andreaferretti/paths-js"
-		},
-		"Twitter Flight": {
-			"cats": [
-				12
-			],
-			"env": "^flight$",
-			"implies": "jQuery",
-			"website": "https://flightjs.github.io/"
-		},
-		"Twitter Emoji (Twemoji)": {
-			"cats": [
-				25
-			],
-			"env": "^twemoji$",
-			"script": "twemoji(\\.min)?\\.js",
-			"website": "https://twitter.github.io/twemoji/"
-		},
-		"Twitter typeahead.js": {
-			"cats": [
-				12
-			],
-			"env": "^typeahead$",
-			"script": "(typeahead|bloodhound)(\\.jquery|\\.bundle)?(\\.min)?\\.js",
-			"implies": "jQuery\\;confidence:50",
-			"website": "https://twitter.github.io/typeahead.js/"
-		},
-		"Rickshaw": {
-			"cats": [
-				25
-			],
-			"env": "^Rickshaw$",
-			"script": "rickshaw(\\.min)?\\.js",
-			"implies": "D3",
-			"website": "http://code.shutterstock.com/rickshaw/"
-		},
-		"GitLab CI": {
-			"cats": [
-				44,
-				47
-			],
-			"meta": {
-				"description": "GitLab Continuous Integration"
-			},
-			"implies": [
-				"Ruby",
-				"Ruby on Rails"
-			],
-			"website": "https://about.gitlab.com/gitlab-ci/"
 		}
 	},
 	"categories": {