Merge pull request #809 from DaAwesomeP/master

Add test for icons and fix icons

.gitignore

@@ -7,8 +7,14 @@ npm-debug.log
 
 node_modules/*
 
+bin/node_modules/
+
+bin/npm/npm-debug.log
+
 drivers/npm/node_modules/
 
 drivers/npm/npm-debug.log
 
 !.gitkeep
+
+Thumbs.db

.travis.yml

@@ -9,6 +9,7 @@ install:
   - sudo apt-get install -y curl zip sudo -y --force-yes
   - sudo apt-get clean
   - npm install jsonlint -g
+  - ln -s bin/package.json package.json && npm install
   - mkdir mozilla && curl -L https://ftp.mozilla.org/pub/mozilla.org/labs/jetpack/jetpack-sdk-latest.tar.gz | tar xvzC mozilla && ln -s $WAPPALYZER_ROOT/mozilla/addon-sdk-*/bin/cfx bin/cfx
   - mkdir phantomjs && curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2 | tar xvjC phantomjs && ln -s $WAPPALYZER_ROOT/phantomjs/phantomjs-*-linux-x86_64/bin/phantomjs bin/phantomjs
 script: wappalyzer build

bin/package.json

@@ -0,0 +1,9 @@
+{
+  "devDependencies": {
+    "async": "0.9.*",
+    "file-type": "2.2.*",
+    "fs-extra": "0.16.*",
+    "glob": "4.4.*",
+    "read-chunk": "1.0.*"
+  }
+}

bin/wappalyzer-validate

@@ -28,3 +28,7 @@ path="$path/src"
 echo "Validating apps.json..."
 
 jsonlint --quiet -V $WAPPALYZER_ROOT/schema.json $path/apps.json
+
+echo "Validating icons..."
+
+node $WAPPALYZER_ROOT/bin/wappalyzer-validate-icons.js $path/apps.json $path/icons

bin/wappalyzer-validate-icons.js

@@ -0,0 +1,68 @@
+var readChunk = require('read-chunk');
+var fileType = require('file-type');
+var path = require('path');
+var fs = require('fs-extra');
+var async = require('async');
+var glob = require('glob');
+
+var appsJSON = require(process.argv[2]);
+var iconsDir = process.argv[3];
+
+var appsIconPaths = [];
+
+function arrayDiff(a1, a2) {
+  var o1={}, o2={}, diff=[], i, len, k;
+  for (i=0, len=a1.length; i<len; i++) { o1[a1[i]] = true; }
+  for (i=0, len=a2.length; i<len; i++) { o2[a2[i]] = true; }
+  for (k in o1) { if (!(k in o2)) { diff.push(k); } }
+  for (k in o2) { if (!(k in o1)) { diff.push(k); } }
+  return diff;
+}
+function removeA(arr) {
+    var what, a = arguments, L = a.length, ax;
+    while (L > 1 && arr.length) {
+        what = a[--L];
+        while ((ax= arr.indexOf(what)) !== -1) {
+            arr.splice(ax, 1);
+        }
+    }
+    return arr;
+}
+
+async.each(Object.keys(appsJSON.apps), function (app, callback) {
+	glob(iconsDir + "/" + app + ".+(png|gif|jpg|jpeg|ico|icon|icns|tiff|tif|svg|bmp|psd|pspimage|thm|yuv|ai|drw|eps|ps)", function (err, files) {
+		if (err) throw err;
+		if (files.length < 1) {
+			var err = new Error("There is no icon for '" + app + "'!");
+			throw err;
+		} else if (files.length > 1) {
+			var err = new Error("There is more than one icon for '" + app + "'!");
+			throw err;
+		} else {
+			if (files[0].split('.').pop() !== 'png') {
+				var err = new Error("The icon at " + files[0] + " does not have a '.png' extension!");
+				throw err;
+			} else {
+				var buffer = fileType(readChunk.sync(files[0], 0, 262));
+				if (buffer.mime !== 'image/png' || buffer.ext !== 'png') {
+					var err = new Error("The icon at " + files[0] + " has a '.png' extension, but it is not actually a PNG file! It is actually a " + buffer.mime + " which usually has an extension of '" + buffer.ext + "'.");
+					throw err;
+				} else {
+					appsIconPaths.push(path.basename(files[0]));
+					callback();
+				}
+			}
+		}
+	});
+}, function(err) {
+	if (err) throw err;
+	fs.readdir(iconsDir, function(err, iconsList) {
+		if (err) throw err;
+		iconsList = removeA(iconsList, 'Thumbs.db');				// While Thumbs.db is excluded from git, Windows still adds it and it messes up tests
+		appsIconPaths.push("default.png");
+		if (appsIconPaths.length < iconsList.length) {
+			var err = new Error("There are " + (iconsList.length - appsIconPaths.length) + " more files in the icons directory (" + iconsDir + ") than there are apps! There are " + appsIconPaths.length + " verified icons (one is the default), but there are " + iconsList.length + " total files." + "\n" + "The extra files are: " + arrayDiff(iconsList, appsIconPaths));
+			throw err;
+		}
+	});
+});

docker/Dockerfile

@@ -11,7 +11,6 @@ RUN ln -s /usr/bin/nodejs /usr/bin/node
 
 RUN npm install jsonlint -g
 
-
 # Add user
 RUN useradd -ms /bin/bash wappalyzer && echo "wappalyzer:wappalyzer" | chpasswd
 RUN echo 'wappalyzer ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
@@ -28,6 +27,9 @@ RUN cat .profile >> /tmp/profile && mv /tmp/profile .profile
 RUN echo "cd \$WAPPALYZER_ROOT" >> .bashrc
 RUN echo "wappalyzer" >> .bashrc
 
+RUN su wappalyzer -c "\
+	ln -s bin/package.json package.json && \
+	npm install"
 
 # Mozilla Add-on SDK
 RUN su wappalyzer -c "\

src/apps.json

@@ -5638,7 +5638,7 @@
 				23
 			],
 			"headers": {
-				"Via": "Varnish",
+				"Via": ".*Varnish",
 				"X-Varnish": "",
 				"X-Varnish-Action": "",
 				"X-Varnish-Age": "",