diff --git a/.gitignore b/.gitignore index a5a316f22..2f0d797d8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,14 @@ npm-debug.log node_modules/* +bin/node_modules/ + +bin/npm/npm-debug.log + drivers/npm/node_modules/ drivers/npm/npm-debug.log !.gitkeep + +Thumbs.db diff --git a/.travis.yml b/.travis.yml index 717f35318..4e740fe58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ install: - sudo apt-get install -y curl zip sudo -y --force-yes - sudo apt-get clean - npm install jsonlint -g + - ln -s bin/package.json package.json && npm install - mkdir mozilla && curl -L https://ftp.mozilla.org/pub/mozilla.org/labs/jetpack/jetpack-sdk-latest.tar.gz | tar xvzC mozilla && ln -s $WAPPALYZER_ROOT/mozilla/addon-sdk-*/bin/cfx bin/cfx - mkdir phantomjs && curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2 | tar xvjC phantomjs && ln -s $WAPPALYZER_ROOT/phantomjs/phantomjs-*-linux-x86_64/bin/phantomjs bin/phantomjs script: wappalyzer build diff --git a/bin/package.json b/bin/package.json new file mode 100644 index 000000000..052b3ccf1 --- /dev/null +++ b/bin/package.json @@ -0,0 +1,9 @@ +{ + "devDependencies": { + "async": "0.9.*", + "file-type": "2.2.*", + "fs-extra": "0.16.*", + "glob": "4.4.*", + "read-chunk": "1.0.*" + } +} diff --git a/bin/wappalyzer-validate b/bin/wappalyzer-validate index cb91bc240..deadd724b 100755 --- a/bin/wappalyzer-validate +++ b/bin/wappalyzer-validate @@ -28,3 +28,7 @@ path="$path/src" echo "Validating apps.json..." jsonlint --quiet -V $WAPPALYZER_ROOT/schema.json $path/apps.json + +echo "Validating icons..." + +node $WAPPALYZER_ROOT/bin/wappalyzer-validate-icons.js $path/apps.json $path/icons \ No newline at end of file diff --git a/bin/wappalyzer-validate-icons.js b/bin/wappalyzer-validate-icons.js new file mode 100644 index 000000000..9e984d96a --- /dev/null +++ b/bin/wappalyzer-validate-icons.js @@ -0,0 +1,68 @@ +var readChunk = require('read-chunk'); +var fileType = require('file-type'); +var path = require('path'); +var fs = require('fs-extra'); +var async = require('async'); +var glob = require('glob'); + +var appsJSON = require(process.argv[2]); +var iconsDir = process.argv[3]; + +var appsIconPaths = []; + +function arrayDiff(a1, a2) { + var o1={}, o2={}, diff=[], i, len, k; + for (i=0, len=a1.length; i 1 && arr.length) { + what = a[--L]; + while ((ax= arr.indexOf(what)) !== -1) { + arr.splice(ax, 1); + } + } + return arr; +} + +async.each(Object.keys(appsJSON.apps), function (app, callback) { + glob(iconsDir + "/" + app + ".+(png|gif|jpg|jpeg|ico|icon|icns|tiff|tif|svg|bmp|psd|pspimage|thm|yuv|ai|drw|eps|ps)", function (err, files) { + if (err) throw err; + if (files.length < 1) { + var err = new Error("There is no icon for '" + app + "'!"); + throw err; + } else if (files.length > 1) { + var err = new Error("There is more than one icon for '" + app + "'!"); + throw err; + } else { + if (files[0].split('.').pop() !== 'png') { + var err = new Error("The icon at " + files[0] + " does not have a '.png' extension!"); + throw err; + } else { + var buffer = fileType(readChunk.sync(files[0], 0, 262)); + if (buffer.mime !== 'image/png' || buffer.ext !== 'png') { + var err = new Error("The icon at " + files[0] + " has a '.png' extension, but it is not actually a PNG file! It is actually a " + buffer.mime + " which usually has an extension of '" + buffer.ext + "'."); + throw err; + } else { + appsIconPaths.push(path.basename(files[0])); + callback(); + } + } + } + }); +}, function(err) { + if (err) throw err; + fs.readdir(iconsDir, function(err, iconsList) { + if (err) throw err; + iconsList = removeA(iconsList, 'Thumbs.db'); // While Thumbs.db is excluded from git, Windows still adds it and it messes up tests + appsIconPaths.push("default.png"); + if (appsIconPaths.length < iconsList.length) { + var err = new Error("There are " + (iconsList.length - appsIconPaths.length) + " more files in the icons directory (" + iconsDir + ") than there are apps! There are " + appsIconPaths.length + " verified icons (one is the default), but there are " + iconsList.length + " total files." + "\n" + "The extra files are: " + arrayDiff(iconsList, appsIconPaths)); + throw err; + } + }); +}); \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 1067f0e6a..5d60ae6c2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,7 +11,6 @@ RUN ln -s /usr/bin/nodejs /usr/bin/node RUN npm install jsonlint -g - # Add user RUN useradd -ms /bin/bash wappalyzer && echo "wappalyzer:wappalyzer" | chpasswd RUN echo 'wappalyzer ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers @@ -28,6 +27,9 @@ RUN cat .profile >> /tmp/profile && mv /tmp/profile .profile RUN echo "cd \$WAPPALYZER_ROOT" >> .bashrc RUN echo "wappalyzer" >> .bashrc +RUN su wappalyzer -c "\ + ln -s bin/package.json package.json && \ + npm install" # Mozilla Add-on SDK RUN su wappalyzer -c "\ diff --git a/src/apps.json b/src/apps.json index fd7df8d88..bbc9e1c09 100755 --- a/src/apps.json +++ b/src/apps.json @@ -5638,7 +5638,7 @@ 23 ], "headers": { - "Via": "Varnish", + "Via": ".*Varnish", "X-Varnish": "", "X-Varnish-Action": "", "X-Varnish-Age": "", diff --git a/src/icons/Alloy.png b/src/icons/Alloy.png deleted file mode 100755 index 56a9952fd..000000000 Binary files a/src/icons/Alloy.png and /dev/null differ diff --git a/src/icons/BlueKai.png b/src/icons/BlueKai.png deleted file mode 100644 index b2861d007..000000000 Binary files a/src/icons/BlueKai.png and /dev/null differ diff --git a/src/icons/ckan.png b/src/icons/Ckan.png similarity index 100% rename from src/icons/ckan.png rename to src/icons/Ckan.png diff --git a/src/icons/DAV.png b/src/icons/DAV.png deleted file mode 100644 index 2e51915b0..000000000 Binary files a/src/icons/DAV.png and /dev/null differ diff --git a/src/icons/Dart.png b/src/icons/Dart.png deleted file mode 100644 index 35c6ccd1a..000000000 Binary files a/src/icons/Dart.png and /dev/null differ diff --git a/src/icons/Ecodoo.png b/src/icons/Ecodoo.png deleted file mode 100755 index d08a72652..000000000 Binary files a/src/icons/Ecodoo.png and /dev/null differ diff --git a/src/icons/flask.png b/src/icons/Flask.png similarity index 100% rename from src/icons/flask.png rename to src/icons/Flask.png diff --git a/src/icons/Google Friend Connect.png b/src/icons/Google Friend Connect.png deleted file mode 100644 index 78b69568a..000000000 Binary files a/src/icons/Google Friend Connect.png and /dev/null differ diff --git a/src/icons/HERE.png b/src/icons/HERE.png new file mode 100644 index 000000000..cc541f523 Binary files /dev/null and b/src/icons/HERE.png differ diff --git a/src/icons/Koken.png b/src/icons/Koken.png new file mode 100644 index 000000000..69e214c13 Binary files /dev/null and b/src/icons/Koken.png differ diff --git a/src/icons/Lucene.png b/src/icons/Lucene.png new file mode 100644 index 000000000..0bce5f5d8 Binary files /dev/null and b/src/icons/Lucene.png differ diff --git a/src/icons/Mono.png b/src/icons/Mono.png index f2340acc8..dd9578a86 100644 Binary files a/src/icons/Mono.png and b/src/icons/Mono.png differ diff --git a/src/icons/Nokia Maps.png b/src/icons/Nokia Maps.png deleted file mode 100644 index 5045e96bd..000000000 Binary files a/src/icons/Nokia Maps.png and /dev/null differ diff --git a/src/icons/PostgreSQL.png b/src/icons/PostgreSQL.png new file mode 100644 index 000000000..3ed911796 Binary files /dev/null and b/src/icons/PostgreSQL.png differ diff --git a/src/icons/ReallyCMS.png b/src/icons/ReallyCMS.png deleted file mode 100644 index a0b6a3045..000000000 Binary files a/src/icons/ReallyCMS.png and /dev/null differ diff --git a/src/icons/SiteDNK.png b/src/icons/SiteDNK.png deleted file mode 100644 index 2e51915b0..000000000 Binary files a/src/icons/SiteDNK.png and /dev/null differ diff --git a/src/icons/Skimlinks.png b/src/icons/Skimlinks.png deleted file mode 100644 index e5dea4cfb..000000000 Binary files a/src/icons/Skimlinks.png and /dev/null differ diff --git a/src/icons/Solr.png b/src/icons/Solr.png new file mode 100644 index 000000000..0df60b89b Binary files /dev/null and b/src/icons/Solr.png differ diff --git a/src/icons/TYPOlight.png b/src/icons/TYPOlight.png deleted file mode 100644 index fd786f02b..000000000 Binary files a/src/icons/TYPOlight.png and /dev/null differ diff --git a/src/icons/Timeline.png b/src/icons/Timeline.png deleted file mode 100644 index 2e51915b0..000000000 Binary files a/src/icons/Timeline.png and /dev/null differ diff --git a/src/icons/Webs.png b/src/icons/Webs.png new file mode 100644 index 000000000..c022e716f Binary files /dev/null and b/src/icons/Webs.png differ diff --git a/src/icons/WordPress.com.png b/src/icons/WordPress.com.png deleted file mode 100644 index f23c8c210..000000000 Binary files a/src/icons/WordPress.com.png and /dev/null differ diff --git a/src/icons/XiTi.png b/src/icons/XiTi.png deleted file mode 100755 index 6e20ea90c..000000000 Binary files a/src/icons/XiTi.png and /dev/null differ diff --git a/src/icons/io4-CMS.png b/src/icons/io4 CMS.png similarity index 100% rename from src/icons/io4-CMS.png rename to src/icons/io4 CMS.png