diff --git a/.gitignore b/.gitignore index a8d724d9a..4c244fbaa 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ drivers/npm/node_modules drivers/npm/npm-debug.log node_modules/ +npm-debug.log Thumbs.db ehthumbs.db diff --git a/.travis.yml b/.travis.yml index 62527435c..ddb439bb3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,15 @@ language: node_js node_js: - - "0.12" -sudo: false + - "4.1.0" +sudo: required before_install: - export WAPPALYZER_ROOT=$TRAVIS_BUILD_DIR - export WAPPALYZER_NODE_PATH=$TRAVIS_BUILD_DIR - export PATH=$PATH:$TRAVIS_BUILD_DIR/bin install: + - sudo apt-get update -y + - sudo apt-get install -y curl zip sudo -y --force-yes + - sudo apt-get clean - npm install jsonlint jpm imagemin-cli -g - ln -s docker/node/package.json package.json && npm install - mkdir phantomjs && curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2 | tar xvjC phantomjs && ln -s $WAPPALYZER_ROOT/phantomjs/phantomjs-*-linux-x86_64/bin/phantomjs bin/phantomjs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..bffe6a2e1 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,3 @@ +# Contributing + +You can find our contribution guide [in our wiki](https://github.com/AliasIO/Wappalyzer/wiki/Contributing). diff --git a/bin/wappalyzer-validate b/bin/wappalyzer-validate index a6e8c691e..a15d5d3c3 100755 --- a/bin/wappalyzer-validate +++ b/bin/wappalyzer-validate @@ -29,9 +29,9 @@ echo "Validating apps.json..." jsonlint --quiet -V $WAPPALYZER_ROOT/schema.json $path/apps.json -echo "Validating regular expressions..." - -wappalyzer validate-regex +# echo "Validating regular expressions..." +# +# wappalyzer validate-regex echo "Validating icons..." diff --git a/bin/wappalyzer-validate-regex b/bin/wappalyzer-validate-regex index e55e70648..445479aaa 100755 --- a/bin/wappalyzer-validate-regex +++ b/bin/wappalyzer-validate-regex @@ -25,12 +25,43 @@ for ( app in json.apps ) { patterns.forEach(function(pattern) { var attrs = pattern.split('\\;'), - regex = '/' + attrs.shift().replace('/', '\/') + '/'; + regex = '/' + attrs.shift().replace('/', '\/') + '/', + version = attrs.find( function (attr){ + return attr.indexOf('version:')===0; + } ), + amountOfCaptureGroups, + captureGroupsNeeded, + amountOfCaptureGroupsNeeded, + highestBackReference; //Check if the pattern is a valid RegExp //Note: unlike when used in Wappalyzer, the modifier i isn't added here new RegExp(regex); + //Actual amount of capture groups + amountOfCaptureGroups = ''.match(new RegExp('(?:'+pattern+')?')).length - 1; + + //Capture groups of which the result is used + captureGroupsNeeded = version ? (version.match(/\\\d/g) || []).filter( function(value, index, self){ + return self.indexOf(value) === index; + }).map( function(value){//Because Math.max needs the integers only + return parseInt(value.charAt(1), 10);//Will only work if backreferences cannot be any longer than a single digit + }) : []; + + //Amount of capture groups needed + amountOfCaptureGroupsNeeded = captureGroupsNeeded.length; + + //Highest back reference number used + highestBackReference = Math.max.apply(null, captureGroupsNeeded); + + //Report error + if(amountOfCaptureGroups > amountOfCaptureGroupsNeeded) { + throw new Error('The pattern uses more capture groups than needed. Use non-capturing roups where appropriate.\n' + app + ': ' + type + ': ' + pattern); + }else if(amountOfCaptureGroups < amountOfCaptureGroupsNeeded){ + throw new Error('The version string references more capture groups than there are in the pattern! Remove any incorrect back references from the version string and/or add the missing capture groups to the pattern.\n' + app + ': ' + type + ': ' + pattern); + } else if(highestBackReference > amountOfCaptureGroups){ + throw new Error('The version string references one or more capture groups whose index is higher than the amount of capture groups in the pattern. Please use the correct index instead of \\\\'+highestBackReference+'.\n' + app + ': ' + type + ': ' + pattern); + } if ( /^\/(?:\^\$|\.\+|\.\*)\/$/.test(regex) ) { throw new Error('Pattern should be replaced with empty string.\n' + app + ': ' + type + ': ' + pattern); diff --git a/docker/Dockerfile b/docker/Dockerfile index 142804515..627c8a774 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ ENV WAPPALYZER_NODE_PATH /home/wappalyzer/node # Install packages RUN apt-get update && apt-get install -y curl zip libfreetype6 libfontconfig -RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash - +RUN curl -sL https://deb.nodesource.com/setup_4.x | bash - RUN apt-get install -y nodejs RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/src/apps.json b/src/apps.json index 4b29569db..729f3a1b8 100755 --- a/src/apps.json +++ b/src/apps.json @@ -182,18 +182,6 @@ "script": "addthis\\.com/js/", "website": "www.addthis.com" }, - "Adobe CQ5": { - "cats": [ - 1 - ], - "html": [ - "