diff --git a/.gitignore b/.gitignore index af6546711..6ee2ee009 100644 --- a/.gitignore +++ b/.gitignore @@ -1,19 +1,9 @@ .vagrant build/* - -drivers/npm/node_modules -drivers/npm/npm-debug.log - src/icons/converted/* -package.json -node_modules/ +node_modules npm-debug.log -Thumbs.db -ehthumbs.db -Desktop.ini -$RECYCLE.BIN/ - !.gitkeep diff --git a/bin/wappalyzer-build b/bin/wappalyzer-build index 8cd9e48c1..45de3da42 100755 --- a/bin/wappalyzer-build +++ b/bin/wappalyzer-build @@ -50,10 +50,6 @@ node $WAPPALYZER_NODE_PATH/node_modules/imagemin-cli/cli.js $WAPPALYZER_ROOT/src wappalyzer links -# Npm Module -mkdir -p $WAPPALYZER_ROOT/build/wappalyzer_npm/ -cp -R $WAPPALYZER_ROOT/src/drivers/npm/* $WAPPALYZER_ROOT/build/wappalyzer_npm/ - # Mozilla Firefox echo "Building Firefox driver..." diff --git a/src/drivers/html/.gitignore b/src/drivers/html/.gitignore deleted file mode 100644 index 01d2786a1..000000000 --- a/src/drivers/html/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -apps.json -images/icons/*.png -images/icons/*.svg -js/wappalyzer.js diff --git a/src/drivers/html/images/icon.png b/src/drivers/html/images/icon.png deleted file mode 100644 index 0e1e1fb0c..000000000 Binary files a/src/drivers/html/images/icon.png and /dev/null differ diff --git a/src/drivers/html/images/icons/.gitkeep b/src/drivers/html/images/icons/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/drivers/html/index.html b/src/drivers/html/index.html deleted file mode 100644 index 32ec5f46a..000000000 --- a/src/drivers/html/index.html +++ /dev/null @@ -1,30 +0,0 @@ - - - - - Wappalyzer - - - - - - - - - -
- - diff --git a/src/drivers/html/js/driver.js b/src/drivers/html/js/driver.js deleted file mode 100644 index 0e14f59af..000000000 --- a/src/drivers/html/js/driver.js +++ /dev/null @@ -1,67 +0,0 @@ -(function() { - if ( wappalyzer == null ) { return }; - - var w = wappalyzer; - - w.driver = { - /** - * Log messages to console - */ - log: function(args) { - if ( console != null ) { console[args.type](args.message) }; - }, - - /** - * Initialize - */ - init: function() { - // Load apps.json - var xhr = new XMLHttpRequest(); - - xhr.open('GET', 'apps.json', true); - - xhr.overrideMimeType('application/json'); - - xhr.onload = function() { - var json = JSON.parse(xhr.responseText); - - w.categories = json.categories; - w.apps = json.apps; - - window.document.addEventListener('DOMContentLoaded', function() { - w.analyze('google.com', 'http://google.com', { - html: ' - - - - - - - - - -
-
-
-
-
-

Preso Engines

- "Presentation Engines" -
-
- -
-
By | CodeLanka
-
-
-
-

-
-
-
-
-
- - {{presentation.name}} - - - - {{presentation.website}} - - - - {{presentation.description}} - -
-
- -

{{presentation.stargazers_count}}

-
-
- -

{{presentation.watchers_count}}

-
-
- -

{{presentation.open_issues_count}}

-
- -
-
-

Github - - {{presentation.github}} -

-

Demo - - {{presentation.demo}} -

- -

License - {{presentation.license}}

- -

Language - {{presentation.language}}

-
-
-
-
-
-
-
-
-
- - - - - - - \ No newline at end of file diff --git a/src/drivers/npm/test/test.js b/src/drivers/npm/test/test.js deleted file mode 100644 index 8e994d1cc..000000000 --- a/src/drivers/npm/test/test.js +++ /dev/null @@ -1,65 +0,0 @@ -var assert = require("assert") -var fs = require('fs') - -describe('wappalyzer', function(){ - describe('detectFromUrl', function(){ - it('should have the expected apps detected', function(done){ - - var wappalyzer = require("../index"); - - var expect = ['AngularJS','Font Awesome','Google Font API','Twitter Bootstrap','jQuery']; - - var options={ - url : "http://codelanka.github.io/Presentation-Engines", - hostname:"codelanka.github.io", - debug:false - } - - wappalyzer.detectFromUrl(options,function (err,apps) { - assert.equal(expect[0], apps[0]); - assert.equal(expect[1], apps[1]); - assert.equal(expect[2], apps[2]); - assert.equal(expect[3], apps[3]); - done(); - }) - - }) - }) - describe('detectFromHTML', function(){ - it('should have the expected apps detected when passed raw info', function(done){ - - var wappalyzer = require("../index"); - - var expect = ['AngularJS','Font Awesome','Google Font API','Twitter Bootstrap','jQuery']; - - var options={ - url : "http://codelanka.github.io/Presentation-Engines", - hostname:"codelanka.github.io", - debug:false - } - - var data = { - - url: options.url, - headers: require('./sample.headers.json'), - html: fs.readFileSync('./test/sample.html').toString(), - headers: { - - headers: {} - - } - - }; - - wappalyzer.detectFromHTML(options, data, function (err,apps) { - - assert.equal(expect[0], apps[0]); - assert.equal(expect[1], apps[1]); - assert.equal(expect[2], apps[2]); - assert.equal(expect[3], apps[3]); - done(); - }) - - }) - }) -}) diff --git a/src/drivers/phantomjs/Dockerfile b/src/drivers/phantomjs/Dockerfile index 52a75669e..95e46592d 100644 --- a/src/drivers/phantomjs/Dockerfile +++ b/src/drivers/phantomjs/Dockerfile @@ -1,30 +1,31 @@ -FROM phusion/baseimage +FROM ubuntu:latest MAINTAINER Elbert Alias ENV DEBIAN_FRONTEND noninteractive +RUN sed -i 's/archive\.ubuntu\.com/au.archive.ubuntu.com/g' /etc/apt/sources.list + RUN \ apt-get update && apt-get install -y \ - bzip2 \ libfreetype6 \ libfontconfig \ + nodejs \ + npm \ && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -WORKDIR /usr/local +RUN ln -s $(which nodejs) /usr/bin/node -# PhantomJS -RUN \ - mkdir phantomjs && \ - curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2 | tar xvjC phantomjs --strip 1 +RUN mkdir /usr/local/wappalyzer -# Wappalyzer -RUN \ - mkdir wappalyzer && \ - curl -sSL https://github.com/AliasIO/Wappalyzer/archive/master.tar.gz | tar xzC wappalyzer --strip 1 +WORKDIR /usr/local/wappalyzer -RUN wappalyzer/bin/wappalyzer-links wappalyzer +ADD apps.json . +ADD driver.js . +ADD index.js . +ADD package.json . +ADD wappalyzer.js . -WORKDIR wappalyzer/src/drivers/phantomjs +RUN npm i -ENTRYPOINT ["/usr/local/phantomjs/bin/phantomjs", "--load-images=false", "--ignore-ssl-errors=yes", "--ssl-protocol=any", "driver.js"] +ENTRYPOINT ["node", "index.js"] diff --git a/src/drivers/phantomjs/README.md b/src/drivers/phantomjs/README.md new file mode 100644 index 000000000..7e1dd105c --- /dev/null +++ b/src/drivers/phantomjs/README.md @@ -0,0 +1,57 @@ +# Wappalyzer + +[Wappalyzer](https://wappalyzer.com/) is a +[cross-platform](https://github.com/AliasIO/Wappalyzer/wiki/Drivers) utility that uncovers the +technologies used on websites. It detects +[content management systems](https://wappalyzer.com/categories/cms), +[eCommerce platforms](https://wappalyzer.com/categories/ecommerce), +[web servers](https://wappalyzer.com/categories/web-servers), +[JavaScript frameworks](https://wappalyzer.com/categories/javascript-frameworks), +[analytics tools](https://wappalyzer.com/categories/analytics) and +[many more](https://wappalyzer.com/applications). + + +## Installation + +```shell +$ npm i wappalyzer +``` + + +## Run from the command line + +```shell +$ node index.js https://wappalyzer.com --quiet +``` + + +## Run from a script + +```javascript +const wappalyzer = require('@wappalyzer/wappalyzer'); + +wappalyzer.run(['https://wappalyzer.com', '--quiet'], function(stdout, stderr) { + if ( stdout ) { + process.stdout.write(stdout); + } + + if ( stderr ) { + process.stderr.write(stderr); + } +}); +``` + + +## Arguments + +**-v, --verbose** + +Display debug output. + +**-q, --quiet** + +Suppress errors. + +**--resource-timeout=ms** + +Abort the connection after 'ms' milliseconds. diff --git a/src/drivers/phantomjs/driver.js b/src/drivers/phantomjs/driver.js index db7f2e52f..a11c07da8 100644 --- a/src/drivers/phantomjs/driver.js +++ b/src/drivers/phantomjs/driver.js @@ -163,6 +163,10 @@ } }; + page.onResourceError = function(resourceError) { + wappalyzer.log(resourceError.errorString, 'error'); + }; + page.open(url, function(status) { var html, environmentVars; diff --git a/src/drivers/phantomjs/index.js b/src/drivers/phantomjs/index.js new file mode 100644 index 000000000..3b090b191 --- /dev/null +++ b/src/drivers/phantomjs/index.js @@ -0,0 +1,32 @@ +'use strict'; + +const + path = require('path'), + spawn = require('child_process').spawn, + phantomjs = require('phantomjs-prebuilt'); + +exports.run = function(args, callback) { + args.push.apply(['--web-security=no', '--load-images=false', '--ignore-ssl-errors=yes', '--ssl-protocol=any']); + + var driver = phantomjs.exec('driver.js', args); + + driver.stdout.on('data', (data) => { + callback(`${data}`, null); + }); + + driver.stderr.on('data', (data) => { + callback(null, `${data}`); + }); +} + +if ( !module.parent ) { + exports.run(process.argv.slice(2), function(stdout, stderr) { + if ( stdout ) { + process.stdout.write(stdout); + } + + if ( stderr ) { + process.stderr.write(stderr); + } + }); +} diff --git a/src/drivers/phantomjs/package.json b/src/drivers/phantomjs/package.json new file mode 100644 index 000000000..5544576af --- /dev/null +++ b/src/drivers/phantomjs/package.json @@ -0,0 +1,21 @@ +{ + "name": "wappalyzer", + "description": "Uncovers the technologies used on websites", + "homepage": "https://github.com/AliasIO/Wappalyzer", + "version": "3.0.3", + "author": "Elbert Alias", + "license": "GPL-3.0", + "repository": { + "type": "git", + "url": "https://github.com/AliasIO/Wappalyzer" + }, + "main": "index.js", + "files": [ + "apps.json", + "driver.js", + "wappalyzer.js" + ], + "dependencies": { + "phantomjs-prebuilt": "*" + } +} diff --git a/src/drivers/php/.gitignore b/src/drivers/php/.gitignore deleted file mode 100644 index 2d6f40c2f..000000000 --- a/src/drivers/php/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -js/wappalyzer.js -apps.json diff --git a/src/drivers/php/Wappalyzer.php b/src/drivers/php/Wappalyzer.php deleted file mode 100644 index 2fcdd93d3..000000000 --- a/src/drivers/php/Wappalyzer.php +++ /dev/null @@ -1,193 +0,0 @@ -v8 = new V8Js(); - - $this->url = $url; - - $json = json_decode(file_get_contents('apps.json')); - - $this->apps = $json->apps; - $this->categories = $json->categories; - } - - /** - * Analyze a website - * @param string $url - */ - public function analyze() - { - try { - $this->load(array('wappalyzer.js', 'driver.js')); - - $result = $this->curl($this->url); - - //$env = $this->executeScripts($result); - - $json = json_encode(array( - 'host' => $result->host, - 'url' => $result->url, - 'html' => $result->html, - 'headers' => $result->headers, - //'env' => $env - )); - - $result = $this->v8->executeString(' - w.apps = ' . json_encode($this->apps) . '; - w.categories = ' . json_encode($this->categories) . '; - w.driver.debug = ' . ( $this->debug ? 'true' : 'false' ) . '; - w.driver.data = ' . $json . '; - - w.driver.init(); - '); - - return json_decode($result); - } catch ( V8JsException $e ) { - throw new WappalyzerException('JavaScript error: ' . $e->getMessage()); - } - } - - /** - * Load and execute one or more JavaScript files - * @param mixed $files - */ - protected function load($files) - { - if ( !is_array($files) ) { - $files = array($files); - } - - foreach ( $files as $file ) { - $this->v8->executeString(file_get_contents('js/' . $file), $file); - } - } - - /** - * Perform a cURL request - * @param string $url - */ - protected function curl($url) - { - if ( $this->debug ) { - echo 'cURL request: ' . $url . "\n"; - } - - $ch = curl_init($url); - - curl_setopt_array($ch, array( - CURLOPT_SSL_VERIFYPEER => false, - CURLOPT_HEADER => true, - CURLOPT_RETURNTRANSFER => true, - CURLOPT_FOLLOWLOCATION => $this->curlFollowLocation, - CURLOPT_MAXREDIRS => $this->curlMaxRedirects, - CURLOPT_TIMEOUT => $this->curlTimeout, - CURLOPT_USERAGENT => $this->curlUserAgent - )); - - $response = curl_exec($ch); - - if ( curl_errno($ch) !== 0 ) { - throw new WappalyzerException('cURL error: ' . curl_error($ch)); - } - - $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - - if ( $httpCode != 200 ) { - throw new WappalyzerException('cURL request returned HTTP code ' . $httpCode); - } - - $result = new stdClass(); - - $result->url = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL); - - $result->host = parse_url($result->url, PHP_URL_HOST); - - $headerSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE); - - $result->html = substr($response, $headerSize); - - $result->html = mb_check_encoding($result->html, 'UTF-8') ? $result->html : utf8_encode($result->html); - - $headers = trim(substr($response, 0, $headerSize)); - $headers = preg_split('/^\s*$/m', $headers); - $headers = end($headers); - $lines = array_slice(explode("\n", $headers), 1); - - foreach ( $lines as $line ) { - if ( strpos(trim($line), ': ') !== false ) { - list($key, $value) = explode(': ', trim($line, "\r")); - - $result->headers[strtolower($key)] = $value; - } - } - - return $result; - } - - /** - * - */ - protected function executeScripts($page) - { - preg_match_all('/]+src=("|\')(.+?)\1/i', $page->html, $matches); - - if ( $urls = $matches[2] ) { - foreach ( $urls as $url ) { - if ( !preg_match('/^https?:\/\//', $url) ) { - $url = $page->url . '/' . $url; - } - - try { - $result = $this->curl($url); - } catch ( WappalyzerException $e ) { - if ( $this->debug ) echo $e->getMessage() . "\n"; - - continue; - } - - $v8 = new V8Js(); - - try { - $v8->executeString(' - var - document = {}, - window = { document: document } - ; - '); - - $v8->executeString($result->html, $url); - - $result = $v8->executeString('Object.keys(window);'); - - var_dump($result); - } catch ( V8JsException $e ) { - if ( $this->debug ) echo "\n", print_r($e->getJsTrace()), "\n\n"; - - continue; - } - } - } - } -} diff --git a/src/drivers/php/WappalyzerException.php b/src/drivers/php/WappalyzerException.php deleted file mode 100644 index 89b6a71bc..000000000 --- a/src/drivers/php/WappalyzerException.php +++ /dev/null @@ -1,4 +0,0 @@ -\n"; - - exit(0); - } - - $wappalyzer = new Wappalyzer($url); - - $detectedApps = $wappalyzer->analyze(); - - if ( $detectedApps ) { - foreach ( $detectedApps as $detectedApp => $data ) { - echo $detectedApp . ', ' . $data->version . ', ', $data->confidence . '%, ', implode(', ', $data->categories) . "\n"; - } - } else { - echo "No applications detected\n"; - } - - exit(0); -} catch ( Exception $e ) { - echo $e->getMessage() . "\n"; - - exit(1); -} diff --git a/src/drivers/php/js/driver.js b/src/drivers/php/js/driver.js deleted file mode 100644 index f6e6994bd..000000000 --- a/src/drivers/php/js/driver.js +++ /dev/null @@ -1,46 +0,0 @@ -var w = wappalyzer; - -w.driver = { - debug: false, - data: {}, - timeout: 5000, - - /** - * Log messages to console - */ - log: function(args) { - if ( w.driver.debug ) { print(args.type + ': ' + args.message + "\n"); } - }, - - /** - * Initialize - */ - init: function() { - var app, apps = {}; - - w.analyze(w.driver.data.host, w.driver.data.url, { - html: w.driver.data.html, - headers: w.driver.data.headers - }); - - for ( app in w.detected[w.driver.data.url] ) { - apps[app] = { - categories: [], - confidence: w.detected[w.driver.data.url][app].confidenceTotal, - version: w.detected[w.driver.data.url][app].version - }; - - w.apps[app].cats.forEach(function(cat) { - apps[app].categories.push(w.categories[cat]); - }); - }; - - return JSON.stringify(apps); - }, - - /** - * Dummy - */ - displayApps: function() { - } -}; diff --git a/src/drivers/python/.gitignore b/src/drivers/python/.gitignore deleted file mode 100644 index 2d6f40c2f..000000000 --- a/src/drivers/python/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -js/wappalyzer.js -apps.json diff --git a/src/drivers/python/js/.gitkeep b/src/drivers/python/js/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/drivers/python/wappalyzer.py b/src/drivers/python/wappalyzer.py deleted file mode 100755 index a70f75f7f..000000000 --- a/src/drivers/python/wappalyzer.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import sys -import PyV8 -import requests - -from requests.packages.urllib3.exceptions import InsecureRequestWarning -requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - -from urlparse import urlparse - -try: - import json -except ImportError: - import simplejson as json - - -class Wappalyzer(object): - - def __init__(self, url): - self.file_dir = os.path.dirname(__file__) - - f = open(os.path.join(self.file_dir, 'apps.json')) - data = json.loads(f.read()) - f.close() - - self.categories = data['categories'] - self.apps = data['apps'] - self.url = url - - def analyze(self): - ctxt = PyV8.JSContext() - ctxt.enter() - - f1 = open(os.path.join(self.file_dir, 'js/wappalyzer.js')) - f2 = open(os.path.join(self.file_dir, '../php/js/driver.js')) - ctxt.eval(f1.read()) - ctxt.eval(f2.read()) - f1.close() - f2.close() - - host = urlparse(self.url).hostname - response = requests.get(self.url, verify=False) - html = response.text - headers = dict(response.headers) - - data = {'host': host, 'url': self.url, 'html': html, 'headers': headers} - apps = json.dumps(self.apps) - categories = json.dumps(self.categories) - return ctxt.eval("w.apps = %s; w.categories = %s; w.driver.data = %s; w.driver.init();" % (apps, categories, json.dumps(data))) - -if __name__ == '__main__': - try: - w = Wappalyzer(sys.argv[1]) - print w.analyze() - except IndexError: - print ('Usage: python %s ' % sys.argv[0]) diff --git a/src/drivers/ruby/.gitignore b/src/drivers/ruby/.gitignore deleted file mode 100644 index 2d6f40c2f..000000000 --- a/src/drivers/ruby/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -js/wappalyzer.js -apps.json diff --git a/src/drivers/ruby/js/driver.js b/src/drivers/ruby/js/driver.js deleted file mode 100644 index f6e6994bd..000000000 --- a/src/drivers/ruby/js/driver.js +++ /dev/null @@ -1,46 +0,0 @@ -var w = wappalyzer; - -w.driver = { - debug: false, - data: {}, - timeout: 5000, - - /** - * Log messages to console - */ - log: function(args) { - if ( w.driver.debug ) { print(args.type + ': ' + args.message + "\n"); } - }, - - /** - * Initialize - */ - init: function() { - var app, apps = {}; - - w.analyze(w.driver.data.host, w.driver.data.url, { - html: w.driver.data.html, - headers: w.driver.data.headers - }); - - for ( app in w.detected[w.driver.data.url] ) { - apps[app] = { - categories: [], - confidence: w.detected[w.driver.data.url][app].confidenceTotal, - version: w.detected[w.driver.data.url][app].version - }; - - w.apps[app].cats.forEach(function(cat) { - apps[app].categories.push(w.categories[cat]); - }); - }; - - return JSON.stringify(apps); - }, - - /** - * Dummy - */ - displayApps: function() { - } -}; diff --git a/src/drivers/ruby/wappalyzer.rb b/src/drivers/ruby/wappalyzer.rb deleted file mode 100755 index 130c21006..000000000 --- a/src/drivers/ruby/wappalyzer.rb +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env ruby - -require 'net/http' -require 'v8' -require 'json' -require 'openssl' - -OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE -Encoding.default_external = Encoding::UTF_8 - -class Wappalyzer - def initialize - @realdir = File.dirname(File.realpath(__FILE__)) - file = File.join(@realdir, 'apps.json') - @json = JSON.parse(IO.read(file)) - @categories, @apps = @json['categories'], @json['apps'] - end - - def analyze(url) - uri, body, headers = URI(url), nil, {} - Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https', :open_timeout => 5) do |http| - resp = http.get(uri.request_uri) - resp.each_header{|k,v| headers[k.downcase] = v} - body = resp.body.encode('UTF-8', :invalid => :replace, :undef => :replace) - end - - cxt = V8::Context.new - cxt.load File.join(@realdir, 'js', 'wappalyzer.js') - cxt.load File.join(@realdir, 'js', 'driver.js') - data = {'host' => uri.hostname, 'url' => url, 'html' => body, 'headers' => headers} - output = cxt.eval("w.apps = #{@apps.to_json}; w.categories = #{@categories.to_json}; w.driver.data = #{data.to_json}; w.driver.init();") - JSON.load(output) - end -end - -if $0 == __FILE__ - url = ARGV[0] - if url - puts JSON.pretty_generate(Wappalyzer.new.analyze(ARGV[0])) - else - puts "Usage: #{__FILE__} http://example.com" - end -end