From 9a07bb7e8728092e99615a2ae9b50b0d194dfe19 Mon Sep 17 00:00:00 2001 From: Elbert Alias Date: Wed, 2 Jan 2013 15:53:44 +1100 Subject: [PATCH] Added per-pattern confidence levels, paved the way for version detection --- drivers/bookmarklet/js/wappalyzer.js | 109 +++++++++++++--------- drivers/bookmarklet/json | 28 +++--- drivers/chrome/apps.json | 28 +++--- drivers/chrome/js/popup.js | 2 +- drivers/chrome/js/wappalyzer.js | 109 +++++++++++++--------- drivers/firefox-jetpack/data/apps.json | 28 +++--- drivers/firefox-jetpack/lib/wappalyzer.js | 109 +++++++++++++--------- drivers/firefox/content/apps.json | 28 +++--- drivers/firefox/content/js/driver.js | 2 +- drivers/firefox/content/js/wappalyzer.js | 109 +++++++++++++--------- drivers/html/apps.json | 28 +++--- drivers/html/js/wappalyzer.js | 109 +++++++++++++--------- drivers/php/apps.json | 28 +++--- drivers/php/js/driver.js | 2 +- drivers/php/js/wappalyzer.js | 109 +++++++++++++--------- share/apps.json | 28 +++--- share/js/wappalyzer.js | 109 +++++++++++++--------- 17 files changed, 549 insertions(+), 416 deletions(-) diff --git a/drivers/bookmarklet/js/wappalyzer.js b/drivers/bookmarklet/js/wappalyzer.js index a80183a9d..e5791c7d5 100644 --- a/drivers/bookmarklet/js/wappalyzer.js +++ b/drivers/bookmarklet/js/wappalyzer.js @@ -9,6 +9,45 @@ var wappalyzer = (function() { //'use strict'; + /** + * Application class + */ + var Application = function(detected) { + this.confidence = {}; + this.confidenceTotal = 0; + this.detected = Boolean(detected); + this.versions = []; + + /** + * Calculate confidence total + */ + this.getConfidence = function() { + var total = 0; + + for ( id in this.confidence ) { + total += this.confidence[id]; + } + + return this.confidenceTotal = Math.min(total, 100); + } + + /** + * Resolve version number + */ + this.getVersion = function() { + return null; + } + + this.setDetected = function(pattern, type, key) { + this.detected = true; + + this.confidence[type + ' ' + ( key ? ' ' + key : '' ) + pattern.regex] = pattern.confidence ? pattern.confidence : 100; + + if ( pattern.version ) { + } + } + } + /** * Call driver functions */ @@ -127,19 +166,17 @@ var wappalyzer = (function() { */ analyze: function(hostname, url, data) { var - i, j, app, confidence, type, regexMeta, regexScript, match, content, meta, header, + i, j, app, type, regexMeta, regexScript, match, content, meta, header, profiler = { regexCount: 0, startTime: new Date().getTime() }, - apps = [] + apps = {} ; w.log('w.analyze'); - url = url.split('#')[0]; - - data.url = url; + data.url = url = url.split('#')[0]; if ( typeof w.apps === 'undefined' || typeof w.categories === 'undefined' ) { w.log('apps.json not loaded'); @@ -152,23 +189,16 @@ var wappalyzer = (function() { } for ( app in w.apps ) { - // Skip if the app has already been detected - if ( w.detected[url].hasOwnProperty(app) || apps.indexOf(app) !== -1 ) { - continue; - } + apps[app] = new Application(); for ( type in w.apps[app] ) { - confidence = {}; - - confidence[type] = w.apps[app].hasOwnProperty('confidence') && w.apps[app].confidence.hasOwnProperty(type) ? w.apps[app].confidence[type] : 100; - switch ( type ) { case 'url': parse(w.apps[app][type]).map(function(pattern) { profiler.regexCount ++; if ( pattern.regex.test(url) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -182,7 +212,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -201,7 +231,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(match[2]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -226,8 +256,8 @@ var wappalyzer = (function() { parse(w.apps[app].meta[meta]).map(function(pattern) { profiler.regexCount ++; - if ( content && content.length === 4 && regex.test(content[2]) ) { - apps[app] = confidence; + if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { + apps[app].setDetected(pattern, type, meta); } }); } @@ -245,7 +275,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( typeof data[type][header] === 'string' && pattern.regex.test(data[type][header]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type, header); } }); } @@ -261,7 +291,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type][i]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -273,11 +303,17 @@ var wappalyzer = (function() { w.log('Tested ' + profiler.regexCount + ' regular expressions in ' + ( ( ( new Date ).getTime() - profiler.startTime ) / 1000 ) + 's'); + for ( app in apps ) { + if ( !apps[app].detected ) { + delete apps[app]; + } + } + // Implied applications // Run several passes as implied apps may imply other apps for ( i = 0; i < 3; i ++ ) { for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; if ( w.apps[app] && w.apps[app].implies ) { w.apps[app].implies.map(function(implied) { @@ -289,46 +325,33 @@ var wappalyzer = (function() { // Apply app confidence to implied app if ( !apps.hasOwnProperty(implied) ) { - apps[implied] = {}; + apps[implied] = new Application(true); } - for ( type in confidence ) { - if ( !apps[implied].hasOwnProperty(type + ' implied by ' + app) ) { - apps[implied][type + ' implied by ' + app] = confidence[type]; - } + for ( id in confidence ) { + apps[implied].confidence[id + ' implied by ' + app] = confidence[id]; } }); } } } - w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + 'on ' + url); + w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url); // Keep history of detected apps for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; // Per URL if ( !w.detected[url].hasOwnProperty(app)) { - w.detected[url][app] = {}; - } - - for ( type in confidence ) { - w.detected[url][app][type] = confidence[type]; + w.detected[url][app] = new Application(); } - // Calculate confidence total - w.detected[url][app].total = 0; - - for ( type in w.detected[url][app] ) { - if ( type !== 'total' ) { - w.detected[url][app].total += w.detected[url][app][type]; - - w.detected[url][app].total = Math.min(w.detected[url][app].total, 100); - } + for ( id in confidence ) { + w.detected[url][app].confidence[id] = confidence[id]; } - if ( w.detected[url][app].total >= 100 ) { + if ( w.detected[url][app].getConfidence() >= 100 ) { // Per hostname if ( /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/.test(hostname) && !/((local|dev(elopment)?|stag(e|staging)?|test(ing)?|demo(shop)?|admin)\.|\/admin|\.local)/.test(url) ) { if ( !w.ping.hostnames.hasOwnProperty(hostname) ) { diff --git a/drivers/bookmarklet/json b/drivers/bookmarklet/json index e13b9b763..05e4a18f6 100644 --- a/drivers/bookmarklet/json +++ b/drivers/bookmarklet/json @@ -843,13 +843,12 @@ }, "Joomla": { "cats": [ 1 ], - "url": "option=com_", + "url": "option=com_\\;confidence:50", "meta": { "generator": "Joomla" }, - "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)", + "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)\\;confidence:50", "headers": { "X-Content-Encoded-By": "Joomla" }, "env": "^jcomments$", - "implies": [ "PHP" ], - "confidence": { "html": 50, "url": 50 } + "implies": [ "PHP" ] }, "jqPlot": { "cats": [ 25 ], @@ -863,7 +862,7 @@ }, "jQuery": { "cats": [ 12 ], - "script": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], + "scripts": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], "env": "^jQuery$" }, "jQuery Mobile": { @@ -980,10 +979,9 @@ "Magento": { "cats": [ 6 ], "script": "(js/mage|skin/frontend/(default|enterprise))", - "headers": { "Set-Cookie": "frontend=" }, + "headers": { "Set-Cookie": "frontend=\\;confidence:50" }, "env": "^(Mage|VarienForm)$", - "implies": [ "PHP" ], - "confidence": { "headers": 50 } + "implies": [ "PHP" ] }, "Mambo": { "cats": [ 1 ], @@ -1075,7 +1073,7 @@ "mod_rails": { "cats": [ 33 ], "headers": { "Server": "mod_rails(/([\\d\\.]+))?\\;version:\\2", "X-Powered-By": "mod_rails(/([\\d\\.]+))?\\;version:\\2" }, - "implies": [ "Ruby on Rails", "Ruby", "Apache" ] + "implies": [ "Apache" ] }, "mod_ssl": { "cats": [ 33 ], @@ -1476,11 +1474,10 @@ }, "Ruby on Rails": { "cats": [ 18 ], - "script": "/assets/application-[a-z\\d]{32}/\\.js", - "meta": { "csrf-param": "authenticity_token" }, - "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)" }, - "implies": [ "Ruby" ], - "confidence": { "script": 50, "meta": 50, "headers": 50 } + "script": "/assets/application-[a-z\\d]{32}/\\.js\\;confidence:50", + "meta": { "csrf-param": "authenticity_token\\;confidence:50" }, + "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50" }, + "implies": [ "Ruby" ] }, "S.Builder": { "cats": [ 1 ], @@ -1710,8 +1707,7 @@ "cats": [ 18 ], "script": "twitter\\.github\\.com/bootstrap", "html": "]+bootstrap[^>]+css", - "env": "^Twipsy$", - "confidence": { "env": 50 } + "env": "^Twipsy$\\;confidence:50" }, "Typekit": { "cats": [ 17 ], diff --git a/drivers/chrome/apps.json b/drivers/chrome/apps.json index e13b9b763..05e4a18f6 100644 --- a/drivers/chrome/apps.json +++ b/drivers/chrome/apps.json @@ -843,13 +843,12 @@ }, "Joomla": { "cats": [ 1 ], - "url": "option=com_", + "url": "option=com_\\;confidence:50", "meta": { "generator": "Joomla" }, - "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)", + "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)\\;confidence:50", "headers": { "X-Content-Encoded-By": "Joomla" }, "env": "^jcomments$", - "implies": [ "PHP" ], - "confidence": { "html": 50, "url": 50 } + "implies": [ "PHP" ] }, "jqPlot": { "cats": [ 25 ], @@ -863,7 +862,7 @@ }, "jQuery": { "cats": [ 12 ], - "script": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], + "scripts": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], "env": "^jQuery$" }, "jQuery Mobile": { @@ -980,10 +979,9 @@ "Magento": { "cats": [ 6 ], "script": "(js/mage|skin/frontend/(default|enterprise))", - "headers": { "Set-Cookie": "frontend=" }, + "headers": { "Set-Cookie": "frontend=\\;confidence:50" }, "env": "^(Mage|VarienForm)$", - "implies": [ "PHP" ], - "confidence": { "headers": 50 } + "implies": [ "PHP" ] }, "Mambo": { "cats": [ 1 ], @@ -1075,7 +1073,7 @@ "mod_rails": { "cats": [ 33 ], "headers": { "Server": "mod_rails(/([\\d\\.]+))?\\;version:\\2", "X-Powered-By": "mod_rails(/([\\d\\.]+))?\\;version:\\2" }, - "implies": [ "Ruby on Rails", "Ruby", "Apache" ] + "implies": [ "Apache" ] }, "mod_ssl": { "cats": [ 33 ], @@ -1476,11 +1474,10 @@ }, "Ruby on Rails": { "cats": [ 18 ], - "script": "/assets/application-[a-z\\d]{32}/\\.js", - "meta": { "csrf-param": "authenticity_token" }, - "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)" }, - "implies": [ "Ruby" ], - "confidence": { "script": 50, "meta": 50, "headers": 50 } + "script": "/assets/application-[a-z\\d]{32}/\\.js\\;confidence:50", + "meta": { "csrf-param": "authenticity_token\\;confidence:50" }, + "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50" }, + "implies": [ "Ruby" ] }, "S.Builder": { "cats": [ 1 ], @@ -1710,8 +1707,7 @@ "cats": [ 18 ], "script": "twitter\\.github\\.com/bootstrap", "html": "]+bootstrap[^>]+css", - "env": "^Twipsy$", - "confidence": { "env": 50 } + "env": "^Twipsy$\\;confidence:50" }, "Typekit": { "cats": [ 17 ], diff --git a/drivers/chrome/js/popup.js b/drivers/chrome/js/popup.js index a837ce39b..fd1b154bf 100644 --- a/drivers/chrome/js/popup.js +++ b/drivers/chrome/js/popup.js @@ -58,7 +58,7 @@ document.addEventListener('DOMContentLoaded', function() { detectedApps.innerHTML = ''; for ( appName in response.tabCache.appsDetected ) { - confidence = response.tabCache.appsDetected[appName].total; + confidence = response.tabCache.appsDetected[appName].confidenceTotal; html = '
' + diff --git a/drivers/chrome/js/wappalyzer.js b/drivers/chrome/js/wappalyzer.js index a80183a9d..e5791c7d5 100644 --- a/drivers/chrome/js/wappalyzer.js +++ b/drivers/chrome/js/wappalyzer.js @@ -9,6 +9,45 @@ var wappalyzer = (function() { //'use strict'; + /** + * Application class + */ + var Application = function(detected) { + this.confidence = {}; + this.confidenceTotal = 0; + this.detected = Boolean(detected); + this.versions = []; + + /** + * Calculate confidence total + */ + this.getConfidence = function() { + var total = 0; + + for ( id in this.confidence ) { + total += this.confidence[id]; + } + + return this.confidenceTotal = Math.min(total, 100); + } + + /** + * Resolve version number + */ + this.getVersion = function() { + return null; + } + + this.setDetected = function(pattern, type, key) { + this.detected = true; + + this.confidence[type + ' ' + ( key ? ' ' + key : '' ) + pattern.regex] = pattern.confidence ? pattern.confidence : 100; + + if ( pattern.version ) { + } + } + } + /** * Call driver functions */ @@ -127,19 +166,17 @@ var wappalyzer = (function() { */ analyze: function(hostname, url, data) { var - i, j, app, confidence, type, regexMeta, regexScript, match, content, meta, header, + i, j, app, type, regexMeta, regexScript, match, content, meta, header, profiler = { regexCount: 0, startTime: new Date().getTime() }, - apps = [] + apps = {} ; w.log('w.analyze'); - url = url.split('#')[0]; - - data.url = url; + data.url = url = url.split('#')[0]; if ( typeof w.apps === 'undefined' || typeof w.categories === 'undefined' ) { w.log('apps.json not loaded'); @@ -152,23 +189,16 @@ var wappalyzer = (function() { } for ( app in w.apps ) { - // Skip if the app has already been detected - if ( w.detected[url].hasOwnProperty(app) || apps.indexOf(app) !== -1 ) { - continue; - } + apps[app] = new Application(); for ( type in w.apps[app] ) { - confidence = {}; - - confidence[type] = w.apps[app].hasOwnProperty('confidence') && w.apps[app].confidence.hasOwnProperty(type) ? w.apps[app].confidence[type] : 100; - switch ( type ) { case 'url': parse(w.apps[app][type]).map(function(pattern) { profiler.regexCount ++; if ( pattern.regex.test(url) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -182,7 +212,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -201,7 +231,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(match[2]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -226,8 +256,8 @@ var wappalyzer = (function() { parse(w.apps[app].meta[meta]).map(function(pattern) { profiler.regexCount ++; - if ( content && content.length === 4 && regex.test(content[2]) ) { - apps[app] = confidence; + if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { + apps[app].setDetected(pattern, type, meta); } }); } @@ -245,7 +275,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( typeof data[type][header] === 'string' && pattern.regex.test(data[type][header]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type, header); } }); } @@ -261,7 +291,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type][i]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -273,11 +303,17 @@ var wappalyzer = (function() { w.log('Tested ' + profiler.regexCount + ' regular expressions in ' + ( ( ( new Date ).getTime() - profiler.startTime ) / 1000 ) + 's'); + for ( app in apps ) { + if ( !apps[app].detected ) { + delete apps[app]; + } + } + // Implied applications // Run several passes as implied apps may imply other apps for ( i = 0; i < 3; i ++ ) { for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; if ( w.apps[app] && w.apps[app].implies ) { w.apps[app].implies.map(function(implied) { @@ -289,46 +325,33 @@ var wappalyzer = (function() { // Apply app confidence to implied app if ( !apps.hasOwnProperty(implied) ) { - apps[implied] = {}; + apps[implied] = new Application(true); } - for ( type in confidence ) { - if ( !apps[implied].hasOwnProperty(type + ' implied by ' + app) ) { - apps[implied][type + ' implied by ' + app] = confidence[type]; - } + for ( id in confidence ) { + apps[implied].confidence[id + ' implied by ' + app] = confidence[id]; } }); } } } - w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + 'on ' + url); + w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url); // Keep history of detected apps for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; // Per URL if ( !w.detected[url].hasOwnProperty(app)) { - w.detected[url][app] = {}; - } - - for ( type in confidence ) { - w.detected[url][app][type] = confidence[type]; + w.detected[url][app] = new Application(); } - // Calculate confidence total - w.detected[url][app].total = 0; - - for ( type in w.detected[url][app] ) { - if ( type !== 'total' ) { - w.detected[url][app].total += w.detected[url][app][type]; - - w.detected[url][app].total = Math.min(w.detected[url][app].total, 100); - } + for ( id in confidence ) { + w.detected[url][app].confidence[id] = confidence[id]; } - if ( w.detected[url][app].total >= 100 ) { + if ( w.detected[url][app].getConfidence() >= 100 ) { // Per hostname if ( /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/.test(hostname) && !/((local|dev(elopment)?|stag(e|staging)?|test(ing)?|demo(shop)?|admin)\.|\/admin|\.local)/.test(url) ) { if ( !w.ping.hostnames.hasOwnProperty(hostname) ) { diff --git a/drivers/firefox-jetpack/data/apps.json b/drivers/firefox-jetpack/data/apps.json index e13b9b763..05e4a18f6 100644 --- a/drivers/firefox-jetpack/data/apps.json +++ b/drivers/firefox-jetpack/data/apps.json @@ -843,13 +843,12 @@ }, "Joomla": { "cats": [ 1 ], - "url": "option=com_", + "url": "option=com_\\;confidence:50", "meta": { "generator": "Joomla" }, - "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)", + "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)\\;confidence:50", "headers": { "X-Content-Encoded-By": "Joomla" }, "env": "^jcomments$", - "implies": [ "PHP" ], - "confidence": { "html": 50, "url": 50 } + "implies": [ "PHP" ] }, "jqPlot": { "cats": [ 25 ], @@ -863,7 +862,7 @@ }, "jQuery": { "cats": [ 12 ], - "script": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], + "scripts": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], "env": "^jQuery$" }, "jQuery Mobile": { @@ -980,10 +979,9 @@ "Magento": { "cats": [ 6 ], "script": "(js/mage|skin/frontend/(default|enterprise))", - "headers": { "Set-Cookie": "frontend=" }, + "headers": { "Set-Cookie": "frontend=\\;confidence:50" }, "env": "^(Mage|VarienForm)$", - "implies": [ "PHP" ], - "confidence": { "headers": 50 } + "implies": [ "PHP" ] }, "Mambo": { "cats": [ 1 ], @@ -1075,7 +1073,7 @@ "mod_rails": { "cats": [ 33 ], "headers": { "Server": "mod_rails(/([\\d\\.]+))?\\;version:\\2", "X-Powered-By": "mod_rails(/([\\d\\.]+))?\\;version:\\2" }, - "implies": [ "Ruby on Rails", "Ruby", "Apache" ] + "implies": [ "Apache" ] }, "mod_ssl": { "cats": [ 33 ], @@ -1476,11 +1474,10 @@ }, "Ruby on Rails": { "cats": [ 18 ], - "script": "/assets/application-[a-z\\d]{32}/\\.js", - "meta": { "csrf-param": "authenticity_token" }, - "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)" }, - "implies": [ "Ruby" ], - "confidence": { "script": 50, "meta": 50, "headers": 50 } + "script": "/assets/application-[a-z\\d]{32}/\\.js\\;confidence:50", + "meta": { "csrf-param": "authenticity_token\\;confidence:50" }, + "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50" }, + "implies": [ "Ruby" ] }, "S.Builder": { "cats": [ 1 ], @@ -1710,8 +1707,7 @@ "cats": [ 18 ], "script": "twitter\\.github\\.com/bootstrap", "html": "]+bootstrap[^>]+css", - "env": "^Twipsy$", - "confidence": { "env": 50 } + "env": "^Twipsy$\\;confidence:50" }, "Typekit": { "cats": [ 17 ], diff --git a/drivers/firefox-jetpack/lib/wappalyzer.js b/drivers/firefox-jetpack/lib/wappalyzer.js index a80183a9d..e5791c7d5 100644 --- a/drivers/firefox-jetpack/lib/wappalyzer.js +++ b/drivers/firefox-jetpack/lib/wappalyzer.js @@ -9,6 +9,45 @@ var wappalyzer = (function() { //'use strict'; + /** + * Application class + */ + var Application = function(detected) { + this.confidence = {}; + this.confidenceTotal = 0; + this.detected = Boolean(detected); + this.versions = []; + + /** + * Calculate confidence total + */ + this.getConfidence = function() { + var total = 0; + + for ( id in this.confidence ) { + total += this.confidence[id]; + } + + return this.confidenceTotal = Math.min(total, 100); + } + + /** + * Resolve version number + */ + this.getVersion = function() { + return null; + } + + this.setDetected = function(pattern, type, key) { + this.detected = true; + + this.confidence[type + ' ' + ( key ? ' ' + key : '' ) + pattern.regex] = pattern.confidence ? pattern.confidence : 100; + + if ( pattern.version ) { + } + } + } + /** * Call driver functions */ @@ -127,19 +166,17 @@ var wappalyzer = (function() { */ analyze: function(hostname, url, data) { var - i, j, app, confidence, type, regexMeta, regexScript, match, content, meta, header, + i, j, app, type, regexMeta, regexScript, match, content, meta, header, profiler = { regexCount: 0, startTime: new Date().getTime() }, - apps = [] + apps = {} ; w.log('w.analyze'); - url = url.split('#')[0]; - - data.url = url; + data.url = url = url.split('#')[0]; if ( typeof w.apps === 'undefined' || typeof w.categories === 'undefined' ) { w.log('apps.json not loaded'); @@ -152,23 +189,16 @@ var wappalyzer = (function() { } for ( app in w.apps ) { - // Skip if the app has already been detected - if ( w.detected[url].hasOwnProperty(app) || apps.indexOf(app) !== -1 ) { - continue; - } + apps[app] = new Application(); for ( type in w.apps[app] ) { - confidence = {}; - - confidence[type] = w.apps[app].hasOwnProperty('confidence') && w.apps[app].confidence.hasOwnProperty(type) ? w.apps[app].confidence[type] : 100; - switch ( type ) { case 'url': parse(w.apps[app][type]).map(function(pattern) { profiler.regexCount ++; if ( pattern.regex.test(url) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -182,7 +212,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -201,7 +231,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(match[2]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -226,8 +256,8 @@ var wappalyzer = (function() { parse(w.apps[app].meta[meta]).map(function(pattern) { profiler.regexCount ++; - if ( content && content.length === 4 && regex.test(content[2]) ) { - apps[app] = confidence; + if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { + apps[app].setDetected(pattern, type, meta); } }); } @@ -245,7 +275,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( typeof data[type][header] === 'string' && pattern.regex.test(data[type][header]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type, header); } }); } @@ -261,7 +291,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type][i]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -273,11 +303,17 @@ var wappalyzer = (function() { w.log('Tested ' + profiler.regexCount + ' regular expressions in ' + ( ( ( new Date ).getTime() - profiler.startTime ) / 1000 ) + 's'); + for ( app in apps ) { + if ( !apps[app].detected ) { + delete apps[app]; + } + } + // Implied applications // Run several passes as implied apps may imply other apps for ( i = 0; i < 3; i ++ ) { for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; if ( w.apps[app] && w.apps[app].implies ) { w.apps[app].implies.map(function(implied) { @@ -289,46 +325,33 @@ var wappalyzer = (function() { // Apply app confidence to implied app if ( !apps.hasOwnProperty(implied) ) { - apps[implied] = {}; + apps[implied] = new Application(true); } - for ( type in confidence ) { - if ( !apps[implied].hasOwnProperty(type + ' implied by ' + app) ) { - apps[implied][type + ' implied by ' + app] = confidence[type]; - } + for ( id in confidence ) { + apps[implied].confidence[id + ' implied by ' + app] = confidence[id]; } }); } } } - w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + 'on ' + url); + w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url); // Keep history of detected apps for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; // Per URL if ( !w.detected[url].hasOwnProperty(app)) { - w.detected[url][app] = {}; - } - - for ( type in confidence ) { - w.detected[url][app][type] = confidence[type]; + w.detected[url][app] = new Application(); } - // Calculate confidence total - w.detected[url][app].total = 0; - - for ( type in w.detected[url][app] ) { - if ( type !== 'total' ) { - w.detected[url][app].total += w.detected[url][app][type]; - - w.detected[url][app].total = Math.min(w.detected[url][app].total, 100); - } + for ( id in confidence ) { + w.detected[url][app].confidence[id] = confidence[id]; } - if ( w.detected[url][app].total >= 100 ) { + if ( w.detected[url][app].getConfidence() >= 100 ) { // Per hostname if ( /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/.test(hostname) && !/((local|dev(elopment)?|stag(e|staging)?|test(ing)?|demo(shop)?|admin)\.|\/admin|\.local)/.test(url) ) { if ( !w.ping.hostnames.hasOwnProperty(hostname) ) { diff --git a/drivers/firefox/content/apps.json b/drivers/firefox/content/apps.json index e13b9b763..05e4a18f6 100644 --- a/drivers/firefox/content/apps.json +++ b/drivers/firefox/content/apps.json @@ -843,13 +843,12 @@ }, "Joomla": { "cats": [ 1 ], - "url": "option=com_", + "url": "option=com_\\;confidence:50", "meta": { "generator": "Joomla" }, - "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)", + "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)\\;confidence:50", "headers": { "X-Content-Encoded-By": "Joomla" }, "env": "^jcomments$", - "implies": [ "PHP" ], - "confidence": { "html": 50, "url": 50 } + "implies": [ "PHP" ] }, "jqPlot": { "cats": [ 25 ], @@ -863,7 +862,7 @@ }, "jQuery": { "cats": [ 12 ], - "script": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], + "scripts": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], "env": "^jQuery$" }, "jQuery Mobile": { @@ -980,10 +979,9 @@ "Magento": { "cats": [ 6 ], "script": "(js/mage|skin/frontend/(default|enterprise))", - "headers": { "Set-Cookie": "frontend=" }, + "headers": { "Set-Cookie": "frontend=\\;confidence:50" }, "env": "^(Mage|VarienForm)$", - "implies": [ "PHP" ], - "confidence": { "headers": 50 } + "implies": [ "PHP" ] }, "Mambo": { "cats": [ 1 ], @@ -1075,7 +1073,7 @@ "mod_rails": { "cats": [ 33 ], "headers": { "Server": "mod_rails(/([\\d\\.]+))?\\;version:\\2", "X-Powered-By": "mod_rails(/([\\d\\.]+))?\\;version:\\2" }, - "implies": [ "Ruby on Rails", "Ruby", "Apache" ] + "implies": [ "Apache" ] }, "mod_ssl": { "cats": [ 33 ], @@ -1476,11 +1474,10 @@ }, "Ruby on Rails": { "cats": [ 18 ], - "script": "/assets/application-[a-z\\d]{32}/\\.js", - "meta": { "csrf-param": "authenticity_token" }, - "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)" }, - "implies": [ "Ruby" ], - "confidence": { "script": 50, "meta": 50, "headers": 50 } + "script": "/assets/application-[a-z\\d]{32}/\\.js\\;confidence:50", + "meta": { "csrf-param": "authenticity_token\\;confidence:50" }, + "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50" }, + "implies": [ "Ruby" ] }, "S.Builder": { "cats": [ 1 ], @@ -1710,8 +1707,7 @@ "cats": [ 18 ], "script": "twitter\\.github\\.com/bootstrap", "html": "]+bootstrap[^>]+css", - "env": "^Twipsy$", - "confidence": { "env": 50 } + "env": "^Twipsy$\\;confidence:50" }, "Typekit": { "cats": [ 17 ], diff --git a/drivers/firefox/content/js/driver.js b/drivers/firefox/content/js/driver.js index 8bc41d946..e208fb712 100644 --- a/drivers/firefox/content/js/driver.js +++ b/drivers/firefox/content/js/driver.js @@ -174,7 +174,7 @@ } for ( app in w.detected[url] ) { - confidence = w.detected[url][app].total; + confidence = w.detected[url][app].confidenceTotal; var j, cat, showCat, categories = []; diff --git a/drivers/firefox/content/js/wappalyzer.js b/drivers/firefox/content/js/wappalyzer.js index a80183a9d..e5791c7d5 100644 --- a/drivers/firefox/content/js/wappalyzer.js +++ b/drivers/firefox/content/js/wappalyzer.js @@ -9,6 +9,45 @@ var wappalyzer = (function() { //'use strict'; + /** + * Application class + */ + var Application = function(detected) { + this.confidence = {}; + this.confidenceTotal = 0; + this.detected = Boolean(detected); + this.versions = []; + + /** + * Calculate confidence total + */ + this.getConfidence = function() { + var total = 0; + + for ( id in this.confidence ) { + total += this.confidence[id]; + } + + return this.confidenceTotal = Math.min(total, 100); + } + + /** + * Resolve version number + */ + this.getVersion = function() { + return null; + } + + this.setDetected = function(pattern, type, key) { + this.detected = true; + + this.confidence[type + ' ' + ( key ? ' ' + key : '' ) + pattern.regex] = pattern.confidence ? pattern.confidence : 100; + + if ( pattern.version ) { + } + } + } + /** * Call driver functions */ @@ -127,19 +166,17 @@ var wappalyzer = (function() { */ analyze: function(hostname, url, data) { var - i, j, app, confidence, type, regexMeta, regexScript, match, content, meta, header, + i, j, app, type, regexMeta, regexScript, match, content, meta, header, profiler = { regexCount: 0, startTime: new Date().getTime() }, - apps = [] + apps = {} ; w.log('w.analyze'); - url = url.split('#')[0]; - - data.url = url; + data.url = url = url.split('#')[0]; if ( typeof w.apps === 'undefined' || typeof w.categories === 'undefined' ) { w.log('apps.json not loaded'); @@ -152,23 +189,16 @@ var wappalyzer = (function() { } for ( app in w.apps ) { - // Skip if the app has already been detected - if ( w.detected[url].hasOwnProperty(app) || apps.indexOf(app) !== -1 ) { - continue; - } + apps[app] = new Application(); for ( type in w.apps[app] ) { - confidence = {}; - - confidence[type] = w.apps[app].hasOwnProperty('confidence') && w.apps[app].confidence.hasOwnProperty(type) ? w.apps[app].confidence[type] : 100; - switch ( type ) { case 'url': parse(w.apps[app][type]).map(function(pattern) { profiler.regexCount ++; if ( pattern.regex.test(url) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -182,7 +212,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -201,7 +231,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(match[2]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -226,8 +256,8 @@ var wappalyzer = (function() { parse(w.apps[app].meta[meta]).map(function(pattern) { profiler.regexCount ++; - if ( content && content.length === 4 && regex.test(content[2]) ) { - apps[app] = confidence; + if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { + apps[app].setDetected(pattern, type, meta); } }); } @@ -245,7 +275,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( typeof data[type][header] === 'string' && pattern.regex.test(data[type][header]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type, header); } }); } @@ -261,7 +291,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type][i]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -273,11 +303,17 @@ var wappalyzer = (function() { w.log('Tested ' + profiler.regexCount + ' regular expressions in ' + ( ( ( new Date ).getTime() - profiler.startTime ) / 1000 ) + 's'); + for ( app in apps ) { + if ( !apps[app].detected ) { + delete apps[app]; + } + } + // Implied applications // Run several passes as implied apps may imply other apps for ( i = 0; i < 3; i ++ ) { for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; if ( w.apps[app] && w.apps[app].implies ) { w.apps[app].implies.map(function(implied) { @@ -289,46 +325,33 @@ var wappalyzer = (function() { // Apply app confidence to implied app if ( !apps.hasOwnProperty(implied) ) { - apps[implied] = {}; + apps[implied] = new Application(true); } - for ( type in confidence ) { - if ( !apps[implied].hasOwnProperty(type + ' implied by ' + app) ) { - apps[implied][type + ' implied by ' + app] = confidence[type]; - } + for ( id in confidence ) { + apps[implied].confidence[id + ' implied by ' + app] = confidence[id]; } }); } } } - w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + 'on ' + url); + w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url); // Keep history of detected apps for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; // Per URL if ( !w.detected[url].hasOwnProperty(app)) { - w.detected[url][app] = {}; - } - - for ( type in confidence ) { - w.detected[url][app][type] = confidence[type]; + w.detected[url][app] = new Application(); } - // Calculate confidence total - w.detected[url][app].total = 0; - - for ( type in w.detected[url][app] ) { - if ( type !== 'total' ) { - w.detected[url][app].total += w.detected[url][app][type]; - - w.detected[url][app].total = Math.min(w.detected[url][app].total, 100); - } + for ( id in confidence ) { + w.detected[url][app].confidence[id] = confidence[id]; } - if ( w.detected[url][app].total >= 100 ) { + if ( w.detected[url][app].getConfidence() >= 100 ) { // Per hostname if ( /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/.test(hostname) && !/((local|dev(elopment)?|stag(e|staging)?|test(ing)?|demo(shop)?|admin)\.|\/admin|\.local)/.test(url) ) { if ( !w.ping.hostnames.hasOwnProperty(hostname) ) { diff --git a/drivers/html/apps.json b/drivers/html/apps.json index e13b9b763..05e4a18f6 100644 --- a/drivers/html/apps.json +++ b/drivers/html/apps.json @@ -843,13 +843,12 @@ }, "Joomla": { "cats": [ 1 ], - "url": "option=com_", + "url": "option=com_\\;confidence:50", "meta": { "generator": "Joomla" }, - "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)", + "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)\\;confidence:50", "headers": { "X-Content-Encoded-By": "Joomla" }, "env": "^jcomments$", - "implies": [ "PHP" ], - "confidence": { "html": 50, "url": 50 } + "implies": [ "PHP" ] }, "jqPlot": { "cats": [ 25 ], @@ -863,7 +862,7 @@ }, "jQuery": { "cats": [ 12 ], - "script": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], + "scripts": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], "env": "^jQuery$" }, "jQuery Mobile": { @@ -980,10 +979,9 @@ "Magento": { "cats": [ 6 ], "script": "(js/mage|skin/frontend/(default|enterprise))", - "headers": { "Set-Cookie": "frontend=" }, + "headers": { "Set-Cookie": "frontend=\\;confidence:50" }, "env": "^(Mage|VarienForm)$", - "implies": [ "PHP" ], - "confidence": { "headers": 50 } + "implies": [ "PHP" ] }, "Mambo": { "cats": [ 1 ], @@ -1075,7 +1073,7 @@ "mod_rails": { "cats": [ 33 ], "headers": { "Server": "mod_rails(/([\\d\\.]+))?\\;version:\\2", "X-Powered-By": "mod_rails(/([\\d\\.]+))?\\;version:\\2" }, - "implies": [ "Ruby on Rails", "Ruby", "Apache" ] + "implies": [ "Apache" ] }, "mod_ssl": { "cats": [ 33 ], @@ -1476,11 +1474,10 @@ }, "Ruby on Rails": { "cats": [ 18 ], - "script": "/assets/application-[a-z\\d]{32}/\\.js", - "meta": { "csrf-param": "authenticity_token" }, - "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)" }, - "implies": [ "Ruby" ], - "confidence": { "script": 50, "meta": 50, "headers": 50 } + "script": "/assets/application-[a-z\\d]{32}/\\.js\\;confidence:50", + "meta": { "csrf-param": "authenticity_token\\;confidence:50" }, + "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50" }, + "implies": [ "Ruby" ] }, "S.Builder": { "cats": [ 1 ], @@ -1710,8 +1707,7 @@ "cats": [ 18 ], "script": "twitter\\.github\\.com/bootstrap", "html": "]+bootstrap[^>]+css", - "env": "^Twipsy$", - "confidence": { "env": 50 } + "env": "^Twipsy$\\;confidence:50" }, "Typekit": { "cats": [ 17 ], diff --git a/drivers/html/js/wappalyzer.js b/drivers/html/js/wappalyzer.js index a80183a9d..e5791c7d5 100644 --- a/drivers/html/js/wappalyzer.js +++ b/drivers/html/js/wappalyzer.js @@ -9,6 +9,45 @@ var wappalyzer = (function() { //'use strict'; + /** + * Application class + */ + var Application = function(detected) { + this.confidence = {}; + this.confidenceTotal = 0; + this.detected = Boolean(detected); + this.versions = []; + + /** + * Calculate confidence total + */ + this.getConfidence = function() { + var total = 0; + + for ( id in this.confidence ) { + total += this.confidence[id]; + } + + return this.confidenceTotal = Math.min(total, 100); + } + + /** + * Resolve version number + */ + this.getVersion = function() { + return null; + } + + this.setDetected = function(pattern, type, key) { + this.detected = true; + + this.confidence[type + ' ' + ( key ? ' ' + key : '' ) + pattern.regex] = pattern.confidence ? pattern.confidence : 100; + + if ( pattern.version ) { + } + } + } + /** * Call driver functions */ @@ -127,19 +166,17 @@ var wappalyzer = (function() { */ analyze: function(hostname, url, data) { var - i, j, app, confidence, type, regexMeta, regexScript, match, content, meta, header, + i, j, app, type, regexMeta, regexScript, match, content, meta, header, profiler = { regexCount: 0, startTime: new Date().getTime() }, - apps = [] + apps = {} ; w.log('w.analyze'); - url = url.split('#')[0]; - - data.url = url; + data.url = url = url.split('#')[0]; if ( typeof w.apps === 'undefined' || typeof w.categories === 'undefined' ) { w.log('apps.json not loaded'); @@ -152,23 +189,16 @@ var wappalyzer = (function() { } for ( app in w.apps ) { - // Skip if the app has already been detected - if ( w.detected[url].hasOwnProperty(app) || apps.indexOf(app) !== -1 ) { - continue; - } + apps[app] = new Application(); for ( type in w.apps[app] ) { - confidence = {}; - - confidence[type] = w.apps[app].hasOwnProperty('confidence') && w.apps[app].confidence.hasOwnProperty(type) ? w.apps[app].confidence[type] : 100; - switch ( type ) { case 'url': parse(w.apps[app][type]).map(function(pattern) { profiler.regexCount ++; if ( pattern.regex.test(url) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -182,7 +212,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -201,7 +231,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(match[2]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -226,8 +256,8 @@ var wappalyzer = (function() { parse(w.apps[app].meta[meta]).map(function(pattern) { profiler.regexCount ++; - if ( content && content.length === 4 && regex.test(content[2]) ) { - apps[app] = confidence; + if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { + apps[app].setDetected(pattern, type, meta); } }); } @@ -245,7 +275,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( typeof data[type][header] === 'string' && pattern.regex.test(data[type][header]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type, header); } }); } @@ -261,7 +291,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type][i]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -273,11 +303,17 @@ var wappalyzer = (function() { w.log('Tested ' + profiler.regexCount + ' regular expressions in ' + ( ( ( new Date ).getTime() - profiler.startTime ) / 1000 ) + 's'); + for ( app in apps ) { + if ( !apps[app].detected ) { + delete apps[app]; + } + } + // Implied applications // Run several passes as implied apps may imply other apps for ( i = 0; i < 3; i ++ ) { for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; if ( w.apps[app] && w.apps[app].implies ) { w.apps[app].implies.map(function(implied) { @@ -289,46 +325,33 @@ var wappalyzer = (function() { // Apply app confidence to implied app if ( !apps.hasOwnProperty(implied) ) { - apps[implied] = {}; + apps[implied] = new Application(true); } - for ( type in confidence ) { - if ( !apps[implied].hasOwnProperty(type + ' implied by ' + app) ) { - apps[implied][type + ' implied by ' + app] = confidence[type]; - } + for ( id in confidence ) { + apps[implied].confidence[id + ' implied by ' + app] = confidence[id]; } }); } } } - w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + 'on ' + url); + w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url); // Keep history of detected apps for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; // Per URL if ( !w.detected[url].hasOwnProperty(app)) { - w.detected[url][app] = {}; - } - - for ( type in confidence ) { - w.detected[url][app][type] = confidence[type]; + w.detected[url][app] = new Application(); } - // Calculate confidence total - w.detected[url][app].total = 0; - - for ( type in w.detected[url][app] ) { - if ( type !== 'total' ) { - w.detected[url][app].total += w.detected[url][app][type]; - - w.detected[url][app].total = Math.min(w.detected[url][app].total, 100); - } + for ( id in confidence ) { + w.detected[url][app].confidence[id] = confidence[id]; } - if ( w.detected[url][app].total >= 100 ) { + if ( w.detected[url][app].getConfidence() >= 100 ) { // Per hostname if ( /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/.test(hostname) && !/((local|dev(elopment)?|stag(e|staging)?|test(ing)?|demo(shop)?|admin)\.|\/admin|\.local)/.test(url) ) { if ( !w.ping.hostnames.hasOwnProperty(hostname) ) { diff --git a/drivers/php/apps.json b/drivers/php/apps.json index e13b9b763..05e4a18f6 100644 --- a/drivers/php/apps.json +++ b/drivers/php/apps.json @@ -843,13 +843,12 @@ }, "Joomla": { "cats": [ 1 ], - "url": "option=com_", + "url": "option=com_\\;confidence:50", "meta": { "generator": "Joomla" }, - "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)", + "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)\\;confidence:50", "headers": { "X-Content-Encoded-By": "Joomla" }, "env": "^jcomments$", - "implies": [ "PHP" ], - "confidence": { "html": 50, "url": 50 } + "implies": [ "PHP" ] }, "jqPlot": { "cats": [ 25 ], @@ -863,7 +862,7 @@ }, "jQuery": { "cats": [ 12 ], - "script": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], + "scripts": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], "env": "^jQuery$" }, "jQuery Mobile": { @@ -980,10 +979,9 @@ "Magento": { "cats": [ 6 ], "script": "(js/mage|skin/frontend/(default|enterprise))", - "headers": { "Set-Cookie": "frontend=" }, + "headers": { "Set-Cookie": "frontend=\\;confidence:50" }, "env": "^(Mage|VarienForm)$", - "implies": [ "PHP" ], - "confidence": { "headers": 50 } + "implies": [ "PHP" ] }, "Mambo": { "cats": [ 1 ], @@ -1075,7 +1073,7 @@ "mod_rails": { "cats": [ 33 ], "headers": { "Server": "mod_rails(/([\\d\\.]+))?\\;version:\\2", "X-Powered-By": "mod_rails(/([\\d\\.]+))?\\;version:\\2" }, - "implies": [ "Ruby on Rails", "Ruby", "Apache" ] + "implies": [ "Apache" ] }, "mod_ssl": { "cats": [ 33 ], @@ -1476,11 +1474,10 @@ }, "Ruby on Rails": { "cats": [ 18 ], - "script": "/assets/application-[a-z\\d]{32}/\\.js", - "meta": { "csrf-param": "authenticity_token" }, - "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)" }, - "implies": [ "Ruby" ], - "confidence": { "script": 50, "meta": 50, "headers": 50 } + "script": "/assets/application-[a-z\\d]{32}/\\.js\\;confidence:50", + "meta": { "csrf-param": "authenticity_token\\;confidence:50" }, + "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50" }, + "implies": [ "Ruby" ] }, "S.Builder": { "cats": [ 1 ], @@ -1710,8 +1707,7 @@ "cats": [ 18 ], "script": "twitter\\.github\\.com/bootstrap", "html": "]+bootstrap[^>]+css", - "env": "^Twipsy$", - "confidence": { "env": 50 } + "env": "^Twipsy$\\;confidence:50" }, "Typekit": { "cats": [ 17 ], diff --git a/drivers/php/js/driver.js b/drivers/php/js/driver.js index 6cc6fc3f9..8c2499e0a 100644 --- a/drivers/php/js/driver.js +++ b/drivers/php/js/driver.js @@ -25,7 +25,7 @@ w.driver = { for ( app in w.detected[w.driver.data.url] ) { apps[app] = { categories: [], - confidence: w.detected[w.driver.data.url][app].total + confidence: w.detected[w.driver.data.url][app].confidenceTotal }; w.apps[app].cats.map(function(cat) { diff --git a/drivers/php/js/wappalyzer.js b/drivers/php/js/wappalyzer.js index a80183a9d..e5791c7d5 100644 --- a/drivers/php/js/wappalyzer.js +++ b/drivers/php/js/wappalyzer.js @@ -9,6 +9,45 @@ var wappalyzer = (function() { //'use strict'; + /** + * Application class + */ + var Application = function(detected) { + this.confidence = {}; + this.confidenceTotal = 0; + this.detected = Boolean(detected); + this.versions = []; + + /** + * Calculate confidence total + */ + this.getConfidence = function() { + var total = 0; + + for ( id in this.confidence ) { + total += this.confidence[id]; + } + + return this.confidenceTotal = Math.min(total, 100); + } + + /** + * Resolve version number + */ + this.getVersion = function() { + return null; + } + + this.setDetected = function(pattern, type, key) { + this.detected = true; + + this.confidence[type + ' ' + ( key ? ' ' + key : '' ) + pattern.regex] = pattern.confidence ? pattern.confidence : 100; + + if ( pattern.version ) { + } + } + } + /** * Call driver functions */ @@ -127,19 +166,17 @@ var wappalyzer = (function() { */ analyze: function(hostname, url, data) { var - i, j, app, confidence, type, regexMeta, regexScript, match, content, meta, header, + i, j, app, type, regexMeta, regexScript, match, content, meta, header, profiler = { regexCount: 0, startTime: new Date().getTime() }, - apps = [] + apps = {} ; w.log('w.analyze'); - url = url.split('#')[0]; - - data.url = url; + data.url = url = url.split('#')[0]; if ( typeof w.apps === 'undefined' || typeof w.categories === 'undefined' ) { w.log('apps.json not loaded'); @@ -152,23 +189,16 @@ var wappalyzer = (function() { } for ( app in w.apps ) { - // Skip if the app has already been detected - if ( w.detected[url].hasOwnProperty(app) || apps.indexOf(app) !== -1 ) { - continue; - } + apps[app] = new Application(); for ( type in w.apps[app] ) { - confidence = {}; - - confidence[type] = w.apps[app].hasOwnProperty('confidence') && w.apps[app].confidence.hasOwnProperty(type) ? w.apps[app].confidence[type] : 100; - switch ( type ) { case 'url': parse(w.apps[app][type]).map(function(pattern) { profiler.regexCount ++; if ( pattern.regex.test(url) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -182,7 +212,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -201,7 +231,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(match[2]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -226,8 +256,8 @@ var wappalyzer = (function() { parse(w.apps[app].meta[meta]).map(function(pattern) { profiler.regexCount ++; - if ( content && content.length === 4 && regex.test(content[2]) ) { - apps[app] = confidence; + if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { + apps[app].setDetected(pattern, type, meta); } }); } @@ -245,7 +275,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( typeof data[type][header] === 'string' && pattern.regex.test(data[type][header]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type, header); } }); } @@ -261,7 +291,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type][i]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -273,11 +303,17 @@ var wappalyzer = (function() { w.log('Tested ' + profiler.regexCount + ' regular expressions in ' + ( ( ( new Date ).getTime() - profiler.startTime ) / 1000 ) + 's'); + for ( app in apps ) { + if ( !apps[app].detected ) { + delete apps[app]; + } + } + // Implied applications // Run several passes as implied apps may imply other apps for ( i = 0; i < 3; i ++ ) { for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; if ( w.apps[app] && w.apps[app].implies ) { w.apps[app].implies.map(function(implied) { @@ -289,46 +325,33 @@ var wappalyzer = (function() { // Apply app confidence to implied app if ( !apps.hasOwnProperty(implied) ) { - apps[implied] = {}; + apps[implied] = new Application(true); } - for ( type in confidence ) { - if ( !apps[implied].hasOwnProperty(type + ' implied by ' + app) ) { - apps[implied][type + ' implied by ' + app] = confidence[type]; - } + for ( id in confidence ) { + apps[implied].confidence[id + ' implied by ' + app] = confidence[id]; } }); } } } - w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + 'on ' + url); + w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url); // Keep history of detected apps for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; // Per URL if ( !w.detected[url].hasOwnProperty(app)) { - w.detected[url][app] = {}; - } - - for ( type in confidence ) { - w.detected[url][app][type] = confidence[type]; + w.detected[url][app] = new Application(); } - // Calculate confidence total - w.detected[url][app].total = 0; - - for ( type in w.detected[url][app] ) { - if ( type !== 'total' ) { - w.detected[url][app].total += w.detected[url][app][type]; - - w.detected[url][app].total = Math.min(w.detected[url][app].total, 100); - } + for ( id in confidence ) { + w.detected[url][app].confidence[id] = confidence[id]; } - if ( w.detected[url][app].total >= 100 ) { + if ( w.detected[url][app].getConfidence() >= 100 ) { // Per hostname if ( /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/.test(hostname) && !/((local|dev(elopment)?|stag(e|staging)?|test(ing)?|demo(shop)?|admin)\.|\/admin|\.local)/.test(url) ) { if ( !w.ping.hostnames.hasOwnProperty(hostname) ) { diff --git a/share/apps.json b/share/apps.json index e13b9b763..05e4a18f6 100644 --- a/share/apps.json +++ b/share/apps.json @@ -843,13 +843,12 @@ }, "Joomla": { "cats": [ 1 ], - "url": "option=com_", + "url": "option=com_\\;confidence:50", "meta": { "generator": "Joomla" }, - "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)", + "html": "(]+id=\\\"wrapper_r\\\"|<[^>]+(feed|components)/com_|]+class=\\\"pill)\\;confidence:50", "headers": { "X-Content-Encoded-By": "Joomla" }, "env": "^jcomments$", - "implies": [ "PHP" ], - "confidence": { "html": 50, "url": 50 } + "implies": [ "PHP" ] }, "jqPlot": { "cats": [ 25 ], @@ -863,7 +862,7 @@ }, "jQuery": { "cats": [ 12 ], - "script": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], + "scripts": [ "jquery(\\-|\\.)([\\d\\.]*\\d)[^/]*\\.js\\;version:\\2", "([\\d\\.]+)/jquery(\\.min)?\\.js\\;version:\\1", "jquery.*\\.js" ], "env": "^jQuery$" }, "jQuery Mobile": { @@ -980,10 +979,9 @@ "Magento": { "cats": [ 6 ], "script": "(js/mage|skin/frontend/(default|enterprise))", - "headers": { "Set-Cookie": "frontend=" }, + "headers": { "Set-Cookie": "frontend=\\;confidence:50" }, "env": "^(Mage|VarienForm)$", - "implies": [ "PHP" ], - "confidence": { "headers": 50 } + "implies": [ "PHP" ] }, "Mambo": { "cats": [ 1 ], @@ -1075,7 +1073,7 @@ "mod_rails": { "cats": [ 33 ], "headers": { "Server": "mod_rails(/([\\d\\.]+))?\\;version:\\2", "X-Powered-By": "mod_rails(/([\\d\\.]+))?\\;version:\\2" }, - "implies": [ "Ruby on Rails", "Ruby", "Apache" ] + "implies": [ "Apache" ] }, "mod_ssl": { "cats": [ 33 ], @@ -1476,11 +1474,10 @@ }, "Ruby on Rails": { "cats": [ 18 ], - "script": "/assets/application-[a-z\\d]{32}/\\.js", - "meta": { "csrf-param": "authenticity_token" }, - "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)" }, - "implies": [ "Ruby" ], - "confidence": { "script": 50, "meta": 50, "headers": 50 } + "script": "/assets/application-[a-z\\d]{32}/\\.js\\;confidence:50", + "meta": { "csrf-param": "authenticity_token\\;confidence:50" }, + "headers": { "Server": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50", "X-Powered-By": "(mod_rails|mod_rack|Phusion(\\.|_)Passenger)\\;confidence:50" }, + "implies": [ "Ruby" ] }, "S.Builder": { "cats": [ 1 ], @@ -1710,8 +1707,7 @@ "cats": [ 18 ], "script": "twitter\\.github\\.com/bootstrap", "html": "]+bootstrap[^>]+css", - "env": "^Twipsy$", - "confidence": { "env": 50 } + "env": "^Twipsy$\\;confidence:50" }, "Typekit": { "cats": [ 17 ], diff --git a/share/js/wappalyzer.js b/share/js/wappalyzer.js index a80183a9d..e5791c7d5 100644 --- a/share/js/wappalyzer.js +++ b/share/js/wappalyzer.js @@ -9,6 +9,45 @@ var wappalyzer = (function() { //'use strict'; + /** + * Application class + */ + var Application = function(detected) { + this.confidence = {}; + this.confidenceTotal = 0; + this.detected = Boolean(detected); + this.versions = []; + + /** + * Calculate confidence total + */ + this.getConfidence = function() { + var total = 0; + + for ( id in this.confidence ) { + total += this.confidence[id]; + } + + return this.confidenceTotal = Math.min(total, 100); + } + + /** + * Resolve version number + */ + this.getVersion = function() { + return null; + } + + this.setDetected = function(pattern, type, key) { + this.detected = true; + + this.confidence[type + ' ' + ( key ? ' ' + key : '' ) + pattern.regex] = pattern.confidence ? pattern.confidence : 100; + + if ( pattern.version ) { + } + } + } + /** * Call driver functions */ @@ -127,19 +166,17 @@ var wappalyzer = (function() { */ analyze: function(hostname, url, data) { var - i, j, app, confidence, type, regexMeta, regexScript, match, content, meta, header, + i, j, app, type, regexMeta, regexScript, match, content, meta, header, profiler = { regexCount: 0, startTime: new Date().getTime() }, - apps = [] + apps = {} ; w.log('w.analyze'); - url = url.split('#')[0]; - - data.url = url; + data.url = url = url.split('#')[0]; if ( typeof w.apps === 'undefined' || typeof w.categories === 'undefined' ) { w.log('apps.json not loaded'); @@ -152,23 +189,16 @@ var wappalyzer = (function() { } for ( app in w.apps ) { - // Skip if the app has already been detected - if ( w.detected[url].hasOwnProperty(app) || apps.indexOf(app) !== -1 ) { - continue; - } + apps[app] = new Application(); for ( type in w.apps[app] ) { - confidence = {}; - - confidence[type] = w.apps[app].hasOwnProperty('confidence') && w.apps[app].confidence.hasOwnProperty(type) ? w.apps[app].confidence[type] : 100; - switch ( type ) { case 'url': parse(w.apps[app][type]).map(function(pattern) { profiler.regexCount ++; if ( pattern.regex.test(url) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -182,7 +212,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } }); @@ -201,7 +231,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(match[2]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -226,8 +256,8 @@ var wappalyzer = (function() { parse(w.apps[app].meta[meta]).map(function(pattern) { profiler.regexCount ++; - if ( content && content.length === 4 && regex.test(content[2]) ) { - apps[app] = confidence; + if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { + apps[app].setDetected(pattern, type, meta); } }); } @@ -245,7 +275,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( typeof data[type][header] === 'string' && pattern.regex.test(data[type][header]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type, header); } }); } @@ -261,7 +291,7 @@ var wappalyzer = (function() { profiler.regexCount ++; if ( pattern.regex.test(data[type][i]) ) { - apps[app] = confidence; + apps[app].setDetected(pattern, type); } } }); @@ -273,11 +303,17 @@ var wappalyzer = (function() { w.log('Tested ' + profiler.regexCount + ' regular expressions in ' + ( ( ( new Date ).getTime() - profiler.startTime ) / 1000 ) + 's'); + for ( app in apps ) { + if ( !apps[app].detected ) { + delete apps[app]; + } + } + // Implied applications // Run several passes as implied apps may imply other apps for ( i = 0; i < 3; i ++ ) { for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; if ( w.apps[app] && w.apps[app].implies ) { w.apps[app].implies.map(function(implied) { @@ -289,46 +325,33 @@ var wappalyzer = (function() { // Apply app confidence to implied app if ( !apps.hasOwnProperty(implied) ) { - apps[implied] = {}; + apps[implied] = new Application(true); } - for ( type in confidence ) { - if ( !apps[implied].hasOwnProperty(type + ' implied by ' + app) ) { - apps[implied][type + ' implied by ' + app] = confidence[type]; - } + for ( id in confidence ) { + apps[implied].confidence[id + ' implied by ' + app] = confidence[id]; } }); } } } - w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + 'on ' + url); + w.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url); // Keep history of detected apps for ( app in apps ) { - confidence = apps[app]; + confidence = apps[app].confidence; // Per URL if ( !w.detected[url].hasOwnProperty(app)) { - w.detected[url][app] = {}; - } - - for ( type in confidence ) { - w.detected[url][app][type] = confidence[type]; + w.detected[url][app] = new Application(); } - // Calculate confidence total - w.detected[url][app].total = 0; - - for ( type in w.detected[url][app] ) { - if ( type !== 'total' ) { - w.detected[url][app].total += w.detected[url][app][type]; - - w.detected[url][app].total = Math.min(w.detected[url][app].total, 100); - } + for ( id in confidence ) { + w.detected[url][app].confidence[id] = confidence[id]; } - if ( w.detected[url][app].total >= 100 ) { + if ( w.detected[url][app].getConfidence() >= 100 ) { // Per hostname if ( /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/.test(hostname) && !/((local|dev(elopment)?|stag(e|staging)?|test(ing)?|demo(shop)?|admin)\.|\/admin|\.local)/.test(url) ) { if ( !w.ping.hostnames.hasOwnProperty(hostname) ) {