diff --git a/run b/run index a8ec820ef..102fc812d 100755 --- a/run +++ b/run @@ -10,13 +10,7 @@ fi cmd="docker run --rm -v "$(pwd):/opt/wappalyzer" -it wappalyzer/dev" -$cmd yarn install - -pushd src/drivers/webextension - -$cmd yarn install - -popd +$cmd sh -c "yarn install; cd src/drivers/webextension; yarn install" $cmd ./bin/run links $cmd ./bin/run $@ diff --git a/src/apps.json b/src/apps.json index ac052dc09..24326e933 100755 --- a/src/apps.json +++ b/src/apps.json @@ -401,7 +401,7 @@ "headers": { "X-Akamai-Transformed": "" }, - "icon": "Akamai.png", + "icon": "akamai.svg", "website": "http://akamai.com" }, "Akka HTTP": { @@ -939,6 +939,14 @@ "script": "bittads\\.com/js/bitt\\.js$", "website": "http://bittads.com" }, + "Bizweb":{ + "cats": [ + "6" + ], + "env": "^Bizweb$", + "icon": "bizweb.png", + "website": "https://www.bizweb.vn" + }, "Blesta": { "cats": [ "6" @@ -5331,7 +5339,7 @@ "cats": [ "12" ], - "icon": "moon.png", + "icon": "moon.svg", "script": "/moon(?:\\.min)?\\.js$", "website": "http://moonjs.ga/" }, @@ -5585,6 +5593,24 @@ "icon": "New Relic.png", "website": "http://newrelic.com" }, + "Next.js": { + "cats": [ + "18", + "22" + ], + "headers": { + "x-powered-by": "^Next.js ?([0-9.]+)?\\;version:\\1" + }, + "html": "<[^>]+__next", + "env": "^__NEXT_DATA__$", + "icon": "zeit.svg", + "implies": [ + "React", + "webpack", + "Node.js" + ], + "website": "https://zeit.co/next" + }, "Nginx": { "cats": [ "22" @@ -5614,6 +5640,16 @@ "script": "^/nodebb\\.min\\.js\\?", "website": "https://nodebb.org" }, + "Now": { + "cats": [ + "22" + ], + "headers": { + "server": "now" + }, + "icon": "zeit.svg", + "website": "https://zeit.co/now" + }, "OWL Carousel": { "cats": [ "5" @@ -8430,6 +8466,10 @@ ], "icon": "typecho.svg", "implies": "PHP", + "env": "^TypechoComment$", + "meta": { + "generator": "Typecho( [\\d.]+)?\\;version:\\1" + }, "url": "/admin/login\\.php?referer=http%3A%2F%2F", "website": "http://typecho.org/" }, @@ -8521,7 +8561,7 @@ }, "UMI.CMS": { "cats": [ - 1 + "1" ], "headers": { "X-Generated-By": "UMI.CMS" diff --git a/src/drivers/bookmarklet/driver.js b/src/drivers/bookmarklet/driver.js index 230ff6aaf..28cfca9c1 100644 --- a/src/drivers/bookmarklet/driver.js +++ b/src/drivers/bookmarklet/driver.js @@ -105,9 +105,9 @@ '' + '
'; - if ( w.detected[url] != null && Object.keys(w.detected[url]).length ) { - for ( app in w.detected[url] ) { - if ( !hasOwn.call(w.detected[url], app) ) { + if ( detected[url] != null && Object.keys(detected[url]).length ) { + for ( app in detected[url] ) { + if ( !hasOwn.call(detected[url], app) ) { continue; } @@ -154,5 +154,5 @@ } }; - w.init(); + w.driver.init(); })(); diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index b31c66205..ef7569b17 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -1,19 +1,21 @@ 'use strict'; -const wappalyzer = require('./wappalyzer'); +const Wappalyzer = require('./wappalyzer'); const request = require('request'); const fs = require('fs'); const Browser = require('zombie'); const json = JSON.parse(fs.readFileSync(__dirname + '/apps.json')); -wappalyzer.apps = json.apps; -wappalyzer.categories = json.categories; - const driver = { quiet: true, analyze: url => { + const wappalyzer = new Wappalyzer(); + + wappalyzer.apps = json.apps; + wappalyzer.categories = json.categories; + return new Promise((resolve, reject) => { wappalyzer.driver.log = (message, source, type) => { if ( type === 'error' ) { diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index 891db4dec..9f9e36f00 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -2,7 +2,7 @@ "name": "wappalyzer", "description": "Uncovers the technologies used on websites", "homepage": "https://github.com/AliasIO/Wappalyzer", - "version": "5.0.5", + "version": "5.1.0", "author": "Elbert Alias", "license": "GPL-3.0", "repository": { diff --git a/src/drivers/webextension/css/popup.css b/src/drivers/webextension/css/popup.css index 015a333bd..98c56c2de 100644 --- a/src/drivers/webextension/css/popup.css +++ b/src/drivers/webextension/css/popup.css @@ -1,31 +1,95 @@ body { background: #fff; - color: #4a4a4a; font-family: Helvetica, Arial, sans-serif; - font-size: 13px; - line-height: 16px; + font-size: .8rem; + height: 20.8rem; margin: 0; - min-width: 200px; - overflow-x: hidden; - padding: 15px; + overflow: hidden; + width: 30rem; } -a { - color: #4a4a4a; +.header { + align-items: center; + background: linear-gradient(160deg, #32067c, #150233); + height: 4rem; + display: flex; } -a:focus { - outline: 0; +.header__link:focus { + outline: none; } -img { +.header__logo { + display: inline-block; + margin: .2rem 0 0 1.5rem; + -webkit-backface-visibility: hidden; + -webkit-transform: translateZ(0) scale(1.0, 1.0); + transform: translateZ(0); + height: 2rem; +} + +.container { + height: 15.8rem; + overflow: scroll; + padding: 1rem 1.5rem 0rem 1.5rem; +} + +.detected { + columns: 2; + column-gap: 1.5rem; + line-height: 1.4rem; +} + +.detected__category { + break-inside: avoid-column; + padding-bottom: 1rem; +} + +.detected__category-link { + border-bottom: 1px solid #dbdbdb; + display: block; + margin-bottom: .5rem; + text-decoration: none; +} + +.detected__category-name { + color: #4608ad; + display: block; + font-weight: bold; + line-height: 2rem; +} + +.detected__category-link:hover .detected__category-name { + color: #4a4a4a; +} + +.detected__app { + display: block; + line-height: 1.7rem; + text-decoration: none; +} + +.detected__app:focus { + display: block; + outline: 0; +} + +.detected__app-icon { display: inline-block; height: 16px; - margin-right: 8px; - vertical-align: top; + margin-right: .5rem; + vertical-align: -.2rem; width: 16px; } +.detected__app-name { + color: #4a4a4a; +} + +.detected__app:hover .detected__app-name { + border-bottom: 1px solid #4a4a4a; +} + .detected-app { padding: 7px 0; } @@ -39,40 +103,12 @@ img { padding-bottom: 0; } - .detected-app a { - color: #4608ad; - display: block; - text-decoration: none; - } - - .detected-app a .label .name { - border-bottom: 1px solid transparent; - } - - .detected-app a:hover .label .name { - border-bottom: 1px solid #4608ad; - } - - .detected-app a .category .name { - color: #4a4a4a; - border-bottom: 1px solid transparent; - } - - .detected-app a:hover .category .name { - border-bottom: 1px solid #4a4a4a; - } - -.label { - font-weight: bold; -} - -.category { - display: block; - margin: 5px 0 0 24px; +.empty { + display: flex; + height: 16rem; + align-items: center; + justify-content: center; } -.empty { - color: #999; - font-style: italic; - text-align: center; +.empty__text { } diff --git a/src/drivers/webextension/html/popup.html b/src/drivers/webextension/html/popup.html index 2f4e90ad2..1f17595cf 100644 --- a/src/drivers/webextension/html/popup.html +++ b/src/drivers/webextension/html/popup.html @@ -11,5 +11,12 @@ +
+ + + +
+ +
diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index 79b547100..e9c6c97a6 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -3,7 +3,9 @@ */ /** global: browser */ -/** global: wappalyzer */ +/** global: Wappalyzer */ + +const wappalyzer = new Wappalyzer(); var tabCache = {}; var headersCache = {}; @@ -26,7 +28,7 @@ function getOption(name, defaultValue) { // Chrome, Firefox browser.storage.local.get(name) .then(callback) - .catch(console.error); + .catch(error => wappalyzer.log(error, 'driver', 'error')); } catch ( e ) { // Edge browser.storage.local.get(name, callback); @@ -123,7 +125,7 @@ var callback = tabs => { try { browser.tabs.query({}) .then(callback) - .catch(console.error); + .catch(error => wappalyzer.log(error, 'driver', 'error')); } catch ( e ) { browser.tabs.query({}, callback); } @@ -279,10 +281,10 @@ wappalyzer.driver.getRobotsTxt = (host, secure = false) => { fetch('http' + ( secure ? 's' : '' ) + '://' + host + '/robots.txt') .then(response => { if ( !response.ok ) { - if (response.status === 404) { - return ''; + if ( response.status === 404 ) { + return ''; } else { - throw 'GET ' + response.url + ' was not ok'; + throw 'GET ' + response.url + ' was not ok'; } } diff --git a/src/drivers/webextension/js/iframe.js b/src/drivers/webextension/js/iframe.js index ea3e2a990..0dd339d58 100644 --- a/src/drivers/webextension/js/iframe.js +++ b/src/drivers/webextension/js/iframe.js @@ -181,7 +181,7 @@ var exports = {}; video_assets: opt_video_assets, assets: opt_assets, version: '3', - mrev: '082d7cb-d', + mrev: '4d79384-d', msgNum: this.msgNum, timestamp: new Date().getTime(), pageVis: document.visibilityState, @@ -890,7 +890,7 @@ var exports = {}; var _pageTags; var INIT_MS_BW_SEARCHES = 2000; var PAGE_TAG_RE = new RegExp('gpt|oascentral'); - var POST_MSG_ID = '1501281986-4236-27733-5465-12184'; + var POST_MSG_ID = '1503096304-372-12333-31563-11152'; var AD_SERVER_RE = new RegExp('^(google_ads_iframe|oas_frame|atwAdFrame)'); function getPageTags(doc) { @@ -1104,6 +1104,23 @@ var exports = {}; } }, + blockedRobotsMsgGen: function(sendFcn, origUrl) { + + if ( origUrl.indexOf('google.com/_/chrome/newtab') === -1 ) { + var onBlockedRobotsMessage = function() { + var log; + log = _logGen.log('invalid-robotstxt', []); + log.doc.finalPageUrl = log.doc.url; + log.doc.url = exports.utils.normalizeUrl(origUrl); + + sendFcn(log); + }; + return onBlockedRobotsMessage; + } else { + return function() {}; + } + }, + init: function(onAdFound) { if ( exports.utils.SCRIPT_IN_FRIENDLY_IFRAME ) { @@ -1137,6 +1154,8 @@ if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { init: exports.coordinator.init, addPostMessageListener: exports.coordinator.addPostMessageListener, askIfTrackingEnabled: exports.utils.askIfTrackingEnabled, + blockedRobotsMsgGen: exports.coordinator.blockedRobotsMsgGen, + inWindowTop: exports.utils.SCRIPT_IN_WINDOW_TOP, sendToBackground: exports.utils.sendToBackground }; } else { @@ -1149,18 +1168,18 @@ if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) { ); } })(window); -(function(adparser) { +(function(adparser, pageUrl) { function onAdFound(log) { adparser.sendToBackground({ id: 'ad_log', subject: log }, 'ad_log', '', function(){}); } - if ( window === window.top ) { + if ( adparser && adparser.inWindowTop ) { adparser.addPostMessageListener(); adparser.askIfTrackingEnabled( function() { adparser.init(onAdFound); }, - function() {} + adparser.blockedRobotsMsgGen(onAdFound, pageUrl) ) } -})(window.adparser); +})(window.adparser, window.location.href); diff --git a/src/drivers/webextension/js/network.js b/src/drivers/webextension/js/network.js index 8a0421b88..d5bc4458c 100644 --- a/src/drivers/webextension/js/network.js +++ b/src/drivers/webextension/js/network.js @@ -66,6 +66,7 @@ 'washingtonpost.com' ]; + var robotsTxtAllows = wappalyzer.robotsTxtAllows; if ( !String.prototype.endsWith ) { String.prototype.endsWith = function(searchString, position) { var subjectString = this.toString(); @@ -114,10 +115,10 @@ } } - function ifTrackingEnabled(url, ifCallback, elseCallback) { + function ifTrackingEnabled(details, ifCallback, elseCallback) { var fullIfCallback = function() { - allowedByRobotsTxt(url, ifCallback, elseCallback); + allowedByRobotsTxt(details, ifCallback, elseCallback); }; browser.storage.local.get('tracking').then(function(item) { @@ -135,9 +136,9 @@ } - function allowedByRobotsTxt(url, ifCallback, elseCallback) { - if ( ! url.startsWith('chrome://') ) { - wappalyzer.robotsTxtAllows(url).then(ifCallback, elseCallback); + function allowedByRobotsTxt(details, ifCallback, elseCallback) { + if ( details.url && !details.url.startsWith('chrome://') ) { + robotsTxtAllows(details.url).then(ifCallback, elseCallback); } else { elseCallback(); } @@ -219,7 +220,7 @@ this.cleanupCollector(tabId); ifTrackingEnabled( - details.url, + details, function() { if ( !areListenersRegistered ) { @@ -279,18 +280,20 @@ browserProxy.tabs.sendMessage(this.tabId, message); }; - PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, isValidAd) { + PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, nonAdTrackingEvent) { var msg = {}; msg.assets = []; + msg.requests = []; msg.event_data = {}; - if ( isValidAd ) { + if ( !nonAdTrackingEvent ) { msg.event = 'new-video-ad'; msg.requests = reqs; msg.requests.sort(function(reqA, reqB) {return reqA.requestTimestamp - reqB.requestTimestamp;}); if ( assetReq ) { msg.assets = [assetReq]; } - } else { + } else if ( nonAdTrackingEvent === 'new-invalid-video-ad' ) { + msg.event = nonAdTrackingEvent; msg.requests = reqs.map(function(request) { return parseHostnameFromUrl(request.url); }); @@ -301,7 +304,8 @@ contentType: assetReq.contentType, size: assetReq.size }]; - msg.event = 'new-invalid-video-ad'; + } else if ( nonAdTrackingEvent === 'robots-txt-no-scraping' ) { + msg.event = nonAdTrackingEvent; } msg.origUrl = curPageUrl; msg.displayAdFound = this.displayAdFound; @@ -805,7 +809,7 @@ browserProxy.runtime.onMessage.addListener(function(request, sender, sendResponse) { if ( request === 'is_tracking_enabled' ) { ifTrackingEnabled( - sender.tab.url, + sender.tab, function() { try {sendResponse({'tracking_enabled': true});} catch(err) {} }, diff --git a/src/drivers/webextension/js/popup.js b/src/drivers/webextension/js/popup.js index 94be5f258..95184f8ea 100644 --- a/src/drivers/webextension/js/popup.js +++ b/src/drivers/webextension/js/popup.js @@ -32,26 +32,16 @@ function replaceDomWhenReady(dom) { } function replaceDom(domTemplate) { - var body = document.body; + var container = document.getElementsByClassName('container')[0]; - while ( body.firstChild ) { - body.removeChild(body.firstChild); + while ( container.firstChild ) { + container.removeChild(container.firstChild); } - body.appendChild(jsonToDOM(domTemplate, document, {})); + container.appendChild(jsonToDOM(domTemplate, document, {})); var nodes = document.querySelectorAll('[data-i18n]'); - for ( let ms = 200; ms < 500; ms += 50 ) { - setTimeout(() => { - let div = document.createElement('div'); - - div.style.display = 'none'; - - body.appendChild(div); - }, ms); - }; - Array.prototype.forEach.call(nodes, node => { node.childNodes[0].nodeValue = browser.i18n.getMessage(node.dataset.i18n); }); @@ -63,65 +53,87 @@ function appsToDomTemplate(response) { template = []; if ( response.tabCache && Object.keys(response.tabCache.detected).length > 0 ) { - for ( appName in response.tabCache.detected ) { - confidence = response.tabCache.detected[appName].confidenceTotal; - version = response.tabCache.detected[appName].version; - categories = []; + const categories = {}; + // Group apps by category + for ( appName in response.tabCache.detected ) { response.apps[appName].cats.forEach(cat => { - categories.push( + categories[cat] = categories[cat] || { apps: [] }; + + categories[cat].apps[appName] = appName; + }); + } + + for ( cat in categories ) { + const apps = []; + + for ( appName in categories[cat].apps ) { + confidence = response.tabCache.detected[appName].confidenceTotal; + version = response.tabCache.detected[appName].version; + + apps.push( [ 'a', { + class: 'detected__app', target: '_blank', - href: 'https://wappalyzer.com/categories/' + slugify(response.categories[cat].name) + href: 'https://wappalyzer.com/applications/' + slugify(appName) }, [ + 'img', { + class: 'detected__app-icon', + src: '../images/icons/' + ( response.apps[appName].icon || 'default.svg' ) + }, + ], [ 'span', { - class: 'category' - }, [ - 'span', { - class: 'name' - }, - browser.i18n.getMessage('categoryName' + cat) - ] + class: 'detected__app-name' + }, + appName + ( version ? ' ' + version : '' ) + ( confidence < 100 ? ' (' + confidence + '% sure)' : '' ) ] ] ); - }); + } template.push( [ 'div', { - class: 'detected-app' + class: 'detected__category' }, [ 'a', { + class: 'detected__category-link', target: '_blank', - href: 'https://wappalyzer.com/applications/' + slugify(appName) + href: 'https://wappalyzer.com/categories/' + slugify(response.categories[cat].name) }, [ - 'img', { - src: '../images/icons/' + ( response.apps[appName].icon || 'default.svg' ) - } - ], [ 'span', { - class: 'label' - }, [ - 'span', { - class: 'name' - }, - appName - ], - ( version ? ' ' + version : '' ) + ( confidence < 100 ? ' (' + confidence + '% sure)' : '' ) + class: 'detected__category-name' + }, + browser.i18n.getMessage('categoryName' + cat) ] - ], - categories + ], [ + 'div', { + class: 'detected__apps' + }, + apps + ] ] ); } + + template = [ + 'div', { + class: 'detected' + }, + template + ]; } else { template = [ 'div', { class: 'empty' }, - browser.i18n.getMessage('noAppsDetected') + [ + 'span', { + class: 'empty__text' + }, + browser.i18n.getMessage('noAppsDetected') + ], ]; } diff --git a/src/drivers/webextension/manifest.edge.json b/src/drivers/webextension/manifest.edge.json index cc27ef105..f00a16d05 100644 --- a/src/drivers/webextension/manifest.edge.json +++ b/src/drivers/webextension/manifest.edge.json @@ -4,7 +4,7 @@ "author": "Elbert Alias", "homepage_url": "https://wappalyzer.com/", "description": "Identify web technologies", - "version": "5.0.5", + "version": "5.1.0", "default_locale": "en", "manifest_version": 2, "icons": { diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json index 2c28dd9fe..0d0c48123 100644 --- a/src/drivers/webextension/manifest.json +++ b/src/drivers/webextension/manifest.json @@ -4,7 +4,7 @@ "author": "Elbert Alias", "homepage_url": "https://wappalyzer.com/", "description": "Identify web technologies", - "version": "5.0.5", + "version": "5.1.0", "default_locale": "en", "manifest_version": 2, "icons": { diff --git a/src/icons/Akamai.png b/src/icons/Akamai.png deleted file mode 100644 index fa2be98a2..000000000 Binary files a/src/icons/Akamai.png and /dev/null differ diff --git a/src/icons/RockRMS.svg b/src/icons/RockRMS.svg index 6b352f6ca..b66f8e881 100644 --- a/src/icons/RockRMS.svg +++ b/src/icons/RockRMS.svg @@ -1 +1,4 @@ -rock-logo \ No newline at end of file + diff --git a/src/icons/akamai.svg b/src/icons/akamai.svg new file mode 100644 index 000000000..ab4168ba7 --- /dev/null +++ b/src/icons/akamai.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/icons/bizweb.png b/src/icons/bizweb.png new file mode 100644 index 000000000..fbe062bb2 Binary files /dev/null and b/src/icons/bizweb.png differ diff --git a/src/icons/moon.png b/src/icons/moon.png deleted file mode 100644 index eb8ff3afd..000000000 Binary files a/src/icons/moon.png and /dev/null differ diff --git a/src/icons/moon.svg b/src/icons/moon.svg new file mode 100644 index 000000000..1a7e75325 --- /dev/null +++ b/src/icons/moon.svg @@ -0,0 +1 @@ +image/svg+xmlMoon LogoMoon Logo diff --git a/src/icons/zeit.svg b/src/icons/zeit.svg new file mode 100644 index 000000000..8ce0bfb63 --- /dev/null +++ b/src/icons/zeit.svg @@ -0,0 +1 @@ +ZeitCreated with Sketch. \ No newline at end of file diff --git a/src/wappalyzer.js b/src/wappalyzer.js index 94b950c28..093f1b0bb 100644 --- a/src/wappalyzer.js +++ b/src/wappalyzer.js @@ -13,517 +13,523 @@ const validation = { hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/ }; -var wappalyzer = { - apps: {}, - categories: {}, - driver: {} -}; +class Wappalyzer { + constructor() { + this.apps = {}; + this.categories = {}; + this.driver = {}; + + this.detected = {}; + this.hostnameCache = {}; + this.adCache = []; + + this.config = { + websiteURL: 'https://wappalyzer.com/', + twitterURL: 'https://twitter.com/Wappalyzer', + githubURL: 'https://github.com/AliasIO/Wappalyzer', + }; + } -var detected = {}; -var hostnameCache = {}; -var adCache = []; + /** + * Log messages to console + */ + log(message, source, type) { + this.driver.log(message, source || '', type || 'debug'); + } -wappalyzer.config = { - websiteURL: 'https://wappalyzer.com/', - twitterURL: 'https://twitter.com/Wappalyzer', - githubURL: 'https://github.com/AliasIO/Wappalyzer', -}; + analyze(hostname, url, data, context) { + var apps = {}; -/** - * Log messages to console - */ -wappalyzer.log = (message, source, type) => { - wappalyzer.driver.log(message, source || '', type || 'debug'); -}; + // Remove hash from URL + data.url = url = url.split('#')[0]; -wappalyzer.analyze = (hostname, url, data, context) => { - var apps = {}; - - // Remove hash from URL - data.url = url = url.split('#')[0]; + if ( typeof data.html !== 'string' ) { + data.html = ''; + } - if ( typeof data.html !== 'string' ) { - data.html = ''; - } + if ( this.detected[url] === undefined ) { + this.detected[url] = {}; + } - if ( detected[url] === undefined ) { - detected[url] = {}; - } + Object.keys(this.apps).forEach(appName => { + apps[appName] = this.detected[url] && this.detected[url][appName] ? this.detected[url][appName] : new Application(appName, this.apps[appName]); - Object.keys(wappalyzer.apps).forEach(appName => { - apps[appName] = detected[url] && detected[url][appName] ? detected[url][appName] : new Application(appName, wappalyzer.apps[appName]); + var app = apps[appName]; - var app = apps[appName]; + if ( url ) { + this.analyzeUrl(app, url); + } - if ( url ) { - analyzeUrl(app, url); - } + if ( data.html ) { + this.analyzeHtml(app, data.html); + this.analyzeScript(app, data.html); + this.analyzeMeta(app, data.html); + } - if ( data.html ) { - analyzeHtml(app, data.html); - analyzeScript(app, data.html); - analyzeMeta(app, data.html); - } + if ( data.headers ) { + this.analyzeHeaders(app, data.headers); + } - if ( data.headers ) { - analyzeHeaders(app, data.headers); - } + if ( data.env ) { + this.analyzeEnv(app, data.env); + } - if ( data.env ) { - analyzeEnv(app, data.env); - } + if ( data.robotsTxt ) { + this.analyzeRobotsTxt(app, data.robotsTxt); + } + }) - if ( data.robotsTxt ) { - analyzeRobotsTxt(app, data.robotsTxt); - } - }) + Object.keys(apps).forEach(appName => { + var app = apps[appName]; - Object.keys(apps).forEach(appName => { - var app = apps[appName]; + if ( !app.detected || !app.getConfidence() ) { + delete apps[app.name]; + } + }); - if ( !app.detected || !app.getConfidence() ) { - delete apps[app.name]; - } - }); + this.resolveExcludes(apps); + this.resolveImplies(apps, url); - resolveExcludes(apps); - resolveImplies(apps, url); + this.cacheDetectedApps(apps, url); + this.trackDetectedApps(apps, url, hostname, data.html); - cacheDetectedApps(apps, url); - trackDetectedApps(apps, url, hostname, data.html); + if ( Object.keys(apps).length ) { + this.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url, 'core'); + } - if ( Object.keys(apps).length ) { - wappalyzer.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url, 'core'); + this.driver.displayApps(this.detected[url], context); } - wappalyzer.driver.displayApps(detected[url], context); -} + /** + * Cache detected ads + */ + cacheDetectedAds(ad) { + this.adCache.push(ad); + } -/** - * Cache detected ads - */ -wappalyzer.cacheDetectedAds = ad => { - adCache.push(ad); -} + /** + * + */ + robotsTxtAllows(url) { + return new Promise((resolve, reject) => { + var parsed = this.parseUrl(url); + + this.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:') + .then(robotsTxt => { + robotsTxt.forEach(disallow => { + if ( parsed.pathname.indexOf(disallow) === 0 ) { + reject(); + } + }); -/** - * - */ -wappalyzer.robotsTxtAllows = url => { - return new Promise((resolve, reject) => { - var parsed = wappalyzer.parseUrl(url); - - wappalyzer.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:') - .then(robotsTxt => { - robotsTxt.forEach(disallow => { - if ( parsed.pathname.indexOf(disallow) === 0 ) { - reject(); - } + resolve(); }); + }); + }; - resolve(); - }); - }); -}; - -/** - * Parse a URL - */ -wappalyzer.parseUrl = url => { - var a = wappalyzer.driver.document.createElement('a'); + /** + * Parse a URL + */ + parseUrl(url) { + var a = this.driver.document.createElement('a'); - a.href = url; + a.href = url; - a.canonical = a.protocol + '//' + a.host + a.pathname; + a.canonical = a.protocol + '//' + a.host + a.pathname; - return a; -} + return a; + } -/** - * - */ -wappalyzer.parseRobotsTxt = robotsTxt => { - var userAgent; - var disallow = []; + /** + * + */ + parseRobotsTxt(robotsTxt) { + var userAgent; + var disallow = []; - robotsTxt.split('\n').forEach(line => { - var matches = /^User-agent:\s*(.+)$/i.exec(line); + robotsTxt.split('\n').forEach(line => { + var matches = /^User-agent:\s*(.+)$/i.exec(line); - if ( matches ) { - userAgent = matches[1].toLowerCase(); - } else { - if ( userAgent === '*' || userAgent === 'wappalyzer' ) { - matches = /^Disallow:\s*(.+)$/i.exec(line); + if ( matches ) { + userAgent = matches[1].toLowerCase(); + } else { + if ( userAgent === '*' || userAgent === 'wappalyzer' ) { + matches = /^Disallow:\s*(.+)$/i.exec(line); - if ( matches ) { - disallow.push(matches[1]); + if ( matches ) { + disallow.push(matches[1]); + } } } - } - }); - - return disallow; -} - -/** - * - */ -wappalyzer.ping = () => { - if ( Object.keys(hostnameCache).length >= 50 || adCache.length >= 50 ) { - wappalyzer.driver.ping(hostnameCache, adCache); + }); - hostnameCache = {}; - adCache = []; + return disallow; } -} -/** - * Enclose string in array - */ -function asArray(value) { - return typeof value === 'string' ? [ value ] : value; -} - -/** - * Parse apps.json patterns - */ -function parsePatterns(patterns) { - var parsed = {}; + /** + * + */ + ping() { + if ( Object.keys(this.hostnameCache).length >= 50 || this.adCache.length >= 50 ) { + this.driver.ping(this.hostnameCache, this.adCache); - // Convert string to object containing array containing string - if ( typeof patterns === 'string' || patterns instanceof Array ) { - patterns = { - main: asArray(patterns) - }; + this.hostnameCache = {}; + this.adCache = []; + } } - for ( var key in patterns ) { - parsed[key] = []; - - asArray(patterns[key]).forEach(pattern => { - var attrs = {}; - - pattern.split('\\;').forEach((attr, i) => { - if ( i ) { - // Key value pairs - attr = attr.split(':'); - - if ( attr.length > 1 ) { - attrs[attr.shift()] = attr.join(':'); - } - } else { - attrs.string = attr; + /** + * Enclose string in array + */ + asArray(value) { + return typeof value === 'string' ? [ value ] : value; + } - try { - attrs.regex = new RegExp(attr.replace('/', '\/'), 'i'); // Escape slashes in regular expression - } catch (e) { - attrs.regex = new RegExp(); + /** + * Parse apps.json patterns + */ + parsePatterns(patterns) { + var parsed = {}; + + // Convert string to object containing array containing string + if ( typeof patterns === 'string' || patterns instanceof Array ) { + patterns = { + main: this.asArray(patterns) + }; + } - wappalyzer.log(e + ': ' + attr, 'error', 'core'); - } - } - }); + for ( var key in patterns ) { + parsed[key] = []; - parsed[key].push(attrs); - }); - } + this.asArray(patterns[key]).forEach(pattern => { + var attrs = {}; - // Convert back to array if the original pattern list was an array (or string) - if ( 'main' in parsed ) { - parsed = parsed.main; - } + pattern.split('\\;').forEach((attr, i) => { + if ( i ) { + // Key value pairs + attr = attr.split(':'); - return parsed; -} + if ( attr.length > 1 ) { + attrs[attr.shift()] = attr.join(':'); + } + } else { + attrs.string = attr; -function resolveExcludes(apps) { - var excludes = []; + try { + attrs.regex = new RegExp(attr.replace('/', '\/'), 'i'); // Escape slashes in regular expression + } catch (e) { + attrs.regex = new RegExp(); - // Exclude app in detected apps only - Object.keys(apps).forEach(appName => { - var app = apps[appName]; + this.log(e + ': ' + attr, 'error', 'core'); + } + } + }); - if ( app.props.excludes ) { - asArray(app.props.excludes).forEach(excluded => { - excludes.push(excluded); + parsed[key].push(attrs); }); } - }) - // Remove excluded applications - Object.keys(apps).forEach(appName => { - if ( excludes.indexOf(appName) !== -1 ) { - delete apps[appName]; + // Convert back to array if the original pattern list was an array (or string) + if ( 'main' in parsed ) { + parsed = parsed.main; } - }) -} -function resolveImplies(apps, url) { - var checkImplies = true; + return parsed; + } - // Implied applications - // Run several passes as implied apps may imply other apps - while ( checkImplies ) { - checkImplies = false; + resolveExcludes(apps) { + var excludes = []; + // Exclude app in detected apps only Object.keys(apps).forEach(appName => { var app = apps[appName]; - if ( app && app.implies ) { - asArray(app.props.implies).forEach(implied => { - implied = parsePatterns(implied)[0]; - - if ( !wappalyzer.apps[implied.string] ) { - wappalyzer.log('Implied application ' + implied.string + ' does not exist', 'core', 'warn'); - - return; - } - - if ( !( implied.string in apps ) ) { - apps[implied.string] = detected[url] && detected[url][implied.string] ? detected[url][implied.string] : new Application(implied.string, true); - - checkImplies = true; - } - - // Apply app confidence to implied app - Object.keys(app.confidence).forEach(id => { - apps[implied.string].confidence[id + ' implied by ' + appName] = app.confidence[id] * ( implied.confidence ? implied.confidence / 100 : 1 ); - }); + if ( app.props.excludes ) { + this.asArray(app.props.excludes).forEach(excluded => { + excludes.push(excluded); }); } - }); + }) + + // Remove excluded applications + Object.keys(apps).forEach(appName => { + if ( excludes.indexOf(appName) !== -1 ) { + delete apps[appName]; + } + }) } -} -/** - * Cache detected applications - */ -function cacheDetectedApps(apps, url) { - if (!wappalyzer.driver.ping instanceof Function) return; + resolveImplies(apps, url) { + var checkImplies = true; - Object.keys(apps).forEach(appName => { - var app = apps[appName]; + // Implied applications + // Run several passes as implied apps may imply other apps + while ( checkImplies ) { + checkImplies = false; - // Per URL - detected[url][appName] = app; + Object.keys(apps).forEach(appName => { + var app = apps[appName]; - Object.keys(app.confidence).forEach(id => { - detected[url][appName].confidence[id] = app.confidence[id]; - }); - }) + if ( app && app.implies ) { + this.asArray(app.props.implies).forEach(implied => { + implied = this.parsePatterns(implied)[0]; - wappalyzer.ping(); -} + if ( !this.apps[implied.string] ) { + this.log('Implied application ' + implied.string + ' does not exist', 'core', 'warn'); -/** - * Track detected applications - */ -function trackDetectedApps(apps, url, hostname, html) { - if (!wappalyzer.driver.ping instanceof Function) return; - - Object.keys(apps).forEach(appName => { - var app = apps[appName]; - - if ( detected[url][appName].getConfidence() >= 100 ) { - if ( validation.hostname.test(hostname) && !validation.hostnameBlacklist.test(url) ) { - wappalyzer.robotsTxtAllows(url) - .then(() => { - if ( !( hostname in hostnameCache ) ) { - hostnameCache[hostname] = { - applications: {}, - meta: {} - }; + return; } - if ( !( appName in hostnameCache[hostname].applications ) ) { - hostnameCache[hostname].applications[appName] = { - hits: 0 - }; + if ( !( implied.string in apps ) ) { + apps[implied.string] = this.detected[url] && this.detected[url][implied.string] ? this.detected[url][implied.string] : new Application(implied.string, true); + + checkImplies = true; } - hostnameCache[hostname].applications[appName].hits ++; + // Apply app confidence to implied app + Object.keys(app.confidence).forEach(id => { + apps[implied.string].confidence[id + ' implied by ' + appName] = app.confidence[id] * ( implied.confidence ? implied.confidence / 100 : 1 ); + }); + }); + } + }); + } + } - if ( apps[appName].version ) { - hostnameCache[hostname].applications[appName].version = app.version; - } - }) - .catch(() => console.log('Disallowed in robots.txt: ' + url)) - } + /** + * Cache detected applications + */ + cacheDetectedApps(apps, url) { + if ( !( this.driver.ping instanceof Function ) ) { + return; } - }); - // Additional information - if ( hostname in hostnameCache ) { - var match = html.match(/]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"/i); + Object.keys(apps).forEach(appName => { + var app = apps[appName]; - if ( match && match.length ) { - hostnameCache[hostname].meta['language'] = match[1]; - } + // Per URL + this.detected[url][appName] = app; + + Object.keys(app.confidence).forEach(id => { + this.detected[url][appName].confidence[id] = app.confidence[id]; + }); + }) + + this.ping(); } - wappalyzer.ping(); -} + /** + * Track detected applications + */ + trackDetectedApps(apps, url, hostname, html) { + if ( !( this.driver.ping instanceof Function ) ) { + return; + } -/** - * Analyze URL - */ -function analyzeUrl(app, url) { - var patterns = parsePatterns(app.props.url); + Object.keys(apps).forEach(appName => { + var app = apps[appName]; - if ( patterns.length ) { - patterns.forEach(pattern => { - if ( pattern.regex.test(url) ) { - addDetected(app, pattern, 'url', url); + if ( this.detected[url][appName].getConfidence() >= 100 ) { + if ( validation.hostname.test(hostname) && !validation.hostnameBlacklist.test(url) ) { + this.robotsTxtAllows(url) + .then(() => { + if ( !( hostname in this.hostnameCache ) ) { + this.hostnameCache[hostname] = { + applications: {}, + meta: {} + }; + } + + if ( !( appName in this.hostnameCache[hostname].applications ) ) { + this.hostnameCache[hostname].applications[appName] = { + hits: 0 + }; + } + + this.hostnameCache[hostname].applications[appName].hits ++; + + if ( apps[appName].version ) { + this.hostnameCache[hostname].applications[appName].version = app.version; + } + }) + .catch(() => console.log('Disallowed in robots.txt: ' + url)) + } } }); - } -} -/** - * Analyze HTML - */ -function analyzeHtml(app, html) { - var patterns = parsePatterns(app.props.html); + // Additional information + if ( hostname in this.hostnameCache ) { + var match = html.match(/]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"/i); - if ( patterns.length ) { - patterns.forEach(pattern => { - if ( pattern.regex.test(html) ) { - addDetected(app, pattern, 'html', html); + if ( match && match.length ) { + this.hostnameCache[hostname].meta['language'] = match[1]; } - }); + } + + this.ping(); } -} -/** - * Analyze script tag - */ -function analyzeScript(app, html) { - var regex = new RegExp(']+src=("|\')([^"\']+)', 'ig'); - var patterns = parsePatterns(app.props.script); + /** + * Analyze URL + */ + analyzeUrl(app, url) { + var patterns = this.parsePatterns(app.props.url); - if ( patterns.length ) { - patterns.forEach(pattern => { - var match; + if ( patterns.length ) { + patterns.forEach(pattern => { + if ( pattern.regex.test(url) ) { + this.addDetected(app, pattern, 'url', url); + } + }); + } + } + + /** + * Analyze HTML + */ + analyzeHtml(app, html) { + var patterns = this.parsePatterns(app.props.html); - while ( ( match = regex.exec(html) ) ) { - if ( pattern.regex.test(match[2]) ) { - addDetected(app, pattern, 'script', match[2]); + if ( patterns.length ) { + patterns.forEach(pattern => { + if ( pattern.regex.test(html) ) { + this.addDetected(app, pattern, 'html', html); } - } - }); + }); + } } -} -/** - * Analyze meta tag - */ -function analyzeMeta(app, html) { - var regex = /]+>/ig; - var patterns = parsePatterns(app.props.meta); - var content; - var match; - - while ( patterns && ( match = regex.exec(html) ) ) { - for ( var meta in patterns ) { - if ( new RegExp('(name|property)=["\']' + meta + '["\']', 'i').test(match) ) { - content = match.toString().match(/content=("|')([^"']+)("|')/i); - - patterns[meta].forEach(pattern => { - if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { - addDetected(app, pattern, 'meta', content[2], meta); + /** + * Analyze script tag + */ + analyzeScript(app, html) { + var regex = new RegExp(']+src=("|\')([^"\']+)', 'ig'); + var patterns = this.parsePatterns(app.props.script); + + if ( patterns.length ) { + patterns.forEach(pattern => { + var match; + + while ( ( match = regex.exec(html) ) ) { + if ( pattern.regex.test(match[2]) ) { + this.addDetected(app, pattern, 'script', match[2]); } - }); + } + }); + } + } + + /** + * Analyze meta tag + */ + analyzeMeta(app, html) { + var regex = /]+>/ig; + var patterns = this.parsePatterns(app.props.meta); + var content; + var match; + + while ( patterns && ( match = regex.exec(html) ) ) { + for ( var meta in patterns ) { + if ( new RegExp('(name|property)=["\']' + meta + '["\']', 'i').test(match) ) { + content = match.toString().match(/content=("|')([^"']+)("|')/i); + + patterns[meta].forEach(pattern => { + if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { + this.addDetected(app, pattern, 'meta', content[2], meta); + } + }); + } } } } -} -/** - * analyze response headers - */ -function analyzeHeaders(app, headers) { - var patterns = parsePatterns(app.props.headers); + /** + * analyze response headers + */ + analyzeHeaders(app, headers) { + var patterns = this.parsePatterns(app.props.headers); - if ( headers ) { - Object.keys(patterns).forEach(header => { - patterns[header].forEach(pattern => { - header = header.toLowerCase(); + if ( headers ) { + Object.keys(patterns).forEach(header => { + patterns[header].forEach(pattern => { + header = header.toLowerCase(); - if ( header in headers && pattern.regex.test(headers[header]) ) { - addDetected(app, pattern, 'headers', headers[header], header); - } + if ( header in headers && pattern.regex.test(headers[header]) ) { + this.addDetected(app, pattern, 'headers', headers[header], header); + } + }); }); - }); + } } -} -/** - * Analyze environment variables - */ -function analyzeEnv(app, envs) { - var patterns = parsePatterns(app.props.env); - - if ( patterns.length ) { - patterns.forEach(pattern => { - Object.keys(envs).forEach(env => { - if ( pattern.regex.test(envs[env]) ) { - addDetected(app, pattern, 'env', envs[env]); - } - }) - }); + /** + * Analyze environment variables + */ + analyzeEnv(app, envs) { + var patterns = this.parsePatterns(app.props.env); + + if ( patterns.length ) { + patterns.forEach(pattern => { + Object.keys(envs).forEach(env => { + if ( pattern.regex.test(envs[env]) ) { + this.addDetected(app, pattern, 'env', envs[env]); + } + }) + }); + } } -} -/** - * Analyze robots.txt - */ -function analyzeRobotsTxt(app, robotsTxt) { - var patterns = parsePatterns(app.props.robotsTxt); + /** + * Analyze robots.txt + */ + analyzeRobotsTxt(app, robotsTxt) { + var patterns = this.parsePatterns(app.props.robotsTxt); - if ( patterns.length ) { - patterns.forEach(pattern => { - if ( pattern.regex.test(robotsTxt) ) { - addDetected(app, pattern, 'robotsTxt', robotsTxt); - } - }); + if ( patterns.length ) { + patterns.forEach(pattern => { + if ( pattern.regex.test(robotsTxt) ) { + this.addDetected(app, pattern, 'robotsTxt', robotsTxt); + } + }); + } } -} -/** - * Mark application as detected, set confidence and version - */ -function addDetected(app, pattern, type, value, key) { - app.detected = true; + /** + * Mark application as detected, set confidence and version + */ + addDetected(app, pattern, type, value, key) { + app.detected = true; - // Set confidence level - app.confidence[type + ' ' + ( key ? key + ' ' : '' ) + pattern.regex] = pattern.confidence || 100; + // Set confidence level + app.confidence[type + ' ' + ( key ? key + ' ' : '' ) + pattern.regex] = pattern.confidence || 100; - // Detect version number - if ( pattern.version ) { - var versions = []; - var version = pattern.version; - var matches = pattern.regex.exec(value); + // Detect version number + if ( pattern.version ) { + var versions = []; + var version = pattern.version; + var matches = pattern.regex.exec(value); - if ( matches ) { - matches.forEach((match, i) => { - // Parse ternary operator - var ternary = new RegExp('\\\\' + i + '\\?([^:]+):(.*)$').exec(version); + if ( matches ) { + matches.forEach((match, i) => { + // Parse ternary operator + var ternary = new RegExp('\\\\' + i + '\\?([^:]+):(.*)$').exec(version); - if ( ternary && ternary.length === 3 ) { - version = version.replace(ternary[0], match ? ternary[1] : ternary[2]); - } + if ( ternary && ternary.length === 3 ) { + version = version.replace(ternary[0], match ? ternary[1] : ternary[2]); + } - // Replace back references - version = version.replace(new RegExp('\\\\' + i, 'g'), match || ''); - }); + // Replace back references + version = version.replace(new RegExp('\\\\' + i, 'g'), match || ''); + }); - if ( version && versions.indexOf(version) === -1 ) { - versions.push(version); - } + if ( version && versions.indexOf(version) === -1 ) { + versions.push(version); + } - if ( versions.length ) { - // Use the longest detected version number - app.version = versions.reduce((a, b) => a.length > b.length ? a : b); + if ( versions.length ) { + // Use the longest detected version number + app.version = versions.reduce((a, b) => a.length > b.length ? a : b); + } } } } @@ -558,5 +564,5 @@ class Application { } if ( typeof module === 'object' ) { - module.exports = wappalyzer; + module.exports = Wappalyzer; }