Migrations
/
wappalyzer
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change wappalyzer object to Wappalyzer class to allow running of multiple instances

Browse Source
main
Elbert Alias 8 years ago
parent f0654e5bc0
commit 3057e396d9
6 changed files with 417 additions and 409 deletions
Show Stats Download Patch File Download Diff File

12
src/drivers/npm/driver.js
Unescape View File

@ -1,19 +1,21 @@
'use strict'; 'use strict';
const wappalyzer = require('./wappalyzer'); const Wappalyzer = require('./wappalyzer');
const request = require('request'); const request = require('request');
const fs = require('fs'); const fs = require('fs');
const Browser = require('zombie'); const Browser = require('zombie');
const json = JSON.parse(fs.readFileSync(__dirname + '/apps.json')); const json = JSON.parse(fs.readFileSync(__dirname + '/apps.json'));
wappalyzer.apps = json.apps;
wappalyzer.categories = json.categories;
const driver = { const driver = {
quiet: true, quiet: true,
analyze: url => { analyze: url => {
const wappalyzer = new Wappalyzer();
wappalyzer.apps = json.apps;
wappalyzer.categories = json.categories;
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
wappalyzer.driver.log = (message, source, type) => { wappalyzer.driver.log = (message, source, type) => {
if ( type === 'error' ) { if ( type === 'error' ) {
@ -51,6 +53,8 @@ const driver = {
}); });
}); });
console.log('resolve ' + url);
resolve(apps); resolve(apps);
}; };

2
src/drivers/npm/package.json
Unescape View File

@ -2,7 +2,7 @@
"name": "wappalyzer", "name": "wappalyzer",
"description": "Uncovers the technologies used on websites", "description": "Uncovers the technologies used on websites",
"homepage": "https://github.com/AliasIO/Wappalyzer", "homepage": "https://github.com/AliasIO/Wappalyzer",
"version": "5.0.5", "version": "5.0.7",
"author": "Elbert Alias", "author": "Elbert Alias",
"license": "GPL-3.0", "license": "GPL-3.0",
"repository": { "repository": {

4
src/drivers/webextension/js/driver.js
Unescape View File

@ -3,7 +3,9 @@
*/ */
/** global: browser */ /** global: browser */
/** global: wappalyzer */ /** global: Wappalyzer */
const wappalyzer = new Wappalyzer();
var tabCache = {}; var tabCache = {};
var headersCache = {}; var headersCache = {};

2
src/drivers/webextension/manifest.edge.json
Unescape View File

@ -4,7 +4,7 @@
"author": "Elbert Alias", "author": "Elbert Alias",
"homepage_url": "https://wappalyzer.com/", "homepage_url": "https://wappalyzer.com/",
"description": "Identify web technologies", "description": "Identify web technologies",
"version": "5.0.5", "version": "5.0.7",
"default_locale": "en", "default_locale": "en",
"manifest_version": 2, "manifest_version": 2,
"icons": { "icons": {

2
src/drivers/webextension/manifest.json
Unescape View File

@ -4,7 +4,7 @@
"author": "Elbert Alias", "author": "Elbert Alias",
"homepage_url": "https://wappalyzer.com/", "homepage_url": "https://wappalyzer.com/",
"description": "Identify web technologies", "description": "Identify web technologies",
"version": "5.0.5", "version": "5.0.7",
"default_locale": "en", "default_locale": "en",
"manifest_version": 2, "manifest_version": 2,
"icons": { "icons": {

272
src/wappalyzer.js
Unescape View File

@ -13,30 +13,31 @@ const validation = {
hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/ hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/
}; };
var wappalyzer = { class Wappalyzer {
apps: {}, constructor() {
categories: {}, this.apps = {};
driver: {} this.categories = {};
}; this.driver = {};
var detected = {}; this.detected = {};
var hostnameCache = {}; this.hostnameCache = {};
var adCache = []; this.adCache = [];
wappalyzer.config = { this.config = {
websiteURL: 'https://wappalyzer.com/', websiteURL: 'https://wappalyzer.com/',
twitterURL: 'https://twitter.com/Wappalyzer', twitterURL: 'https://twitter.com/Wappalyzer',
githubURL: 'https://github.com/AliasIO/Wappalyzer', githubURL: 'https://github.com/AliasIO/Wappalyzer',
}; };
}
/** /**
* Log messages to console * Log messages to console
*/ */
wappalyzer.log = (message, source, type) => { log(message, source, type) {
wappalyzer.driver.log(message, source || '', type || 'debug'); this.driver.log(message, source || '', type || 'debug');
}; }
wappalyzer.analyze = (hostname, url, data, context) => { analyze(hostname, url, data, context) {
var apps = {}; var apps = {};
// Remove hash from URL // Remove hash from URL
@ -46,35 +47,35 @@ wappalyzer.analyze = (hostname, url, data, context) => {
data.html = ''; data.html = '';
} }
if ( detected[url] === undefined ) { if ( this.detected[url] === undefined ) {
detected[url] = {}; this.detected[url] = {};
} }
Object.keys(wappalyzer.apps).forEach(appName => { Object.keys(this.apps).forEach(appName => {
apps[appName] = detected[url] && detected[url][appName] ? detected[url][appName] : new Application(appName, wappalyzer.apps[appName]); apps[appName] = this.detected[url] && this.detected[url][appName] ? this.detected[url][appName] : new Application(appName, this.apps[appName]);
var app = apps[appName]; var app = apps[appName];
if ( url ) { if ( url ) {
analyzeUrl(app, url); this.analyzeUrl(app, url);
} }
if ( data.html ) { if ( data.html ) {
analyzeHtml(app, data.html); this.analyzeHtml(app, data.html);
analyzeScript(app, data.html); this.analyzeScript(app, data.html);
analyzeMeta(app, data.html); this.analyzeMeta(app, data.html);
} }
if ( data.headers ) { if ( data.headers ) {
analyzeHeaders(app, data.headers); this.analyzeHeaders(app, data.headers);
} }
if ( data.env ) { if ( data.env ) {
analyzeEnv(app, data.env); this.analyzeEnv(app, data.env);
} }
if ( data.robotsTxt ) { if ( data.robotsTxt ) {
analyzeRobotsTxt(app, data.robotsTxt); this.analyzeRobotsTxt(app, data.robotsTxt);
} }
}) })
@ -86,34 +87,34 @@ wappalyzer.analyze = (hostname, url, data, context) => {
} }
}); });
resolveExcludes(apps); this.resolveExcludes(apps);
resolveImplies(apps, url); this.resolveImplies(apps, url);
cacheDetectedApps(apps, url); this.cacheDetectedApps(apps, url);
trackDetectedApps(apps, url, hostname, data.html); this.trackDetectedApps(apps, url, hostname, data.html);
if ( Object.keys(apps).length ) { if ( Object.keys(apps).length ) {
wappalyzer.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url, 'core'); this.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url, 'core');
} }
wappalyzer.driver.displayApps(detected[url], context); this.driver.displayApps(this.detected[url], context);
} }
/** /**
* Cache detected ads * Cache detected ads
*/ */
wappalyzer.cacheDetectedAds = ad => { cacheDetectedAds(ad) {
adCache.push(ad); this.adCache.push(ad);
} }
/** /**
* *
*/ */
wappalyzer.robotsTxtAllows = url => { robotsTxtAllows(url) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
var parsed = wappalyzer.parseUrl(url); var parsed = this.parseUrl(url);
wappalyzer.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:') this.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
.then(robotsTxt => { .then(robotsTxt => {
robotsTxt.forEach(disallow => { robotsTxt.forEach(disallow => {
if ( parsed.pathname.indexOf(disallow) === 0 ) { if ( parsed.pathname.indexOf(disallow) === 0 ) {
@ -124,25 +125,25 @@ wappalyzer.robotsTxtAllows = url => {
resolve(); resolve();
}); });
}); });
}; };
/** /**
* Parse a URL * Parse a URL
*/ */
wappalyzer.parseUrl = url => { parseUrl(url) {
var a = wappalyzer.driver.document.createElement('a'); var a = this.driver.document.createElement('a');
a.href = url; a.href = url;
a.canonical = a.protocol + '//' + a.host + a.pathname; a.canonical = a.protocol + '//' + a.host + a.pathname;
return a; return a;
} }
/** /**
* *
*/ */
wappalyzer.parseRobotsTxt = robotsTxt => { parseRobotsTxt(robotsTxt) {
var userAgent; var userAgent;
var disallow = []; var disallow = [];
@ -163,44 +164,44 @@ wappalyzer.parseRobotsTxt = robotsTxt => {
}); });
return disallow; return disallow;
} }
/** /**
* *
*/ */
wappalyzer.ping = () => { ping() {
if ( Object.keys(hostnameCache).length >= 50 || adCache.length >= 50 ) { if ( Object.keys(this.hostnameCache).length >= 50 || this.adCache.length >= 50 ) {
wappalyzer.driver.ping(hostnameCache, adCache); this.driver.ping(this.hostnameCache, this.adCache);
hostnameCache = {}; this.hostnameCache = {};
adCache = []; this.adCache = [];
}
} }
}
/** /**
* Enclose string in array * Enclose string in array
*/ */
function asArray(value) { asArray(value) {
return typeof value === 'string' ? [ value ] : value; return typeof value === 'string' ? [ value ] : value;
} }
/** /**
* Parse apps.json patterns * Parse apps.json patterns
*/ */
function parsePatterns(patterns) { parsePatterns(patterns) {
var parsed = {}; var parsed = {};
// Convert string to object containing array containing string // Convert string to object containing array containing string
if ( typeof patterns === 'string' || patterns instanceof Array ) { if ( typeof patterns === 'string' || patterns instanceof Array ) {
patterns = { patterns = {
main: asArray(patterns) main: this.asArray(patterns)
}; };
} }
for ( var key in patterns ) { for ( var key in patterns ) {
parsed[key] = []; parsed[key] = [];
asArray(patterns[key]).forEach(pattern => { this.asArray(patterns[key]).forEach(pattern => {
var attrs = {}; var attrs = {};
pattern.split('\\;').forEach((attr, i) => { pattern.split('\\;').forEach((attr, i) => {
@ -219,7 +220,7 @@ function parsePatterns(patterns) {
} catch (e) { } catch (e) {
attrs.regex = new RegExp(); attrs.regex = new RegExp();
wappalyzer.log(e + ': ' + attr, 'error', 'core'); this.log(e + ': ' + attr, 'error', 'core');
} }
} }
}); });
@ -234,9 +235,9 @@ function parsePatterns(patterns) {
} }
return parsed; return parsed;
} }
function resolveExcludes(apps) { resolveExcludes(apps) {
var excludes = []; var excludes = [];
// Exclude app in detected apps only // Exclude app in detected apps only
@ -244,7 +245,7 @@ function resolveExcludes(apps) {
var app = apps[appName]; var app = apps[appName];
if ( app.props.excludes ) { if ( app.props.excludes ) {
asArray(app.props.excludes).forEach(excluded => { this.asArray(app.props.excludes).forEach(excluded => {
excludes.push(excluded); excludes.push(excluded);
}); });
} }
@ -256,9 +257,9 @@ function resolveExcludes(apps) {
delete apps[appName]; delete apps[appName];
} }
}) })
} }
function resolveImplies(apps, url) { resolveImplies(apps, url) {
var checkImplies = true; var checkImplies = true;
// Implied applications // Implied applications
@ -270,17 +271,17 @@ function resolveImplies(apps, url) {
var app = apps[appName]; var app = apps[appName];
if ( app && app.implies ) { if ( app && app.implies ) {
asArray(app.props.implies).forEach(implied => { this.asArray(app.props.implies).forEach(implied => {
implied = parsePatterns(implied)[0]; implied = this.parsePatterns(implied)[0];
if ( !wappalyzer.apps[implied.string] ) { if ( !this.apps[implied.string] ) {
wappalyzer.log('Implied application ' + implied.string + ' does not exist', 'core', 'warn'); this.log('Implied application ' + implied.string + ' does not exist', 'core', 'warn');
return; return;
} }
if ( !( implied.string in apps ) ) { if ( !( implied.string in apps ) ) {
apps[implied.string] = detected[url] && detected[url][implied.string] ? detected[url][implied.string] : new Application(implied.string, true); apps[implied.string] = this.detected[url] && this.detected[url][implied.string] ? this.detected[url][implied.string] : new Application(implied.string, true);
checkImplies = true; checkImplies = true;
} }
@ -293,58 +294,58 @@ function resolveImplies(apps, url) {
} }
}); });
} }
} }
/** /**
* Cache detected applications * Cache detected applications
*/ */
function cacheDetectedApps(apps, url) { cacheDetectedApps(apps, url) {
if (!wappalyzer.driver.ping instanceof Function) return; if (!this.driver.ping instanceof Function) return;
Object.keys(apps).forEach(appName => { Object.keys(apps).forEach(appName => {
var app = apps[appName]; var app = apps[appName];
// Per URL // Per URL
detected[url][appName] = app; this.detected[url][appName] = app;
Object.keys(app.confidence).forEach(id => { Object.keys(app.confidence).forEach(id => {
detected[url][appName].confidence[id] = app.confidence[id]; this.detected[url][appName].confidence[id] = app.confidence[id];
}); });
}) })
wappalyzer.ping(); this.ping();
} }
/** /**
* Track detected applications * Track detected applications
*/ */
function trackDetectedApps(apps, url, hostname, html) { trackDetectedApps(apps, url, hostname, html) {
if (!wappalyzer.driver.ping instanceof Function) return; if (!this.driver.ping instanceof Function) return;
Object.keys(apps).forEach(appName => { Object.keys(apps).forEach(appName => {
var app = apps[appName]; var app = apps[appName];
if ( detected[url][appName].getConfidence() >= 100 ) { if ( this.detected[url][appName].getConfidence() >= 100 ) {
if ( validation.hostname.test(hostname) && !validation.hostnameBlacklist.test(url) ) { if ( validation.hostname.test(hostname) && !validation.hostnameBlacklist.test(url) ) {
wappalyzer.robotsTxtAllows(url) this.robotsTxtAllows(url)
.then(() => { .then(() => {
if ( !( hostname in hostnameCache ) ) { if ( !( hostname in this.hostnameCache ) ) {
hostnameCache[hostname] = { this.hostnameCache[hostname] = {
applications: {}, applications: {},
meta: {} meta: {}
}; };
} }
if ( !( appName in hostnameCache[hostname].applications ) ) { if ( !( appName in this.hostnameCache[hostname].applications ) ) {
hostnameCache[hostname].applications[appName] = { this.hostnameCache[hostname].applications[appName] = {
hits: 0 hits: 0
}; };
} }
hostnameCache[hostname].applications[appName].hits ++; this.hostnameCache[hostname].applications[appName].hits ++;
if ( apps[appName].version ) { if ( apps[appName].version ) {
hostnameCache[hostname].applications[appName].version = app.version; this.hostnameCache[hostname].applications[appName].version = app.version;
} }
}) })
.catch(() => console.log('Disallowed in robots.txt: ' + url)) .catch(() => console.log('Disallowed in robots.txt: ' + url))
@ -353,53 +354,53 @@ function trackDetectedApps(apps, url, hostname, html) {
}); });
// Additional information // Additional information
if ( hostname in hostnameCache ) { if ( hostname in this.hostnameCache ) {
var match = html.match(/<html[^>]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"/i); var match = html.match(/<html[^>]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"/i);
if ( match && match.length ) { if ( match && match.length ) {
hostnameCache[hostname].meta['language'] = match[1]; this.hostnameCache[hostname].meta['language'] = match[1];
} }
} }
wappalyzer.ping(); this.ping();
} }
/** /**
* Analyze URL * Analyze URL
*/ */
function analyzeUrl(app, url) { analyzeUrl(app, url) {
var patterns = parsePatterns(app.props.url); var patterns = this.parsePatterns(app.props.url);
if ( patterns.length ) { if ( patterns.length ) {
patterns.forEach(pattern => { patterns.forEach(pattern => {
if ( pattern.regex.test(url) ) { if ( pattern.regex.test(url) ) {
addDetected(app, pattern, 'url', url); this.addDetected(app, pattern, 'url', url);
} }
}); });
} }
} }
/** /**
* Analyze HTML * Analyze HTML
*/ */
function analyzeHtml(app, html) { analyzeHtml(app, html) {
var patterns = parsePatterns(app.props.html); var patterns = this.parsePatterns(app.props.html);
if ( patterns.length ) { if ( patterns.length ) {
patterns.forEach(pattern => { patterns.forEach(pattern => {
if ( pattern.regex.test(html) ) { if ( pattern.regex.test(html) ) {
addDetected(app, pattern, 'html', html); this.addDetected(app, pattern, 'html', html);
} }
}); });
} }
} }
/** /**
* Analyze script tag * Analyze script tag
*/ */
function analyzeScript(app, html) { analyzeScript(app, html) {
var regex = new RegExp('<script[^>]+src=("|\')([^"\']+)', 'ig'); var regex = new RegExp('<script[^>]+src=("|\')([^"\']+)', 'ig');
var patterns = parsePatterns(app.props.script); var patterns = this.parsePatterns(app.props.script);
if ( patterns.length ) { if ( patterns.length ) {
patterns.forEach(pattern => { patterns.forEach(pattern => {
@ -407,19 +408,19 @@ function analyzeScript(app, html) {
while ( ( match = regex.exec(html) ) ) { while ( ( match = regex.exec(html) ) ) {
if ( pattern.regex.test(match[2]) ) { if ( pattern.regex.test(match[2]) ) {
addDetected(app, pattern, 'script', match[2]); this.addDetected(app, pattern, 'script', match[2]);
} }
} }
}); });
} }
} }
/** /**
* Analyze meta tag * Analyze meta tag
*/ */
function analyzeMeta(app, html) { analyzeMeta(app, html) {
var regex = /<meta[^>]+>/ig; var regex = /<meta[^>]+>/ig;
var patterns = parsePatterns(app.props.meta); var patterns = this.parsePatterns(app.props.meta);
var content; var content;
var match; var match;
@ -430,19 +431,19 @@ function analyzeMeta(app, html) {
patterns[meta].forEach(pattern => { patterns[meta].forEach(pattern => {
if ( content && content.length === 4 && pattern.regex.test(content[2]) ) { if ( content && content.length === 4 && pattern.regex.test(content[2]) ) {
addDetected(app, pattern, 'meta', content[2], meta); this.addDetected(app, pattern, 'meta', content[2], meta);
} }
}); });
} }
} }
} }
} }
/** /**
* analyze response headers * analyze response headers
*/ */
function analyzeHeaders(app, headers) { analyzeHeaders(app, headers) {
var patterns = parsePatterns(app.props.headers); var patterns = this.parsePatterns(app.props.headers);
if ( headers ) { if ( headers ) {
Object.keys(patterns).forEach(header => { Object.keys(patterns).forEach(header => {
@ -450,49 +451,49 @@ function analyzeHeaders(app, headers) {
header = header.toLowerCase(); header = header.toLowerCase();
if ( header in headers && pattern.regex.test(headers[header]) ) { if ( header in headers && pattern.regex.test(headers[header]) ) {
addDetected(app, pattern, 'headers', headers[header], header); this.addDetected(app, pattern, 'headers', headers[header], header);
} }
}); });
}); });
} }
} }
/** /**
* Analyze environment variables * Analyze environment variables
*/ */
function analyzeEnv(app, envs) { analyzeEnv(app, envs) {
var patterns = parsePatterns(app.props.env); var patterns = this.parsePatterns(app.props.env);
if ( patterns.length ) { if ( patterns.length ) {
patterns.forEach(pattern => { patterns.forEach(pattern => {
Object.keys(envs).forEach(env => { Object.keys(envs).forEach(env => {
if ( pattern.regex.test(envs[env]) ) { if ( pattern.regex.test(envs[env]) ) {
addDetected(app, pattern, 'env', envs[env]); this.addDetected(app, pattern, 'env', envs[env]);
} }
}) })
}); });
} }
} }
/** /**
* Analyze robots.txt * Analyze robots.txt
*/ */
function analyzeRobotsTxt(app, robotsTxt) { analyzeRobotsTxt(app, robotsTxt) {
var patterns = parsePatterns(app.props.robotsTxt); var patterns = this.parsePatterns(app.props.robotsTxt);
if ( patterns.length ) { if ( patterns.length ) {
patterns.forEach(pattern => { patterns.forEach(pattern => {
if ( pattern.regex.test(robotsTxt) ) { if ( pattern.regex.test(robotsTxt) ) {
addDetected(app, pattern, 'robotsTxt', robotsTxt); this.addDetected(app, pattern, 'robotsTxt', robotsTxt);
} }
}); });
} }
} }
/** /**
* Mark application as detected, set confidence and version * Mark application as detected, set confidence and version
*/ */
function addDetected(app, pattern, type, value, key) { addDetected(app, pattern, type, value, key) {
app.detected = true; app.detected = true;
// Set confidence level // Set confidence level
@ -527,6 +528,7 @@ function addDetected(app, pattern, type, value, key) {
} }
} }
} }
}
} }
/** /**
@ -558,5 +560,5 @@ class Application {
} }
if ( typeof module === 'object' ) { if ( typeof module === 'object' ) {
module.exports = wappalyzer; module.exports = Wappalyzer;
} }