Migrations
/
wappalyzer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change wappalyzer object to Wappalyzer class to allow running of multiple instances

Browse Source
main
Elbert Alias 7 years ago
parent f0654e5bc0
commit 3057e396d9
6 changed files with 417 additions and 409 deletions
Show Stats Download Patch File Download Diff File

12
src/drivers/npm/driver.js
Unescape View File

@ -1,19 +1,21 @@
'use strict';
const wappalyzer = require('./wappalyzer');
const Wappalyzer = require('./wappalyzer');
const request = require('request');
const fs = require('fs');
const Browser = require('zombie');
const json = JSON.parse(fs.readFileSync(__dirname + '/apps.json'));
wappalyzer.apps = json.apps;
wappalyzer.categories = json.categories;
const driver = {
quiet: true,
analyze: url => {
const wappalyzer = new Wappalyzer();
wappalyzer.apps = json.apps;
wappalyzer.categories = json.categories;
return new Promise((resolve, reject) => {
wappalyzer.driver.log = (message, source, type) => {
if ( type === 'error' ) {
@ -51,6 +53,8 @@ const driver = {
});
});
console.log('resolve ' + url);
resolve(apps);
};

2
src/drivers/npm/package.json
Unescape View File

@ -2,7 +2,7 @@
"name": "wappalyzer",
"description": "Uncovers the technologies used on websites",
"homepage": "https://github.com/AliasIO/Wappalyzer",
"version": "5.0.5",
"version": "5.0.7",
"author": "Elbert Alias",
"license": "GPL-3.0",
"repository": {

4
src/drivers/webextension/js/driver.js
Unescape View File

@ -3,7 +3,9 @@
*/
/** global: browser */
/** global: wappalyzer */
/** global: Wappalyzer */
const wappalyzer = new Wappalyzer();
var tabCache = {};
var headersCache = {};

2
src/drivers/webextension/manifest.edge.json
Unescape View File

@ -4,7 +4,7 @@
"author": "Elbert Alias",
"homepage_url": "https://wappalyzer.com/",
"description": "Identify web technologies",
"version": "5.0.5",
"version": "5.0.7",
"default_locale": "en",
"manifest_version": 2,
"icons": {

2
src/drivers/webextension/manifest.json
Unescape View File

@ -4,7 +4,7 @@
"author": "Elbert Alias",
"homepage_url": "https://wappalyzer.com/",
"description": "Identify web technologies",
"version": "5.0.5",
"version": "5.0.7",
"default_locale": "en",
"manifest_version": 2,
"icons": {

272
src/wappalyzer.js
Unescape View File

@ -13,30 +13,31 @@ const validation = {
hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/
};
var wappalyzer = {
apps: {},
categories: {},
driver: {}
};
class Wappalyzer {
constructor() {
this.apps = {};
this.categories = {};
this.driver = {};
var detected = {};
var hostnameCache = {};
var adCache = [];
this.detected = {};
this.hostnameCache = {};
this.adCache = [];
wappalyzer.config = {
this.config = {
websiteURL: 'https://wappalyzer.com/',
twitterURL: 'https://twitter.com/Wappalyzer',
githubURL: 'https://github.com/AliasIO/Wappalyzer',
};
};
}
/**
/**
* Log messages to console
*/
wappalyzer.log = (message, source, type) => {
wappalyzer.driver.log(message, source || '', type || 'debug');
};
log(message, source, type) {
this.driver.log(message, source || '', type || 'debug');
}
wappalyzer.analyze = (hostname, url, data, context) => {
analyze(hostname, url, data, context) {
var apps = {};
// Remove hash from URL
@ -46,35 +47,35 @@ wappalyzer.analyze = (hostname, url, data, context) => {
data.html = '';
}
if ( detected[url] === undefined ) {
detected[url] = {};
if ( this.detected[url] === undefined ) {
this.detected[url] = {};
}
Object.keys(wappalyzer.apps).forEach(appName => {
apps[appName] = detected[url] && detected[url][appName] ? detected[url][appName] : new Application(appName, wappalyzer.apps[appName]);
Object.keys(this.apps).forEach(appName => {
apps[appName] = this.detected[url] && this.detected[url][appName] ? this.detected[url][appName] : new Application(appName, this.apps[appName]);
var app = apps[appName];
if ( url ) {
analyzeUrl(app, url);
this.analyzeUrl(app, url);
}
if ( data.html ) {
analyzeHtml(app, data.html);
analyzeScript(app, data.html);
analyzeMeta(app, data.html);
this.analyzeHtml(app, data.html);
this.analyzeScript(app, data.html);
this.analyzeMeta(app, data.html);
}
if ( data.headers ) {
analyzeHeaders(app, data.headers);
this.analyzeHeaders(app, data.headers);
}
if ( data.env ) {
analyzeEnv(app, data.env);
this.analyzeEnv(app, data.env);
}
if ( data.robotsTxt ) {
analyzeRobotsTxt(app, data.robotsTxt);
this.analyzeRobotsTxt(app, data.robotsTxt);
}
})
@ -86,34 +87,34 @@ wappalyzer.analyze = (hostname, url, data, context) => {
}
});
resolveExcludes(apps);
resolveImplies(apps, url);
this.resolveExcludes(apps);
this.resolveImplies(apps, url);
cacheDetectedApps(apps, url);
trackDetectedApps(apps, url, hostname, data.html);
this.cacheDetectedApps(apps, url);
this.trackDetectedApps(apps, url, hostname, data.html);
if ( Object.keys(apps).length ) {
wappalyzer.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url, 'core');
this.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url, 'core');
}
wappalyzer.driver.displayApps(detected[url], context);
}
this.driver.displayApps(this.detected[url], context);
}
/**
/**
* Cache detected ads
*/
wappalyzer.cacheDetectedAds = ad => {
adCache.push(ad);
}
cacheDetectedAds(ad) {
this.adCache.push(ad);
}
/**
/**
*
*/
wappalyzer.robotsTxtAllows = url => {
robotsTxtAllows(url) {
return new Promise((resolve, reject) => {
var parsed = wappalyzer.parseUrl(url);
var parsed = this.parseUrl(url);
wappalyzer.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
this.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
.then(robotsTxt => {
robotsTxt.forEach(disallow => {
if ( parsed.pathname.indexOf(disallow) === 0 ) {
@ -124,25 +125,25 @@ wappalyzer.robotsTxtAllows = url => {
resolve();
});
});
};
};
/**
/**
* Parse a URL
*/
wappalyzer.parseUrl = url => {
var a = wappalyzer.driver.document.createElement('a');
parseUrl(url) {
var a = this.driver.document.createElement('a');
a.href = url;
a.canonical = a.protocol + '//' + a.host + a.pathname;
return a;
}
}
/**
/**
*
*/
wappalyzer.parseRobotsTxt = robotsTxt => {
parseRobotsTxt(robotsTxt) {
var userAgent;
var disallow = [];
@ -163,44 +164,44 @@ wappalyzer.parseRobotsTxt = robotsTxt => {
});
return disallow;
}
}
/**
/**
*
*/
wappalyzer.ping = () => {
if ( Object.keys(hostnameCache).length >= 50 || adCache.length >= 50 ) {
wappalyzer.driver.ping(hostnameCache, adCache);
ping() {
if ( Object.keys(this.hostnameCache).length >= 50 || this.adCache.length >= 50 ) {
this.driver.ping(this.hostnameCache, this.adCache);
hostnameCache = {};
adCache = [];
this.hostnameCache = {};
this.adCache = [];
}
}
}
/**
/**
* Enclose string in array
*/
function asArray(value) {
asArray(value) {
return typeof value === 'string' ? [ value ] : value;
}
}
/**
/**
* Parse apps.json patterns
*/
function parsePatterns(patterns) {
parsePatterns(patterns) {
var parsed = {};
// Convert string to object containing array containing string
if ( typeof patterns === 'string' || patterns instanceof Array ) {
patterns = {
main: asArray(patterns)
main: this.asArray(patterns)
};
}
for ( var key in patterns ) {
parsed[key] = [];
asArray(patterns[key]).forEach(pattern => {
this.asArray(patterns[key]).forEach(pattern => {
var attrs = {};
pattern.split('\\;').forEach((attr, i) => {
@ -219,7 +220,7 @@ function parsePatterns(patterns) {
} catch (e) {
attrs.regex = new RegExp();
wappalyzer.log(e + ': ' + attr, 'error', 'core');
this.log(e + ': ' + attr, 'error', 'core');
}
}
});
@ -234,9 +235,9 @@ function parsePatterns(patterns) {
}
return parsed;
}
}
function resolveExcludes(apps) {
resolveExcludes(apps) {
var excludes = [];
// Exclude app in detected apps only
@ -244,7 +245,7 @@ function resolveExcludes(apps) {
var app = apps[appName];
if ( app.props.excludes ) {
asArray(app.props.excludes).forEach(excluded => {
this.asArray(app.props.excludes).forEach(excluded => {
excludes.push(excluded);
});
}
@ -256,9 +257,9 @@ function resolveExcludes(apps) {
delete apps[appName];
}
})
}
}
function resolveImplies(apps, url) {
resolveImplies(apps, url) {
var checkImplies = true;
// Implied applications
@ -270,17 +271,17 @@ function resolveImplies(apps, url) {
var app = apps[appName];
if ( app && app.implies ) {
asArray(app.props.implies).forEach(implied => {
implied = parsePatterns(implied)[0];
this.asArray(app.props.implies).forEach(implied => {
implied = this.parsePatterns(implied)[0];
if ( !wappalyzer.apps[implied.string] ) {
wappalyzer.log('Implied application ' + implied.string + ' does not exist', 'core', 'warn');
if ( !this.apps[implied.string] ) {
this.log('Implied application ' + implied.string + ' does not exist', 'core', 'warn');
return;
}
if ( !( implied.string in apps ) ) {
apps[implied.string] = detected[url] && detected[url][implied.string] ? detected[url][implied.string] : new Application(implied.string, true);
apps[implied.string] = this.detected[url] && this.detected[url][implied.string] ? this.detected[url][implied.string] : new Application(implied.string, true);
checkImplies = true;
}
@ -293,58 +294,58 @@ function resolveImplies(apps, url) {
}
});
}
}
}
/**
/**
* Cache detected applications
*/
function cacheDetectedApps(apps, url) {
if (!wappalyzer.driver.ping instanceof Function) return;
cacheDetectedApps(apps, url) {
if (!this.driver.ping instanceof Function) return;
Object.keys(apps).forEach(appName => {
var app = apps[appName];
// Per URL
detected[url][appName] = app;
this.detected[url][appName] = app;
Object.keys(app.confidence).forEach(id => {
detected[url][appName].confidence[id] = app.confidence[id];
this.detected[url][appName].confidence[id] = app.confidence[id];
});
})
wappalyzer.ping();
}
this.ping();
}
/**
/**
* Track detected applications
*/
function trackDetectedApps(apps, url, hostname, html) {
if (!wappalyzer.driver.ping instanceof Function) return;
trackDetectedApps(apps, url, hostname, html) {
if (!this.driver.ping instanceof Function) return;
Object.keys(apps).forEach(appName => {
var app = apps[appName];
if ( detected[url][appName].getConfidence() >= 100 ) {
if ( this.detected[url][appName].getConfidence() >= 100 ) {
if ( validation.hostname.test(hostname) && !validation.hostnameBlacklist.test(url) ) {
wappalyzer.robotsTxtAllows(url)
this.robotsTxtAllows(url)
.then(() => {
if ( !( hostname in hostnameCache ) ) {
hostnameCache[hostname] = {
if ( !( hostname in this.hostnameCache ) ) {
this.hostnameCache[hostname] = {
applications: {},
meta: {}
};
}
if ( !( appName in hostnameCache[hostname].applications ) ) {
hostnameCache[hostname].applications[appName] = {
if ( !( appName in this.hostnameCache[hostname].applications ) ) {
this.hostnameCache[hostname].applications[appName] = {
hits: 0
};
}
hostnameCache[hostname].applications[appName].hits ++;
this.hostnameCache[hostname].applications[appName].hits ++;
if ( apps[appName].version ) {
hostnameCache[hostname].applications[appName].version = app.version;
this.hostnameCache[hostname].applications[appName].version = app.version;
}
})
.catch(() => console.log('Disallowed in robots.txt: ' + url))
@ -353,53 +354,53 @@ function trackDetectedApps(apps, url, hostname, html) {
});
// Additional information
if ( hostname in hostnameCache ) {
if ( hostname in this.hostnameCache ) {
var match = html.match(/<html[^>]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"/i);
if ( match && match.length ) {
hostnameCache[hostname].meta['language'] = match[1];
this.hostnameCache[hostname].meta['language'] = match[1];
}
}
wappalyzer.ping();
}
this.ping();
}
/**
/**
* Analyze URL
*/
function analyzeUrl(app, url) {
var patterns = parsePatterns(app.props.url);
analyzeUrl(app, url) {
var patterns = this.parsePatterns(app.props.url);
if ( patterns.length ) {
patterns.forEach(pattern => {
if ( pattern.regex.test(url) ) {
addDetected(app, pattern, 'url', url);
this.addDetected(app, pattern, 'url', url);
}
});
}
}
}
/**
/**
* Analyze HTML
*/
function analyzeHtml(app, html) {
var patterns = parsePatterns(app.props.html);
analyzeHtml(app, html) {
var patterns = this.parsePatterns(app.props.html);
if ( patterns.length ) {
patterns.forEach(pattern => {
if ( pattern.regex.test(html) ) {
addDetected(app, pattern, 'html', html);
this.addDetected(app, pattern, 'html', html);
}
});
}
}
}
/**
/**
* Analyze script tag
*/
function analyzeScript(app, html) {
analyzeScript(app, html) {
var regex = new RegExp('<script[^>]+src=("|\')([^"\']+)', 'ig');
var patterns = parsePatterns(app.props.script);
var patterns = this.parsePatterns(app.props.script);
if ( patterns.length ) {
patterns.forEach(pattern => {
@ -407,19 +408,19 @@ function analyzeScript(app, html) {
while ( ( match = regex.exec(html) ) ) {
if ( pattern.regex.test(match[2]) ) {
addDetected(app, pattern, 'script', match[2]);
this.addDetected(app, pattern, 'script', match[2]);
}
}
});
}
}
}
/**
/**
* Analyze meta tag
*/
function analyzeMeta(app, html) {
analyzeMeta(app, html) {
var regex = /<meta[^>]+>/ig;
var patterns = parsePatterns(app.props.meta);
var patterns = this.parsePatterns(app.props.meta);
var content;
var match;
@ -430,19 +431,19 @@ function analyzeMeta(app, html) {
patterns[meta].forEach(pattern => {
if ( content && content.length === 4 && pattern.regex.test(content[2]) ) {
addDetected(app, pattern, 'meta', content[2], meta);
this.addDetected(app, pattern, 'meta', content[2], meta);
}
});
}
}
}
}
}
/**
/**
* analyze response headers
*/
function analyzeHeaders(app, headers) {
var patterns = parsePatterns(app.props.headers);
analyzeHeaders(app, headers) {
var patterns = this.parsePatterns(app.props.headers);
if ( headers ) {
Object.keys(patterns).forEach(header => {
@ -450,49 +451,49 @@ function analyzeHeaders(app, headers) {
header = header.toLowerCase();
if ( header in headers && pattern.regex.test(headers[header]) ) {
addDetected(app, pattern, 'headers', headers[header], header);
this.addDetected(app, pattern, 'headers', headers[header], header);
}
});
});
}
}
}
/**
/**
* Analyze environment variables
*/
function analyzeEnv(app, envs) {
var patterns = parsePatterns(app.props.env);
analyzeEnv(app, envs) {
var patterns = this.parsePatterns(app.props.env);
if ( patterns.length ) {
patterns.forEach(pattern => {
Object.keys(envs).forEach(env => {
if ( pattern.regex.test(envs[env]) ) {
addDetected(app, pattern, 'env', envs[env]);
this.addDetected(app, pattern, 'env', envs[env]);
}
})
});
}
}
}
/**
/**
* Analyze robots.txt
*/
function analyzeRobotsTxt(app, robotsTxt) {
var patterns = parsePatterns(app.props.robotsTxt);
analyzeRobotsTxt(app, robotsTxt) {
var patterns = this.parsePatterns(app.props.robotsTxt);
if ( patterns.length ) {
patterns.forEach(pattern => {
if ( pattern.regex.test(robotsTxt) ) {
addDetected(app, pattern, 'robotsTxt', robotsTxt);
this.addDetected(app, pattern, 'robotsTxt', robotsTxt);
}
});
}
}
}
/**
/**
* Mark application as detected, set confidence and version
*/
function addDetected(app, pattern, type, value, key) {
addDetected(app, pattern, type, value, key) {
app.detected = true;
// Set confidence level
@ -527,6 +528,7 @@ function addDetected(app, pattern, type, value, key) {
}
}
}
}
}
/**
@ -558,5 +560,5 @@ class Application {
}
if ( typeof module === 'object' ) {
module.exports = wappalyzer;
module.exports = Wappalyzer;
}

Write Preview
Loading…
Cancel
Save