Migrations
/
wappalyzer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix options, replace Promises with async/await, build v5.5.7

Browse Source
main
Elbert Alias 6 years ago
parent 518b1d1462
commit 879cb32d30
7 changed files with 317 additions and 309 deletions
Show Stats Download Patch File Download Diff File

2
src/drivers/npm/npm-shrinkwrap.json generated
Unescape View File

@ -1,6 +1,6 @@
{ {
"name": "wappalyzer", "name": "wappalyzer",
"version": "5.5.5", "version": "5.5.6",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {

2
src/drivers/npm/package.json
Unescape View File

@ -2,7 +2,7 @@
"name": "wappalyzer", "name": "wappalyzer",
"description": "Uncovers the technologies used on websites", "description": "Uncovers the technologies used on websites",
"homepage": "https://github.com/AliasIO/Wappalyzer", "homepage": "https://github.com/AliasIO/Wappalyzer",
"version": "5.5.6", "version": "5.5.7",
"author": "Elbert Alias", "author": "Elbert Alias",
"license": "GPL-3.0", "license": "GPL-3.0",
"repository": { "repository": {

402
src/drivers/webextension/js/driver.js
Unescape View File

@ -3,20 +3,19 @@
*/ */
/* eslint-env browser */ /* eslint-env browser */
/* global browser, chrome, fetch, Wappalyzer */ /* global browser, fetch, Wappalyzer */
/** global: browser */ /** global: browser */
/** global: chrome */
/** global: fetch */ /** global: fetch */
/** global: Wappalyzer */ /** global: Wappalyzer */
const wappalyzer = new Wappalyzer(); const wappalyzer = new Wappalyzer();
const tabCache = {}; const tabCache = {};
let categoryOrder = [];
const options = {};
const robotsTxtQueue = {}; const robotsTxtQueue = {};
let categoryOrder = [];
browser.tabs.onRemoved.addListener((tabId) => { browser.tabs.onRemoved.addListener((tabId) => {
tabCache[tabId] = null; tabCache[tabId] = null;
}); });
@ -26,17 +25,19 @@ browser.tabs.onRemoved.addListener((tabId) => {
*/ */
function getOption(name, defaultValue = null) { function getOption(name, defaultValue = null) {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
let option = defaultValue; let value = defaultValue;
try { try {
option = await browser.storage.local.get(name); const option = await browser.storage.local.get(name);
if (option[name]) {
value = option[name];
}
} catch (error) { } catch (error) {
wappalyzer.log(error, 'driver', 'error'); wappalyzer.log(error, 'driver', 'error');
} }
options[name] = option; resolve(value);
resolve(option);
}); });
} }
@ -44,13 +45,15 @@ function getOption(name, defaultValue = null) {
* Set a value in localStorage * Set a value in localStorage
*/ */
function setOption(name, value) { function setOption(name, value) {
const option = {}; return new Promise(async (resolve) => {
try {
option[name] = value; await browser.storage.local.set({ [name]: value });
} catch (error) {
browser.storage.local.set(option); wappalyzer.log(error, 'driver', 'error');
}
options[name] = value; resolve();
});
} }
/** /**
@ -66,159 +69,110 @@ function openTab(args) {
/** /**
* Make a POST request * Make a POST request
*/ */
function post(url, body) { async function post(url, body) {
fetch(url, { try {
method: 'POST', const response = await fetch(url, {
body: JSON.stringify(body), method: 'POST',
}) body: JSON.stringify(body),
.then(response => wappalyzer.log(`POST ${url}: ${response.status}`, 'driver')) });
.catch(error => wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error'));
wappalyzer.log(`POST ${url}: ${response.status}`, 'driver');
} catch (error) {
wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error');
}
} }
fetch('../apps.json') // Capture response headers
.then(response => response.json()) browser.webRequest.onCompleted.addListener(async (request) => {
.then((json) => { const headers = {};
wappalyzer.apps = json.apps;
wappalyzer.categories = json.categories;
wappalyzer.parseJsPatterns(); if (request.responseHeaders) {
const url = wappalyzer.parseUrl(request.url);
categoryOrder = Object.keys(wappalyzer.categories) let tab;
.map(categoryId => parseInt(categoryId, 10))
.sort((a, b) => wappalyzer.categories[a].priority - wappalyzer.categories[b].priority);
})
.catch(error => wappalyzer.log(`GET apps.json: ${error}`, 'driver', 'error'));
// Version check try {
const { version } = browser.runtime.getManifest(); [tab] = await browser.tabs.query({ url: [url.href] });
} catch (error) {
wappalyzer.log(error, 'driver', 'error');
}
getOption('version') if (tab) {
.then((previousVersion) => { request.responseHeaders.forEach((header) => {
if (previousVersion === null) { const name = header.name.toLowerCase();
openTab({
url: `${wappalyzer.config.websiteURL}installed`, headers[name] = headers[name] || [];
headers[name].push((header.value || header.binaryValue || '').toString());
}); });
} else if (version !== previousVersion) {
getOption('upgradeMessage', true) if (headers['content-type'] && /\/x?html/.test(headers['content-type'][0])) {
.then((upgradeMessage) => { wappalyzer.analyze(url, { headers }, { tab });
if (upgradeMessage) { }
openTab({
url: `${wappalyzer.config.websiteURL}upgraded?v${version}`,
background: true,
});
}
});
} }
}
}, { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, ['responseHeaders']);
setOption('version', version); // Listen for messages
}); browser.runtime.onMessage.addListener(async (message, sender) => {
if (message.id === undefined) {
return Promise.resolve();
}
getOption('dynamicIcon', true); if (message.id !== 'log') {
getOption('pinnedCategory'); wappalyzer.log(`Message${message.source ? ` from ${message.source}` : ''}: ${message.id}`, 'driver');
}
getOption('hostnameCache', {}) const pinnedCategory = await getOption('pinnedCategory');
.then((hostnameCache) => {
wappalyzer.hostnameCache = hostnameCache;
return hostnameCache; const url = wappalyzer.parseUrl(sender.tab ? sender.tab.url : '');
});
// Run content script on all tabs const cookies = await browser.cookies.getAll({ domain: `.${url.hostname}` });
browser.tabs.query({ url: ['http://*/*', 'https://*/*'] })
.then((tabs) => {
tabs.forEach((tab) => {
browser.tabs.executeScript(tab.id, {
file: '../js/content.js',
});
});
})
.catch(error => wappalyzer.log(error, 'driver', 'error'));
// Capture response headers let response;
browser.webRequest.onCompleted.addListener((request) => {
const headers = {};
if (request.responseHeaders) { switch (message.id) {
const url = wappalyzer.parseUrl(request.url); case 'log':
wappalyzer.log(message.subject, message.source);
browser.tabs.query({ url: [url.href] }) break;
.then((tabs) => { case 'init':
const tab = tabs[0] || null; wappalyzer.analyze(url, { cookies }, { tab: sender.tab });
if (tab) { break;
request.responseHeaders.forEach((header) => { case 'analyze':
const name = header.name.toLowerCase(); wappalyzer.analyze(url, message.subject, { tab: sender.tab });
headers[name] = headers[name] || []; await setOption('hostnameCache', wappalyzer.hostnameCache);
headers[name].push((header.value || header.binaryValue || '').toString()); break;
}); case 'ad_log':
wappalyzer.cacheDetectedAds(message.subject);
if (headers['content-type'] && /\/x?html/.test(headers['content-type'][0])) { break;
wappalyzer.analyze(url, { headers }, { tab }); case 'get_apps':
} response = {
} tabCache: tabCache[message.tab.id],
}) apps: wappalyzer.apps,
.catch(error => wappalyzer.log(error, 'driver', 'error')); categories: wappalyzer.categories,
} pinnedCategory,
}, { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, ['responseHeaders']); };
// Listen for messages break;
(chrome || browser).runtime.onMessage.addListener((message, sender, sendResponse) => { case 'set_option':
if (typeof message.id !== 'undefined') { await setOption(message.key, message.value);
if (message.id !== 'log') {
wappalyzer.log(`Message${message.source ? ` from ${message.source}` : ''}: ${message.id}`, 'driver');
}
const url = wappalyzer.parseUrl(sender.tab ? sender.tab.url : ''); break;
let response; case 'get_js_patterns':
response = {
patterns: wappalyzer.jsPatterns,
};
switch (message.id) { break;
case 'log': default:
wappalyzer.log(message.subject, message.source);
break;
case 'init':
browser.cookies.getAll({ domain: `.${url.hostname}` })
.then(cookies => wappalyzer.analyze(url, { cookies }, { tab: sender.tab }));
break;
case 'analyze':
wappalyzer.analyze(url, message.subject, { tab: sender.tab });
setOption('hostnameCache', wappalyzer.hostnameCache);
break;
case 'ad_log':
wappalyzer.cacheDetectedAds(message.subject);
break;
case 'get_apps':
response = {
tabCache: tabCache[message.tab.id],
apps: wappalyzer.apps,
categories: wappalyzer.categories,
pinnedCategory: options.pinnedCategory,
};
break;
case 'set_option':
setOption(message.key, message.value);
break;
case 'get_js_patterns':
response = {
patterns: wappalyzer.jsPatterns,
};
break;
default:
}
sendResponse(response);
} }
return true; return Promise.resolve(response);
}); });
wappalyzer.driver.document = document; wappalyzer.driver.document = document;
@ -227,13 +181,15 @@ wappalyzer.driver.document = document;
* Log messages to console * Log messages to console
*/ */
wappalyzer.driver.log = (message, source, type) => { wappalyzer.driver.log = (message, source, type) => {
console.log(`[wappalyzer ${type}]`, `[${source}]`, message); const log = ['warn', 'error'].indexOf(type) !== -1 ? type : 'log';
console[log](`[wappalyzer ${type}]`, `[${source}]`, message);
}; };
/** /**
* Display apps * Display apps
*/ */
wappalyzer.driver.displayApps = (detected, meta, context) => { wappalyzer.driver.displayApps = async (detected, meta, context) => {
const { tab } = context; const { tab } = context;
if (tab === undefined) { if (tab === undefined) {
@ -246,20 +202,19 @@ wappalyzer.driver.displayApps = (detected, meta, context) => {
tabCache[tab.id].detected = detected; tabCache[tab.id].detected = detected;
const pinnedCategory = await getOption('pinnedCategory');
const dynamicIcon = await getOption('dynamicIcon', true);
let found = false; let found = false;
// Find the main application to display // Find the main application to display
[options.pinnedCategory].concat(categoryOrder).forEach((match) => { [pinnedCategory].concat(categoryOrder).forEach((match) => {
Object.keys(detected).forEach((appName) => { Object.keys(detected).forEach((appName) => {
const app = detected[appName]; const app = detected[appName];
app.props.cats.forEach((category) => { app.props.cats.forEach((category) => {
if (category === match && !found) { if (category === match && !found) {
let icon = app.props.icon || 'default.svg'; let icon = app.props.icon && dynamicIcon ? app.props.icon : 'default.svg';
if (!options.dynamicIcon) {
icon = 'default.svg';
}
if (/\.svg$/i.test(icon)) { if (/\.svg$/i.test(icon)) {
icon = `converted/${icon.replace(/\.svg$/, '.png')}`; icon = `converted/${icon.replace(/\.svg$/, '.png')}`;
@ -280,61 +235,53 @@ wappalyzer.driver.displayApps = (detected, meta, context) => {
}); });
}); });
if (typeof chrome !== 'undefined') { browser.pageAction.show(tab.id);
// Browser polyfill doesn't seem to work here
chrome.pageAction.show(tab.id);
} else {
browser.pageAction.show(tab.id);
}
}; };
/** /**
* Fetch and cache robots.txt for host * Fetch and cache robots.txt for host
*/ */
wappalyzer.driver.getRobotsTxt = (host, secure = false) => { wappalyzer.driver.getRobotsTxt = async (host, secure = false) => {
if (robotsTxtQueue[host]) { if (robotsTxtQueue[host]) {
return robotsTxtQueue[host]; return robotsTxtQueue[host];
} }
robotsTxtQueue[host] = new Promise((resolve) => { const tracking = await getOption('tracking', true);
getOption('tracking', true) const robotsTxtCache = await getOption('robotsTxtCache', {});
.then((tracking) => {
if (!tracking) {
resolve([]);
return; robotsTxtQueue[host] = new Promise(async (resolve) => {
} if (!tracking) {
return resolve([]);
}
if (host in robotsTxtCache) {
return resolve(robotsTxtCache[host]);
}
getOption('robotsTxtCache') const timeout = setTimeout(() => resolve([]), 3000);
.then((robotsTxtCache) => {
robotsTxtCache = robotsTxtCache || {};
if (host in robotsTxtCache) { let response;
resolve(robotsTxtCache[host]);
return; try {
} response = await fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { redirect: 'follow' });
} catch (error) {
wappalyzer.log(error, 'driver', 'error');
const timeout = setTimeout(() => resolve([]), 3000); return resolve([]);
}
fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { redirect: 'follow' }) clearTimeout(timeout);
.then((response) => {
clearTimeout(timeout);
return response.ok ? response.text() : ''; const robotsTxt = response.ok ? await response.text() : '';
})
.then((robotsTxt) => {
robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt);
setOption('robotsTxtCache', robotsTxtCache); robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt);
resolve(robotsTxtCache[host]); await setOption('robotsTxtCache', robotsTxtCache);
})
.catch(() => resolve([])); delete robotsTxtQueue[host];
});
}); return resolve(robotsTxtCache[host]);
}) });
.finally(() => delete robotsTxtQueue[host]);
return robotsTxtQueue[host]; return robotsTxtQueue[host];
}; };
@ -342,19 +289,72 @@ wappalyzer.driver.getRobotsTxt = (host, secure = false) => {
/** /**
* Anonymously track detected applications for research purposes * Anonymously track detected applications for research purposes
*/ */
wappalyzer.driver.ping = (hostnameCache = {}, adCache = []) => { wappalyzer.driver.ping = async (hostnameCache = {}, adCache = []) => {
getOption('tracking', true) const tracking = await getOption('tracking', true);
.then((tracking) => {
if (tracking) {
if (Object.keys(hostnameCache).length) {
post('https://api.wappalyzer.com/ping/v1/', hostnameCache);
}
if (adCache.length) { if (tracking) {
post('https://ad.wappalyzer.com/log/wp/', adCache); if (Object.keys(hostnameCache).length) {
} post('https://api.wappalyzer.com/ping/v1/', hostnameCache);
}
setOption('robotsTxtCache', {}); if (adCache.length) {
} post('https://ad.wappalyzer.com/log/wp/', adCache);
}); }
await setOption('robotsTxtCache', {});
}
}; };
// Init
(async () => {
// Technologies
try {
const response = await fetch('../apps.json');
const json = await response.json();
wappalyzer.apps = json.apps;
wappalyzer.categories = json.categories;
} catch (error) {
wappalyzer.log(`GET apps.json: ${error.message}`, 'driver', 'error');
}
wappalyzer.parseJsPatterns();
categoryOrder = Object.keys(wappalyzer.categories)
.map(categoryId => parseInt(categoryId, 10))
.sort((a, b) => wappalyzer.categories[a].priority - wappalyzer.categories[b].priority);
// Version check
const { version } = browser.runtime.getManifest();
const previousVersion = await getOption('version');
const upgradeMessage = await getOption('upgradeMessage', true);
if (previousVersion === null) {
openTab({
url: `${wappalyzer.config.websiteURL}installed`,
});
} else if (version !== previousVersion && upgradeMessage) {
//openTab({
// url: `${wappalyzer.config.websiteURL}upgraded?v${version}`,
// background: true,
//});
}
await setOption('version', version);
// Hostname cache
wappalyzer.hostnameCache = await getOption('hostnameCache', {});
// Run content script on all tabs
try {
const tabs = await browser.tabs.query({ url: ['http://*/*', 'https://*/*'] });
tabs.forEach((tab) => {
browser.tabs.executeScript(tab.id, {
file: '../js/content.js',
});
});
} catch (error) {
wappalyzer.log(error, 'driver', 'error');
}
})();

61
src/drivers/webextension/js/popup.js
Unescape View File

@ -1,32 +1,14 @@
/* eslint-env browser */
/* global browser, chrome, jsonToDOM */
/** global: chrome */ /** global: chrome */
/** global: browser */ /** global: browser */
/** global: jsonToDOM */
let pinnedCategory = null; let pinnedCategory = null;
const func = (tabs) => { function slugify(string) {
(chrome || browser).runtime.sendMessage({ return string.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/--+/g, '-').replace(/(?:^-|-$)/, '');
id: 'get_apps',
tab: tabs[0],
source: 'popup.js',
}, (response) => {
pinnedCategory = response.pinnedCategory;
replaceDomWhenReady(appsToDomTemplate(response));
});
};
browser.tabs.query({ active: true, currentWindow: true })
.then(func)
.catch(console.error);
function replaceDomWhenReady(dom) {
if (/complete|interactive|loaded/.test(document.readyState)) {
replaceDom(dom);
} else {
document.addEventListener('DOMContentLoaded', () => {
replaceDom(dom);
});
}
} }
function replaceDom(domTemplate) { function replaceDom(domTemplate) {
@ -73,6 +55,16 @@ function replaceDom(domTemplate) {
}); });
} }
function replaceDomWhenReady(dom) {
if (/complete|interactive|loaded/.test(document.readyState)) {
replaceDom(dom);
} else {
document.addEventListener('DOMContentLoaded', () => {
replaceDom(dom);
});
}
}
function appsToDomTemplate(response) { function appsToDomTemplate(response) {
let template = []; let template = [];
@ -92,8 +84,7 @@ function appsToDomTemplate(response) {
const apps = []; const apps = [];
for (const appName in categories[cat].apps) { for (const appName in categories[cat].apps) {
const confidence = response.tabCache.detected[appName].confidenceTotal; const { confidence, version } = response.tabCache.detected[appName];
const version = response.tabCache.detected[appName].version;
apps.push( apps.push(
[ [
@ -190,6 +181,18 @@ function appsToDomTemplate(response) {
return template; return template;
} }
function slugify(string) { const func = (tabs) => {
return string.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/--+/g, '-').replace(/(?:^-|-$)/, ''); (chrome || browser).runtime.sendMessage({
} id: 'get_apps',
tab: tabs[0],
source: 'popup.js',
}, (response) => {
pinnedCategory = response.pinnedCategory;
replaceDomWhenReady(appsToDomTemplate(response));
});
};
browser.tabs.query({ active: true, currentWindow: true })
.then(func)
.catch(console.error);

151
src/drivers/webextension/manifest.json
Unescape View File

@ -1,78 +1,83 @@
{ {
"name": "Wappalyzer", "name": "Wappalyzer",
"short_name": "Wappalyzer", "short_name": "Wappalyzer",
"author": "Elbert Alias", "author": "Elbert Alias",
"homepage_url": "https://www.wappalyzer.com", "homepage_url": "https://www.wappalyzer.com",
"description": "Identify web technologies", "description": "Identify web technologies",
"version": "5.5.6", "version": "5.5.7",
"default_locale": "en", "default_locale": "en",
"manifest_version": 2, "manifest_version": 2,
"icons": { "icons": {
"16": "images/icon_16.png", "16": "images/icon_16.png",
"19": "images/icon_19.png", "19": "images/icon_19.png",
"32": "images/icon_32.png", "32": "images/icon_32.png",
"38": "images/icon_38.png", "38": "images/icon_38.png",
"128": "images/icon_128.png" "128": "images/icon_128.png"
}, },
"page_action": { "page_action": {
"default_icon": { "default_icon": {
"16": "images/icon_16.png", "16": "images/icon_16.png",
"19": "images/icon_19.png", "19": "images/icon_19.png",
"32": "images/icon_32.png", "32": "images/icon_32.png",
"38": "images/icon_38.png", "38": "images/icon_38.png",
"128": "images/icon_128.png" "128": "images/icon_128.png"
}, },
"default_title": "Wappalyzer", "default_title": "Wappalyzer",
"default_popup": "html/popup.html" "default_popup": "html/popup.html"
}, },
"background": { "background": {
"page": "html/background.html" "page": "html/background.html"
}, },
"content_scripts": [ "content_scripts": [
{ {
"matches": [ "matches": [
"http://*/*",
"https://*/*"
],
"js": [
"node_modules/webextension-polyfill/dist/browser-polyfill.js",
"js/content.js"
],
"run_at": "document_idle"
},
{
"matches": [
"http://*/*", "http://*/*",
"https://*/*" "https://*/*"
], ],
"exclude_matches": [ "js": [
"https://*.modirum.com/*", "node_modules/webextension-polyfill/dist/browser-polyfill.js",
"https://www.alphaecommerce.gr/*" "js/content.js"
], ],
"js": [ "run_at": "document_idle"
"js/lib/iframe.js" },
], {
"run_at": "document_start", "matches": [
"all_frames": true "http://*/*",
} "https://*/*"
], ],
"web_accessible_resources": [ "exclude_matches": [
"js/inject.js" "https://*.modirum.com/*",
], "https://www.alphaecommerce.gr/*"
"options_page": "html/options.html", ],
"options_ui": { "js": [
"page": "html/options.html", "js/lib/iframe.js"
"open_in_tab": false ],
}, "run_at": "document_start",
"permissions": [ "all_frames": true
}
],
"web_accessible_resources": [
"js/inject.js"
],
"options_ui": {
"page": "html/options.html",
"open_in_tab": false
},
"permissions": [
"cookies", "cookies",
"storage", "storage",
"tabs", "tabs",
"webRequest", "webRequest",
"webNavigation", "webNavigation",
"http://*/*", "http://*/*",
"https://*/*" "https://*/*"
], ],
"content_security_policy": "script-src 'self'; object-src 'self'" "content_security_policy": "script-src 'self'; object-src 'self'",
"applications": {
"gecko": {
"id": "wappalyzer@crunchlabz.com",
"strict_min_version": "42.0"
}
}
} }

6
src/drivers/webextension/npm-shrinkwrap.json generated
Unescape View File

@ -3,9 +3,9 @@
"lockfileVersion": 1, "lockfileVersion": 1,
"dependencies": { "dependencies": {
"webextension-polyfill": { "webextension-polyfill": {
"version": "0.2.1", "version": "0.3.1",
"resolved": "https://registry.npmjs.org/webextension-polyfill/-/webextension-polyfill-0.2.1.tgz", "resolved": "https://registry.npmjs.org/webextension-polyfill/-/webextension-polyfill-0.3.1.tgz",
"integrity": "sha1-zfyRJgMwOfFxNVMVfTW+/x1Nb0o=" "integrity": "sha512-ISB42vlgMyM7xE1u6pREeCqmmXjLsYu/nqAR8Dl/gIAnylb+KpRpvKbVkUYNFePhhXn0Obkkc3jasOII9ztUtg=="
} }
} }
} }

2
src/drivers/webextension/package.json
Unescape View File

@ -1,5 +1,5 @@
{ {
"dependencies": { "dependencies": {
"webextension-polyfill": "^0.2.1" "webextension-polyfill": "^0.3.1"
} }
} }

Write Preview
Loading…
Cancel
Save