Migrations
/
wappalyzer
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix options, replace Promises with async/await, build v5.5.7

Browse Source
main
Elbert Alias 6 years ago
parent 518b1d1462
commit 879cb32d30
7 changed files with 317 additions and 309 deletions
Show Stats Download Patch File Download Diff File

2
src/drivers/npm/npm-shrinkwrap.json generated
Unescape View File

@ -1,6 +1,6 @@
{
"name": "wappalyzer",
"version": "5.5.5",
"version": "5.5.6",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

2
src/drivers/npm/package.json
Unescape View File

@ -2,7 +2,7 @@
"name": "wappalyzer",
"description": "Uncovers the technologies used on websites",
"homepage": "https://github.com/AliasIO/Wappalyzer",
"version": "5.5.6",
"version": "5.5.7",
"author": "Elbert Alias",
"license": "GPL-3.0",
"repository": {

402
src/drivers/webextension/js/driver.js
Unescape View File

@ -3,20 +3,19 @@
*/
/* eslint-env browser */
/* global browser, chrome, fetch, Wappalyzer */
/* global browser, fetch, Wappalyzer */
/** global: browser */
/** global: chrome */
/** global: fetch */
/** global: Wappalyzer */
const wappalyzer = new Wappalyzer();
const tabCache = {};
let categoryOrder = [];
const options = {};
const robotsTxtQueue = {};
let categoryOrder = [];
browser.tabs.onRemoved.addListener((tabId) => {
tabCache[tabId] = null;
});
@ -26,17 +25,19 @@ browser.tabs.onRemoved.addListener((tabId) => {
*/
function getOption(name, defaultValue = null) {
return new Promise(async (resolve) => {
let option = defaultValue;
let value = defaultValue;
try {
option = await browser.storage.local.get(name);
const option = await browser.storage.local.get(name);
if (option[name]) {
value = option[name];
}
} catch (error) {
wappalyzer.log(error, 'driver', 'error');
}
options[name] = option;
resolve(option);
resolve(value);
});
}
@ -44,13 +45,15 @@ function getOption(name, defaultValue = null) {
* Set a value in localStorage
*/
function setOption(name, value) {
const option = {};
option[name] = value;
browser.storage.local.set(option);
return new Promise(async (resolve) => {
try {
await browser.storage.local.set({ [name]: value });
} catch (error) {
wappalyzer.log(error, 'driver', 'error');
}
options[name] = value;
resolve();
});
}
/**
@ -66,159 +69,110 @@ function openTab(args) {
/**
* Make a POST request
*/
function post(url, body) {
fetch(url, {
method: 'POST',
body: JSON.stringify(body),
})
.then(response => wappalyzer.log(`POST ${url}: ${response.status}`, 'driver'))
.catch(error => wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error'));
async function post(url, body) {
try {
const response = await fetch(url, {
method: 'POST',
body: JSON.stringify(body),
});
wappalyzer.log(`POST ${url}: ${response.status}`, 'driver');
} catch (error) {
wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error');
}
}
fetch('../apps.json')
.then(response => response.json())
.then((json) => {
wappalyzer.apps = json.apps;
wappalyzer.categories = json.categories;
// Capture response headers
browser.webRequest.onCompleted.addListener(async (request) => {
const headers = {};
wappalyzer.parseJsPatterns();
if (request.responseHeaders) {
const url = wappalyzer.parseUrl(request.url);
categoryOrder = Object.keys(wappalyzer.categories)
.map(categoryId => parseInt(categoryId, 10))
.sort((a, b) => wappalyzer.categories[a].priority - wappalyzer.categories[b].priority);
})
.catch(error => wappalyzer.log(`GET apps.json: ${error}`, 'driver', 'error'));
let tab;
// Version check
const { version } = browser.runtime.getManifest();
try {
[tab] = await browser.tabs.query({ url: [url.href] });
} catch (error) {
wappalyzer.log(error, 'driver', 'error');
}
getOption('version')
.then((previousVersion) => {
if (previousVersion === null) {
openTab({
url: `${wappalyzer.config.websiteURL}installed`,
if (tab) {
request.responseHeaders.forEach((header) => {
const name = header.name.toLowerCase();
headers[name] = headers[name] || [];
headers[name].push((header.value || header.binaryValue || '').toString());
});
} else if (version !== previousVersion) {
getOption('upgradeMessage', true)
.then((upgradeMessage) => {
if (upgradeMessage) {
openTab({
url: `${wappalyzer.config.websiteURL}upgraded?v${version}`,
background: true,
});
}
});
if (headers['content-type'] && /\/x?html/.test(headers['content-type'][0])) {
wappalyzer.analyze(url, { headers }, { tab });
}
}
}
}, { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, ['responseHeaders']);
setOption('version', version);
});
// Listen for messages
browser.runtime.onMessage.addListener(async (message, sender) => {
if (message.id === undefined) {
return Promise.resolve();
}
getOption('dynamicIcon', true);
getOption('pinnedCategory');
if (message.id !== 'log') {
wappalyzer.log(`Message${message.source ? ` from ${message.source}` : ''}: ${message.id}`, 'driver');
}
getOption('hostnameCache', {})
.then((hostnameCache) => {
wappalyzer.hostnameCache = hostnameCache;
const pinnedCategory = await getOption('pinnedCategory');
return hostnameCache;
});
const url = wappalyzer.parseUrl(sender.tab ? sender.tab.url : '');
// Run content script on all tabs
browser.tabs.query({ url: ['http://*/*', 'https://*/*'] })
.then((tabs) => {
tabs.forEach((tab) => {
browser.tabs.executeScript(tab.id, {
file: '../js/content.js',
});
});
})
.catch(error => wappalyzer.log(error, 'driver', 'error'));
const cookies = await browser.cookies.getAll({ domain: `.${url.hostname}` });
// Capture response headers
browser.webRequest.onCompleted.addListener((request) => {
const headers = {};
let response;
if (request.responseHeaders) {
const url = wappalyzer.parseUrl(request.url);
switch (message.id) {
case 'log':
wappalyzer.log(message.subject, message.source);
browser.tabs.query({ url: [url.href] })
.then((tabs) => {
const tab = tabs[0] || null;
break;
case 'init':
wappalyzer.analyze(url, { cookies }, { tab: sender.tab });
if (tab) {
request.responseHeaders.forEach((header) => {
const name = header.name.toLowerCase();
break;
case 'analyze':
wappalyzer.analyze(url, message.subject, { tab: sender.tab });
headers[name] = headers[name] || [];
await setOption('hostnameCache', wappalyzer.hostnameCache);
headers[name].push((header.value || header.binaryValue || '').toString());
});
break;
case 'ad_log':
wappalyzer.cacheDetectedAds(message.subject);
if (headers['content-type'] && /\/x?html/.test(headers['content-type'][0])) {
wappalyzer.analyze(url, { headers }, { tab });
}
}
})
.catch(error => wappalyzer.log(error, 'driver', 'error'));
}
}, { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, ['responseHeaders']);
break;
case 'get_apps':
response = {
tabCache: tabCache[message.tab.id],
apps: wappalyzer.apps,
categories: wappalyzer.categories,
pinnedCategory,
};
// Listen for messages
(chrome || browser).runtime.onMessage.addListener((message, sender, sendResponse) => {
if (typeof message.id !== 'undefined') {
if (message.id !== 'log') {
wappalyzer.log(`Message${message.source ? ` from ${message.source}` : ''}: ${message.id}`, 'driver');
}
break;
case 'set_option':
await setOption(message.key, message.value);
const url = wappalyzer.parseUrl(sender.tab ? sender.tab.url : '');
let response;
break;
case 'get_js_patterns':
response = {
patterns: wappalyzer.jsPatterns,
};
switch (message.id) {
case 'log':
wappalyzer.log(message.subject, message.source);
break;
case 'init':
browser.cookies.getAll({ domain: `.${url.hostname}` })
.then(cookies => wappalyzer.analyze(url, { cookies }, { tab: sender.tab }));
break;
case 'analyze':
wappalyzer.analyze(url, message.subject, { tab: sender.tab });
setOption('hostnameCache', wappalyzer.hostnameCache);
break;
case 'ad_log':
wappalyzer.cacheDetectedAds(message.subject);
break;
case 'get_apps':
response = {
tabCache: tabCache[message.tab.id],
apps: wappalyzer.apps,
categories: wappalyzer.categories,
pinnedCategory: options.pinnedCategory,
};
break;
case 'set_option':
setOption(message.key, message.value);
break;
case 'get_js_patterns':
response = {
patterns: wappalyzer.jsPatterns,
};
break;
default:
}
sendResponse(response);
break;
default:
}
return true;
return Promise.resolve(response);
});
wappalyzer.driver.document = document;
@ -227,13 +181,15 @@ wappalyzer.driver.document = document;
* Log messages to console
*/
wappalyzer.driver.log = (message, source, type) => {
console.log(`[wappalyzer ${type}]`, `[${source}]`, message);
const log = ['warn', 'error'].indexOf(type) !== -1 ? type : 'log';
console[log](`[wappalyzer ${type}]`, `[${source}]`, message);
};
/**
* Display apps
*/
wappalyzer.driver.displayApps = (detected, meta, context) => {
wappalyzer.driver.displayApps = async (detected, meta, context) => {
const { tab } = context;
if (tab === undefined) {
@ -246,20 +202,19 @@ wappalyzer.driver.displayApps = (detected, meta, context) => {
tabCache[tab.id].detected = detected;
const pinnedCategory = await getOption('pinnedCategory');
const dynamicIcon = await getOption('dynamicIcon', true);
let found = false;
// Find the main application to display
[options.pinnedCategory].concat(categoryOrder).forEach((match) => {
[pinnedCategory].concat(categoryOrder).forEach((match) => {
Object.keys(detected).forEach((appName) => {
const app = detected[appName];
app.props.cats.forEach((category) => {
if (category === match && !found) {
let icon = app.props.icon || 'default.svg';
if (!options.dynamicIcon) {
icon = 'default.svg';
}
let icon = app.props.icon && dynamicIcon ? app.props.icon : 'default.svg';
if (/\.svg$/i.test(icon)) {
icon = `converted/${icon.replace(/\.svg$/, '.png')}`;
@ -280,61 +235,53 @@ wappalyzer.driver.displayApps = (detected, meta, context) => {
});
});
if (typeof chrome !== 'undefined') {
// Browser polyfill doesn't seem to work here
chrome.pageAction.show(tab.id);
} else {
browser.pageAction.show(tab.id);
}
browser.pageAction.show(tab.id);
};
/**
* Fetch and cache robots.txt for host
*/
wappalyzer.driver.getRobotsTxt = (host, secure = false) => {
wappalyzer.driver.getRobotsTxt = async (host, secure = false) => {
if (robotsTxtQueue[host]) {
return robotsTxtQueue[host];
}
robotsTxtQueue[host] = new Promise((resolve) => {
getOption('tracking', true)
.then((tracking) => {
if (!tracking) {
resolve([]);
const tracking = await getOption('tracking', true);
const robotsTxtCache = await getOption('robotsTxtCache', {});
return;
}
robotsTxtQueue[host] = new Promise(async (resolve) => {
if (!tracking) {
return resolve([]);
}
if (host in robotsTxtCache) {
return resolve(robotsTxtCache[host]);
}
getOption('robotsTxtCache')
.then((robotsTxtCache) => {
robotsTxtCache = robotsTxtCache || {};
const timeout = setTimeout(() => resolve([]), 3000);
if (host in robotsTxtCache) {
resolve(robotsTxtCache[host]);
let response;
return;
}
try {
response = await fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { redirect: 'follow' });
} catch (error) {
wappalyzer.log(error, 'driver', 'error');
const timeout = setTimeout(() => resolve([]), 3000);
return resolve([]);
}
fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { redirect: 'follow' })
.then((response) => {
clearTimeout(timeout);
clearTimeout(timeout);
return response.ok ? response.text() : '';
})
.then((robotsTxt) => {
robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt);
const robotsTxt = response.ok ? await response.text() : '';
setOption('robotsTxtCache', robotsTxtCache);
robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt);
resolve(robotsTxtCache[host]);
})
.catch(() => resolve([]));
});
});
})
.finally(() => delete robotsTxtQueue[host]);
await setOption('robotsTxtCache', robotsTxtCache);
delete robotsTxtQueue[host];
return resolve(robotsTxtCache[host]);
});
return robotsTxtQueue[host];
};
@ -342,19 +289,72 @@ wappalyzer.driver.getRobotsTxt = (host, secure = false) => {
/**
* Anonymously track detected applications for research purposes
*/
wappalyzer.driver.ping = (hostnameCache = {}, adCache = []) => {
getOption('tracking', true)
.then((tracking) => {
if (tracking) {
if (Object.keys(hostnameCache).length) {
post('https://api.wappalyzer.com/ping/v1/', hostnameCache);
}
wappalyzer.driver.ping = async (hostnameCache = {}, adCache = []) => {
const tracking = await getOption('tracking', true);
if (adCache.length) {
post('https://ad.wappalyzer.com/log/wp/', adCache);
}
if (tracking) {
if (Object.keys(hostnameCache).length) {
post('https://api.wappalyzer.com/ping/v1/', hostnameCache);
}
setOption('robotsTxtCache', {});
}
});
if (adCache.length) {
post('https://ad.wappalyzer.com/log/wp/', adCache);
}
await setOption('robotsTxtCache', {});
}
};
// Init
(async () => {
// Technologies
try {
const response = await fetch('../apps.json');
const json = await response.json();
wappalyzer.apps = json.apps;
wappalyzer.categories = json.categories;
} catch (error) {
wappalyzer.log(`GET apps.json: ${error.message}`, 'driver', 'error');
}
wappalyzer.parseJsPatterns();
categoryOrder = Object.keys(wappalyzer.categories)
.map(categoryId => parseInt(categoryId, 10))
.sort((a, b) => wappalyzer.categories[a].priority - wappalyzer.categories[b].priority);
// Version check
const { version } = browser.runtime.getManifest();
const previousVersion = await getOption('version');
const upgradeMessage = await getOption('upgradeMessage', true);
if (previousVersion === null) {
openTab({
url: `${wappalyzer.config.websiteURL}installed`,
});
} else if (version !== previousVersion && upgradeMessage) {
//openTab({
// url: `${wappalyzer.config.websiteURL}upgraded?v${version}`,
// background: true,
//});
}
await setOption('version', version);
// Hostname cache
wappalyzer.hostnameCache = await getOption('hostnameCache', {});
// Run content script on all tabs
try {
const tabs = await browser.tabs.query({ url: ['http://*/*', 'https://*/*'] });
tabs.forEach((tab) => {
browser.tabs.executeScript(tab.id, {
file: '../js/content.js',
});
});
} catch (error) {
wappalyzer.log(error, 'driver', 'error');
}
})();

61
src/drivers/webextension/js/popup.js
Unescape View File

@ -1,32 +1,14 @@
/* eslint-env browser */
/* global browser, chrome, jsonToDOM */
/** global: chrome */
/** global: browser */
/** global: jsonToDOM */
let pinnedCategory = null;
const func = (tabs) => {
(chrome || browser).runtime.sendMessage({
id: 'get_apps',
tab: tabs[0],
source: 'popup.js',
}, (response) => {
pinnedCategory = response.pinnedCategory;
replaceDomWhenReady(appsToDomTemplate(response));
});
};
browser.tabs.query({ active: true, currentWindow: true })
.then(func)
.catch(console.error);
function replaceDomWhenReady(dom) {
if (/complete|interactive|loaded/.test(document.readyState)) {
replaceDom(dom);
} else {
document.addEventListener('DOMContentLoaded', () => {
replaceDom(dom);
});
}
function slugify(string) {
return string.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/--+/g, '-').replace(/(?:^-|-$)/, '');
}
function replaceDom(domTemplate) {
@ -73,6 +55,16 @@ function replaceDom(domTemplate) {
});
}
function replaceDomWhenReady(dom) {
if (/complete|interactive|loaded/.test(document.readyState)) {
replaceDom(dom);
} else {
document.addEventListener('DOMContentLoaded', () => {
replaceDom(dom);
});
}
}
function appsToDomTemplate(response) {
let template = [];
@ -92,8 +84,7 @@ function appsToDomTemplate(response) {
const apps = [];
for (const appName in categories[cat].apps) {
const confidence = response.tabCache.detected[appName].confidenceTotal;
const version = response.tabCache.detected[appName].version;
const { confidence, version } = response.tabCache.detected[appName];
apps.push(
[
@ -190,6 +181,18 @@ function appsToDomTemplate(response) {
return template;
}
function slugify(string) {
return string.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/--+/g, '-').replace(/(?:^-|-$)/, '');
}
const func = (tabs) => {
(chrome || browser).runtime.sendMessage({
id: 'get_apps',
tab: tabs[0],
source: 'popup.js',
}, (response) => {
pinnedCategory = response.pinnedCategory;
replaceDomWhenReady(appsToDomTemplate(response));
});
};
browser.tabs.query({ active: true, currentWindow: true })
.then(func)
.catch(console.error);

151
src/drivers/webextension/manifest.json
Unescape View File

@ -1,78 +1,83 @@
{
"name": "Wappalyzer",
"short_name": "Wappalyzer",
"author": "Elbert Alias",
"homepage_url": "https://www.wappalyzer.com",
"description": "Identify web technologies",
"version": "5.5.6",
"default_locale": "en",
"manifest_version": 2,
"icons": {
"16": "images/icon_16.png",
"19": "images/icon_19.png",
"32": "images/icon_32.png",
"38": "images/icon_38.png",
"128": "images/icon_128.png"
},
"page_action": {
"default_icon": {
"16": "images/icon_16.png",
"19": "images/icon_19.png",
"32": "images/icon_32.png",
"38": "images/icon_38.png",
"128": "images/icon_128.png"
},
"default_title": "Wappalyzer",
"default_popup": "html/popup.html"
},
"background": {
"page": "html/background.html"
"name": "Wappalyzer",
"short_name": "Wappalyzer",
"author": "Elbert Alias",
"homepage_url": "https://www.wappalyzer.com",
"description": "Identify web technologies",
"version": "5.5.7",
"default_locale": "en",
"manifest_version": 2,
"icons": {
"16": "images/icon_16.png",
"19": "images/icon_19.png",
"32": "images/icon_32.png",
"38": "images/icon_38.png",
"128": "images/icon_128.png"
},
"page_action": {
"default_icon": {
"16": "images/icon_16.png",
"19": "images/icon_19.png",
"32": "images/icon_32.png",
"38": "images/icon_38.png",
"128": "images/icon_128.png"
},
"default_title": "Wappalyzer",
"default_popup": "html/popup.html"
},
"background": {
"page": "html/background.html"
},
"content_scripts": [
{
"matches": [
"http://*/*",
"https://*/*"
],
"js": [
"node_modules/webextension-polyfill/dist/browser-polyfill.js",
"js/content.js"
],
"run_at": "document_idle"
},
{
"matches": [
},
"content_scripts": [
{
"matches": [
"http://*/*",
"https://*/*"
],
"exclude_matches": [
"https://*.modirum.com/*",
"https://www.alphaecommerce.gr/*"
],
"js": [
"js/lib/iframe.js"
],
"run_at": "document_start",
"all_frames": true
}
],
"web_accessible_resources": [
"js/inject.js"
],
"options_page": "html/options.html",
"options_ui": {
"page": "html/options.html",
"open_in_tab": false
},
"permissions": [
"https://*/*"
],
"js": [
"node_modules/webextension-polyfill/dist/browser-polyfill.js",
"js/content.js"
],
"run_at": "document_idle"
},
{
"matches": [
"http://*/*",
"https://*/*"
],
"exclude_matches": [
"https://*.modirum.com/*",
"https://www.alphaecommerce.gr/*"
],
"js": [
"js/lib/iframe.js"
],
"run_at": "document_start",
"all_frames": true
}
],
"web_accessible_resources": [
"js/inject.js"
],
"options_ui": {
"page": "html/options.html",
"open_in_tab": false
},
"permissions": [
"cookies",
"storage",
"tabs",
"webRequest",
"webNavigation",
"http://*/*",
"https://*/*"
],
"content_security_policy": "script-src 'self'; object-src 'self'"
"storage",
"tabs",
"webRequest",
"webNavigation",
"http://*/*",
"https://*/*"
],
"content_security_policy": "script-src 'self'; object-src 'self'",
"applications": {
"gecko": {
"id": "wappalyzer@crunchlabz.com",
"strict_min_version": "42.0"
}
}
}

6
src/drivers/webextension/npm-shrinkwrap.json generated
Unescape View File

@ -3,9 +3,9 @@
"lockfileVersion": 1,
"dependencies": {
"webextension-polyfill": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/webextension-polyfill/-/webextension-polyfill-0.2.1.tgz",
"integrity": "sha1-zfyRJgMwOfFxNVMVfTW+/x1Nb0o="
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/webextension-polyfill/-/webextension-polyfill-0.3.1.tgz",
"integrity": "sha512-ISB42vlgMyM7xE1u6pREeCqmmXjLsYu/nqAR8Dl/gIAnylb+KpRpvKbVkUYNFePhhXn0Obkkc3jasOII9ztUtg=="
}
}
}

2
src/drivers/webextension/package.json
Unescape View File

@ -1,5 +1,5 @@
{
"dependencies": {
"webextension-polyfill": "^0.2.1"
"webextension-polyfill": "^0.3.1"
}
}