From 879cb32d305e6fa7d7f27a2ace1604e1554593bd Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Sat, 8 Dec 2018 10:37:01 +1100 Subject: [PATCH] Fix options, replace Promises with async/await, build v5.5.7 --- src/drivers/npm/npm-shrinkwrap.json | 2 +- src/drivers/npm/package.json | 2 +- src/drivers/webextension/js/driver.js | 402 +++++++++---------- src/drivers/webextension/js/popup.js | 61 +-- src/drivers/webextension/manifest.json | 151 +++---- src/drivers/webextension/npm-shrinkwrap.json | 6 +- src/drivers/webextension/package.json | 2 +- 7 files changed, 317 insertions(+), 309 deletions(-) diff --git a/src/drivers/npm/npm-shrinkwrap.json b/src/drivers/npm/npm-shrinkwrap.json index ae0b7f7cc..e82437f27 100644 --- a/src/drivers/npm/npm-shrinkwrap.json +++ b/src/drivers/npm/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "wappalyzer", - "version": "5.5.5", + "version": "5.5.6", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index 7f0b16551..d6d3e197a 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -2,7 +2,7 @@ "name": "wappalyzer", "description": "Uncovers the technologies used on websites", "homepage": "https://github.com/AliasIO/Wappalyzer", - "version": "5.5.6", + "version": "5.5.7", "author": "Elbert Alias", "license": "GPL-3.0", "repository": { diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index f698fba57..0d3efcd40 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -3,20 +3,19 @@ */ /* eslint-env browser */ -/* global browser, chrome, fetch, Wappalyzer */ +/* global browser, fetch, Wappalyzer */ /** global: browser */ -/** global: chrome */ /** global: fetch */ /** global: Wappalyzer */ const wappalyzer = new Wappalyzer(); const tabCache = {}; -let categoryOrder = []; -const options = {}; const robotsTxtQueue = {}; +let categoryOrder = []; + browser.tabs.onRemoved.addListener((tabId) => { tabCache[tabId] = null; }); @@ -26,17 +25,19 @@ browser.tabs.onRemoved.addListener((tabId) => { */ function getOption(name, defaultValue = null) { return new Promise(async (resolve) => { - let option = defaultValue; + let value = defaultValue; try { - option = await browser.storage.local.get(name); + const option = await browser.storage.local.get(name); + + if (option[name]) { + value = option[name]; + } } catch (error) { wappalyzer.log(error, 'driver', 'error'); } - options[name] = option; - - resolve(option); + resolve(value); }); } @@ -44,13 +45,15 @@ function getOption(name, defaultValue = null) { * Set a value in localStorage */ function setOption(name, value) { - const option = {}; - - option[name] = value; - - browser.storage.local.set(option); + return new Promise(async (resolve) => { + try { + await browser.storage.local.set({ [name]: value }); + } catch (error) { + wappalyzer.log(error, 'driver', 'error'); + } - options[name] = value; + resolve(); + }); } /** @@ -66,159 +69,110 @@ function openTab(args) { /** * Make a POST request */ -function post(url, body) { - fetch(url, { - method: 'POST', - body: JSON.stringify(body), - }) - .then(response => wappalyzer.log(`POST ${url}: ${response.status}`, 'driver')) - .catch(error => wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error')); +async function post(url, body) { + try { + const response = await fetch(url, { + method: 'POST', + body: JSON.stringify(body), + }); + + wappalyzer.log(`POST ${url}: ${response.status}`, 'driver'); + } catch (error) { + wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error'); + } } -fetch('../apps.json') - .then(response => response.json()) - .then((json) => { - wappalyzer.apps = json.apps; - wappalyzer.categories = json.categories; +// Capture response headers +browser.webRequest.onCompleted.addListener(async (request) => { + const headers = {}; - wappalyzer.parseJsPatterns(); + if (request.responseHeaders) { + const url = wappalyzer.parseUrl(request.url); - categoryOrder = Object.keys(wappalyzer.categories) - .map(categoryId => parseInt(categoryId, 10)) - .sort((a, b) => wappalyzer.categories[a].priority - wappalyzer.categories[b].priority); - }) - .catch(error => wappalyzer.log(`GET apps.json: ${error}`, 'driver', 'error')); + let tab; -// Version check -const { version } = browser.runtime.getManifest(); + try { + [tab] = await browser.tabs.query({ url: [url.href] }); + } catch (error) { + wappalyzer.log(error, 'driver', 'error'); + } -getOption('version') - .then((previousVersion) => { - if (previousVersion === null) { - openTab({ - url: `${wappalyzer.config.websiteURL}installed`, + if (tab) { + request.responseHeaders.forEach((header) => { + const name = header.name.toLowerCase(); + + headers[name] = headers[name] || []; + + headers[name].push((header.value || header.binaryValue || '').toString()); }); - } else if (version !== previousVersion) { - getOption('upgradeMessage', true) - .then((upgradeMessage) => { - if (upgradeMessage) { - openTab({ - url: `${wappalyzer.config.websiteURL}upgraded?v${version}`, - background: true, - }); - } - }); + + if (headers['content-type'] && /\/x?html/.test(headers['content-type'][0])) { + wappalyzer.analyze(url, { headers }, { tab }); + } } + } +}, { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, ['responseHeaders']); - setOption('version', version); - }); +// Listen for messages +browser.runtime.onMessage.addListener(async (message, sender) => { + if (message.id === undefined) { + return Promise.resolve(); + } -getOption('dynamicIcon', true); -getOption('pinnedCategory'); + if (message.id !== 'log') { + wappalyzer.log(`Message${message.source ? ` from ${message.source}` : ''}: ${message.id}`, 'driver'); + } -getOption('hostnameCache', {}) - .then((hostnameCache) => { - wappalyzer.hostnameCache = hostnameCache; + const pinnedCategory = await getOption('pinnedCategory'); - return hostnameCache; - }); + const url = wappalyzer.parseUrl(sender.tab ? sender.tab.url : ''); -// Run content script on all tabs -browser.tabs.query({ url: ['http://*/*', 'https://*/*'] }) - .then((tabs) => { - tabs.forEach((tab) => { - browser.tabs.executeScript(tab.id, { - file: '../js/content.js', - }); - }); - }) - .catch(error => wappalyzer.log(error, 'driver', 'error')); + const cookies = await browser.cookies.getAll({ domain: `.${url.hostname}` }); -// Capture response headers -browser.webRequest.onCompleted.addListener((request) => { - const headers = {}; + let response; - if (request.responseHeaders) { - const url = wappalyzer.parseUrl(request.url); + switch (message.id) { + case 'log': + wappalyzer.log(message.subject, message.source); - browser.tabs.query({ url: [url.href] }) - .then((tabs) => { - const tab = tabs[0] || null; + break; + case 'init': + wappalyzer.analyze(url, { cookies }, { tab: sender.tab }); - if (tab) { - request.responseHeaders.forEach((header) => { - const name = header.name.toLowerCase(); + break; + case 'analyze': + wappalyzer.analyze(url, message.subject, { tab: sender.tab }); - headers[name] = headers[name] || []; + await setOption('hostnameCache', wappalyzer.hostnameCache); - headers[name].push((header.value || header.binaryValue || '').toString()); - }); + break; + case 'ad_log': + wappalyzer.cacheDetectedAds(message.subject); - if (headers['content-type'] && /\/x?html/.test(headers['content-type'][0])) { - wappalyzer.analyze(url, { headers }, { tab }); - } - } - }) - .catch(error => wappalyzer.log(error, 'driver', 'error')); - } -}, { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, ['responseHeaders']); + break; + case 'get_apps': + response = { + tabCache: tabCache[message.tab.id], + apps: wappalyzer.apps, + categories: wappalyzer.categories, + pinnedCategory, + }; -// Listen for messages -(chrome || browser).runtime.onMessage.addListener((message, sender, sendResponse) => { - if (typeof message.id !== 'undefined') { - if (message.id !== 'log') { - wappalyzer.log(`Message${message.source ? ` from ${message.source}` : ''}: ${message.id}`, 'driver'); - } + break; + case 'set_option': + await setOption(message.key, message.value); - const url = wappalyzer.parseUrl(sender.tab ? sender.tab.url : ''); - let response; + break; + case 'get_js_patterns': + response = { + patterns: wappalyzer.jsPatterns, + }; - switch (message.id) { - case 'log': - wappalyzer.log(message.subject, message.source); - - break; - case 'init': - browser.cookies.getAll({ domain: `.${url.hostname}` }) - .then(cookies => wappalyzer.analyze(url, { cookies }, { tab: sender.tab })); - - break; - case 'analyze': - wappalyzer.analyze(url, message.subject, { tab: sender.tab }); - - setOption('hostnameCache', wappalyzer.hostnameCache); - - break; - case 'ad_log': - wappalyzer.cacheDetectedAds(message.subject); - - break; - case 'get_apps': - response = { - tabCache: tabCache[message.tab.id], - apps: wappalyzer.apps, - categories: wappalyzer.categories, - pinnedCategory: options.pinnedCategory, - }; - - break; - case 'set_option': - setOption(message.key, message.value); - - break; - case 'get_js_patterns': - response = { - patterns: wappalyzer.jsPatterns, - }; - - break; - default: - } - - sendResponse(response); + break; + default: } - return true; + return Promise.resolve(response); }); wappalyzer.driver.document = document; @@ -227,13 +181,15 @@ wappalyzer.driver.document = document; * Log messages to console */ wappalyzer.driver.log = (message, source, type) => { - console.log(`[wappalyzer ${type}]`, `[${source}]`, message); + const log = ['warn', 'error'].indexOf(type) !== -1 ? type : 'log'; + + console[log](`[wappalyzer ${type}]`, `[${source}]`, message); }; /** * Display apps */ -wappalyzer.driver.displayApps = (detected, meta, context) => { +wappalyzer.driver.displayApps = async (detected, meta, context) => { const { tab } = context; if (tab === undefined) { @@ -246,20 +202,19 @@ wappalyzer.driver.displayApps = (detected, meta, context) => { tabCache[tab.id].detected = detected; + const pinnedCategory = await getOption('pinnedCategory'); + const dynamicIcon = await getOption('dynamicIcon', true); + let found = false; // Find the main application to display - [options.pinnedCategory].concat(categoryOrder).forEach((match) => { + [pinnedCategory].concat(categoryOrder).forEach((match) => { Object.keys(detected).forEach((appName) => { const app = detected[appName]; app.props.cats.forEach((category) => { if (category === match && !found) { - let icon = app.props.icon || 'default.svg'; - - if (!options.dynamicIcon) { - icon = 'default.svg'; - } + let icon = app.props.icon && dynamicIcon ? app.props.icon : 'default.svg'; if (/\.svg$/i.test(icon)) { icon = `converted/${icon.replace(/\.svg$/, '.png')}`; @@ -280,61 +235,53 @@ wappalyzer.driver.displayApps = (detected, meta, context) => { }); }); - if (typeof chrome !== 'undefined') { - // Browser polyfill doesn't seem to work here - chrome.pageAction.show(tab.id); - } else { - browser.pageAction.show(tab.id); - } + browser.pageAction.show(tab.id); }; /** * Fetch and cache robots.txt for host */ -wappalyzer.driver.getRobotsTxt = (host, secure = false) => { +wappalyzer.driver.getRobotsTxt = async (host, secure = false) => { if (robotsTxtQueue[host]) { return robotsTxtQueue[host]; } - robotsTxtQueue[host] = new Promise((resolve) => { - getOption('tracking', true) - .then((tracking) => { - if (!tracking) { - resolve([]); + const tracking = await getOption('tracking', true); + const robotsTxtCache = await getOption('robotsTxtCache', {}); - return; - } + robotsTxtQueue[host] = new Promise(async (resolve) => { + if (!tracking) { + return resolve([]); + } + + if (host in robotsTxtCache) { + return resolve(robotsTxtCache[host]); + } - getOption('robotsTxtCache') - .then((robotsTxtCache) => { - robotsTxtCache = robotsTxtCache || {}; + const timeout = setTimeout(() => resolve([]), 3000); - if (host in robotsTxtCache) { - resolve(robotsTxtCache[host]); + let response; - return; - } + try { + response = await fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { redirect: 'follow' }); + } catch (error) { + wappalyzer.log(error, 'driver', 'error'); - const timeout = setTimeout(() => resolve([]), 3000); + return resolve([]); + } - fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { redirect: 'follow' }) - .then((response) => { - clearTimeout(timeout); + clearTimeout(timeout); - return response.ok ? response.text() : ''; - }) - .then((robotsTxt) => { - robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt); + const robotsTxt = response.ok ? await response.text() : ''; - setOption('robotsTxtCache', robotsTxtCache); + robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt); - resolve(robotsTxtCache[host]); - }) - .catch(() => resolve([])); - }); - }); - }) - .finally(() => delete robotsTxtQueue[host]); + await setOption('robotsTxtCache', robotsTxtCache); + + delete robotsTxtQueue[host]; + + return resolve(robotsTxtCache[host]); + }); return robotsTxtQueue[host]; }; @@ -342,19 +289,72 @@ wappalyzer.driver.getRobotsTxt = (host, secure = false) => { /** * Anonymously track detected applications for research purposes */ -wappalyzer.driver.ping = (hostnameCache = {}, adCache = []) => { - getOption('tracking', true) - .then((tracking) => { - if (tracking) { - if (Object.keys(hostnameCache).length) { - post('https://api.wappalyzer.com/ping/v1/', hostnameCache); - } +wappalyzer.driver.ping = async (hostnameCache = {}, adCache = []) => { + const tracking = await getOption('tracking', true); - if (adCache.length) { - post('https://ad.wappalyzer.com/log/wp/', adCache); - } + if (tracking) { + if (Object.keys(hostnameCache).length) { + post('https://api.wappalyzer.com/ping/v1/', hostnameCache); + } - setOption('robotsTxtCache', {}); - } - }); + if (adCache.length) { + post('https://ad.wappalyzer.com/log/wp/', adCache); + } + + await setOption('robotsTxtCache', {}); + } }; + +// Init +(async () => { + // Technologies + try { + const response = await fetch('../apps.json'); + const json = await response.json(); + + wappalyzer.apps = json.apps; + wappalyzer.categories = json.categories; + } catch (error) { + wappalyzer.log(`GET apps.json: ${error.message}`, 'driver', 'error'); + } + + wappalyzer.parseJsPatterns(); + + categoryOrder = Object.keys(wappalyzer.categories) + .map(categoryId => parseInt(categoryId, 10)) + .sort((a, b) => wappalyzer.categories[a].priority - wappalyzer.categories[b].priority); + + // Version check + const { version } = browser.runtime.getManifest(); + const previousVersion = await getOption('version'); + const upgradeMessage = await getOption('upgradeMessage', true); + + if (previousVersion === null) { + openTab({ + url: `${wappalyzer.config.websiteURL}installed`, + }); + } else if (version !== previousVersion && upgradeMessage) { + //openTab({ + // url: `${wappalyzer.config.websiteURL}upgraded?v${version}`, + // background: true, + //}); + } + + await setOption('version', version); + + // Hostname cache + wappalyzer.hostnameCache = await getOption('hostnameCache', {}); + + // Run content script on all tabs + try { + const tabs = await browser.tabs.query({ url: ['http://*/*', 'https://*/*'] }); + + tabs.forEach((tab) => { + browser.tabs.executeScript(tab.id, { + file: '../js/content.js', + }); + }); + } catch (error) { + wappalyzer.log(error, 'driver', 'error'); + } +})(); diff --git a/src/drivers/webextension/js/popup.js b/src/drivers/webextension/js/popup.js index e560162fd..61e21ec21 100644 --- a/src/drivers/webextension/js/popup.js +++ b/src/drivers/webextension/js/popup.js @@ -1,32 +1,14 @@ +/* eslint-env browser */ +/* global browser, chrome, jsonToDOM */ + /** global: chrome */ /** global: browser */ +/** global: jsonToDOM */ let pinnedCategory = null; -const func = (tabs) => { - (chrome || browser).runtime.sendMessage({ - id: 'get_apps', - tab: tabs[0], - source: 'popup.js', - }, (response) => { - pinnedCategory = response.pinnedCategory; - - replaceDomWhenReady(appsToDomTemplate(response)); - }); -}; - -browser.tabs.query({ active: true, currentWindow: true }) - .then(func) - .catch(console.error); - -function replaceDomWhenReady(dom) { - if (/complete|interactive|loaded/.test(document.readyState)) { - replaceDom(dom); - } else { - document.addEventListener('DOMContentLoaded', () => { - replaceDom(dom); - }); - } +function slugify(string) { + return string.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/--+/g, '-').replace(/(?:^-|-$)/, ''); } function replaceDom(domTemplate) { @@ -73,6 +55,16 @@ function replaceDom(domTemplate) { }); } +function replaceDomWhenReady(dom) { + if (/complete|interactive|loaded/.test(document.readyState)) { + replaceDom(dom); + } else { + document.addEventListener('DOMContentLoaded', () => { + replaceDom(dom); + }); + } +} + function appsToDomTemplate(response) { let template = []; @@ -92,8 +84,7 @@ function appsToDomTemplate(response) { const apps = []; for (const appName in categories[cat].apps) { - const confidence = response.tabCache.detected[appName].confidenceTotal; - const version = response.tabCache.detected[appName].version; + const { confidence, version } = response.tabCache.detected[appName]; apps.push( [ @@ -190,6 +181,18 @@ function appsToDomTemplate(response) { return template; } -function slugify(string) { - return string.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/--+/g, '-').replace(/(?:^-|-$)/, ''); -} +const func = (tabs) => { + (chrome || browser).runtime.sendMessage({ + id: 'get_apps', + tab: tabs[0], + source: 'popup.js', + }, (response) => { + pinnedCategory = response.pinnedCategory; + + replaceDomWhenReady(appsToDomTemplate(response)); + }); +}; + +browser.tabs.query({ active: true, currentWindow: true }) + .then(func) + .catch(console.error); diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json index f24911431..3e3021605 100644 --- a/src/drivers/webextension/manifest.json +++ b/src/drivers/webextension/manifest.json @@ -1,78 +1,83 @@ { - "name": "Wappalyzer", - "short_name": "Wappalyzer", - "author": "Elbert Alias", - "homepage_url": "https://www.wappalyzer.com", - "description": "Identify web technologies", - "version": "5.5.6", - "default_locale": "en", - "manifest_version": 2, - "icons": { - "16": "images/icon_16.png", - "19": "images/icon_19.png", - "32": "images/icon_32.png", - "38": "images/icon_38.png", - "128": "images/icon_128.png" - }, - "page_action": { - "default_icon": { - "16": "images/icon_16.png", - "19": "images/icon_19.png", - "32": "images/icon_32.png", - "38": "images/icon_38.png", - "128": "images/icon_128.png" - }, - "default_title": "Wappalyzer", - "default_popup": "html/popup.html" - }, - "background": { - "page": "html/background.html" + "name": "Wappalyzer", + "short_name": "Wappalyzer", + "author": "Elbert Alias", + "homepage_url": "https://www.wappalyzer.com", + "description": "Identify web technologies", + "version": "5.5.7", + "default_locale": "en", + "manifest_version": 2, + "icons": { + "16": "images/icon_16.png", + "19": "images/icon_19.png", + "32": "images/icon_32.png", + "38": "images/icon_38.png", + "128": "images/icon_128.png" + }, + "page_action": { + "default_icon": { + "16": "images/icon_16.png", + "19": "images/icon_19.png", + "32": "images/icon_32.png", + "38": "images/icon_38.png", + "128": "images/icon_128.png" + }, + "default_title": "Wappalyzer", + "default_popup": "html/popup.html" + }, + "background": { + "page": "html/background.html" - }, - "content_scripts": [ - { - "matches": [ - "http://*/*", - "https://*/*" - ], - "js": [ - "node_modules/webextension-polyfill/dist/browser-polyfill.js", - "js/content.js" - ], - "run_at": "document_idle" - }, - { - "matches": [ + }, + "content_scripts": [ + { + "matches": [ "http://*/*", - "https://*/*" - ], - "exclude_matches": [ - "https://*.modirum.com/*", - "https://www.alphaecommerce.gr/*" - ], - "js": [ - "js/lib/iframe.js" - ], - "run_at": "document_start", - "all_frames": true - } - ], - "web_accessible_resources": [ - "js/inject.js" - ], - "options_page": "html/options.html", - "options_ui": { - "page": "html/options.html", - "open_in_tab": false - }, - "permissions": [ + "https://*/*" + ], + "js": [ + "node_modules/webextension-polyfill/dist/browser-polyfill.js", + "js/content.js" + ], + "run_at": "document_idle" + }, + { + "matches": [ + "http://*/*", + "https://*/*" + ], + "exclude_matches": [ + "https://*.modirum.com/*", + "https://www.alphaecommerce.gr/*" + ], + "js": [ + "js/lib/iframe.js" + ], + "run_at": "document_start", + "all_frames": true + } + ], + "web_accessible_resources": [ + "js/inject.js" + ], + "options_ui": { + "page": "html/options.html", + "open_in_tab": false + }, + "permissions": [ "cookies", - "storage", - "tabs", - "webRequest", - "webNavigation", - "http://*/*", - "https://*/*" - ], - "content_security_policy": "script-src 'self'; object-src 'self'" + "storage", + "tabs", + "webRequest", + "webNavigation", + "http://*/*", + "https://*/*" + ], + "content_security_policy": "script-src 'self'; object-src 'self'", + "applications": { + "gecko": { + "id": "wappalyzer@crunchlabz.com", + "strict_min_version": "42.0" + } + } } diff --git a/src/drivers/webextension/npm-shrinkwrap.json b/src/drivers/webextension/npm-shrinkwrap.json index ebcf0d9eb..7dfcf7d96 100644 --- a/src/drivers/webextension/npm-shrinkwrap.json +++ b/src/drivers/webextension/npm-shrinkwrap.json @@ -3,9 +3,9 @@ "lockfileVersion": 1, "dependencies": { "webextension-polyfill": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/webextension-polyfill/-/webextension-polyfill-0.2.1.tgz", - "integrity": "sha1-zfyRJgMwOfFxNVMVfTW+/x1Nb0o=" + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/webextension-polyfill/-/webextension-polyfill-0.3.1.tgz", + "integrity": "sha512-ISB42vlgMyM7xE1u6pREeCqmmXjLsYu/nqAR8Dl/gIAnylb+KpRpvKbVkUYNFePhhXn0Obkkc3jasOII9ztUtg==" } } } diff --git a/src/drivers/webextension/package.json b/src/drivers/webextension/package.json index 9b344ae21..19c18bd9a 100644 --- a/src/drivers/webextension/package.json +++ b/src/drivers/webextension/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "webextension-polyfill": "^0.2.1" + "webextension-polyfill": "^0.3.1" } }