Refactoring

main
Elbert Alias 5 years ago
parent 2a54f1cd4f
commit 575809793e

@ -150,6 +150,21 @@ a:hover {
color: var(--color-text);
}
.technology__confidence {
opacity: .5;
font-size: .7rem;
margin-left: .2rem;
}
.technology__version {
background: var(--color-secondary);
border-radius: 3px;
font-size: .7rem;
padding: .1rem .3rem;
margin-left: .4rem;
vertical-align: middle;
}
.terms {
align-items: center;
display: flex;
@ -235,6 +250,13 @@ a:hover {
color: #fff
}
.theme-mode .technology__confidence {
}
.theme-mode .technology__version {
background: var(--color-primary);
}
.theme-mode .footer {
border-color: var(--color-secondary-dark)
}

@ -55,8 +55,11 @@
<a class="technology__link" href="#"></a>
<span class="technology__confidence">&nbsp;</span>
<span>
<span class="technology__version">&nbsp;</span>
</span>
<span class="technology__confidence">&nbsp;</span>
</div>
</div>

@ -25,6 +25,19 @@ const Content = {
html = chunks.join('\n')
const language =
document.documentElement.getAttribute('lang') ||
document.documentElement.getAttribute('xml:lang') ||
(await new Promise((resolve) =>
chrome.i18n.detectLanguage(html, ({ languages }) =>
resolve(
languages
.filter(({ percentage }) => percentage >= 75)
.map(({ language: lang }) => lang)[0]
)
)
))
// Script tags
const scripts = Array.from(document.scripts)
.filter(({ src }) => src)
@ -41,7 +54,7 @@ const Content = {
Content.port.postMessage({
func: 'onContentLoad',
args: [location.href, { html, scripts, meta }]
args: [location.href, { html, scripts, meta }, language]
})
Content.port.postMessage({ func: 'getTechnologies' })

@ -7,15 +7,61 @@ const {
setCategories,
analyze,
analyzeManyToMany,
resolve,
unique
resolve
} = Wappalyzer
const { promisify, getOption } = Utils
const { agent, promisify, getOption, setOption } = Utils
const expiry = 1000 * 60 * 60 * 24
const Driver = {
cache: {
hostnames: {},
robots: {}
lastPing: Date.now(),
async init() {
await Driver.loadTechnologies()
const hostnameCache = (await getOption('hostnames')) || {}
Driver.cache = {
hostnames: Object.keys(hostnameCache).reduce(
(cache, hostname) => ({
...cache,
[hostname]: {
...hostnameCache[hostname],
detections: hostnameCache[hostname].detections.map(
({
pattern: { regex, confidence, version },
match,
technology: name,
hits
}) => ({
pattern: {
regex: new RegExp(regex, 'i'),
confidence,
version
},
match,
technology: Wappalyzer.technologies.find(
({ name: _name }) => name === _name
),
hits
})
)
}
}),
{}
),
tabs: {},
robots: (await getOption('robots')) || {},
ads: (await getOption('ads')) || []
}
chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect)
chrome.webRequest.onCompleted.addListener(
Driver.onWebRequestComplete,
{ urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] },
['responseHeaders']
)
chrome.tabs.onRemoved.addListener((id) => (Driver.cache.tabs[id] = null))
},
log(message, source = 'driver', type = 'log') {
@ -97,90 +143,6 @@ const Driver = {
func,
args: await Driver[func].call(port.sender, ...(args || []))
})
/*
const pinnedCategory = await getOption('pinnedCategory')
const url = new URL(port.sender.tab.url)
const cookies = await browser.cookies.getAll({
domain: `.${url.hostname}`
})
let response
switch (message.id) {
case 'log':
wappalyzer.log(message.subject, message.source)
break
case 'analyze':
if (message.subject.html) {
browser.i18n
.detectLanguage(message.subject.html)
.then(({ languages }) => {
const language = languages
.filter(({ percentage }) => percentage >= 75)
.map(({ language: lang }) => lang)[0]
message.subject.language = language
wappalyzer.analyze(url, message.subject, {
tab: port.sender.tab
})
})
} else {
wappalyzer.analyze(url, message.subject, { tab: port.sender.tab })
}
await setOption('hostnameCache', wappalyzer.hostnameCache)
break
case 'ad_log':
wappalyzer.cacheDetectedAds(message.subject)
break
case 'get_apps':
response = {
tabCache: tabCache[message.tab.id],
apps: wappalyzer.apps,
categories: wappalyzer.categories,
pinnedCategory,
termsAccepted:
userAgent() === 'chrome' ||
(await getOption('termsAccepted', false))
}
break
case 'set_option':
await setOption(message.key, message.value)
break
case 'get_js_patterns':
response = {
patterns: wappalyzer.jsPatterns
}
break
case 'update_theme_mode':
// Sync theme mode to popup.
response = {
themeMode: await getOption('themeMode', false)
}
break
default:
// Do nothing
}
if (response) {
port.postMessage({
id: message.id,
response
})
}
})
*/
})
},
@ -220,7 +182,7 @@ const Driver = {
}
},
async onContentLoad(href, items) {
async onContentLoad(href, items, language) {
try {
const url = new URL(href)
@ -228,7 +190,7 @@ const Driver = {
domain: `.${url.hostname}`
})
await Driver.onDetect(url, await analyze(href, items))
await Driver.onDetect(url, await analyze(href, items), language)
} catch (error) {
Driver.error(error)
}
@ -238,15 +200,98 @@ const Driver = {
return Wappalyzer.technologies
},
async onDetect(url, detections = []) {
Driver.cache.hostnames[url.hostname] = unique([
...(Driver.cache.hostnames[url.hostname] || []),
...detections
])
async onDetect(url, detections = [], language) {
// Cache detections
// eslint-disable-next-line standard/computed-property-even-spacing
Driver.cache.hostnames[url.hostname] = {
...(Driver.cache.hostnames[url.hostname] || {
detections: []
}),
dateTime: Date.now()
}
const resolved = resolve(Driver.cache.hostnames[url.hostname])
Driver.cache.hostnames[url.hostname].language =
Driver.cache.hostnames[url.hostname].language || language
detections.forEach((detection) => {
const foo = Driver.cache.hostnames[url.hostname].detections
const {
technology: { name },
pattern: { regex }
} = detection
const cache = foo.find(
({ technology: { name: _name }, pattern: { regex: _regex } }) =>
name === _name && (!regex || regex) === _regex
)
if (cache) {
cache.hits += 1
} else {
foo.push({
...detection,
hits: 1
})
}
})
// Expire cache
Driver.cache.hostnames = Object.keys(Driver.cache.hostnames).reduce(
(hostnames, hostname) => {
const cache = Driver.cache.hostnames[hostname]
if (cache.dateTime > Date.now() - expiry) {
hostnames[hostname] = cache
}
return hostnames
},
{}
)
await setOption(
'hostnames',
Object.keys(Driver.cache.hostnames).reduce(
(cache, hostname) => ({
...cache,
[hostname]: {
...Driver.cache.hostnames[hostname],
detections: Driver.cache.hostnames[hostname].detections.map(
({
pattern: { regex, confidence, version },
match,
technology: { name: technology }
}) => ({
technology,
pattern: {
regex: regex.source,
confidence,
version
},
match
})
)
}
}),
{}
)
)
const resolved = resolve(Driver.cache.hostnames[url.hostname].detections)
await Driver.setIcon(url, resolved)
const tabs = await promisify(chrome.tabs, 'query', { url: [url.href] })
tabs.forEach(({ id }) => (Driver.cache.tabs[id] = resolved))
await Driver.ping()
},
async onAd(ad) {
Driver.cache.ads.push(ad)
await setOption('ads', Driver.cache.ads)
},
async setIcon(url, technologies) {
@ -292,24 +337,152 @@ const Driver = {
},
async getDetections() {
const [{ url: href }] = await promisify(chrome.tabs, 'query', {
const [{ id }] = await promisify(chrome.tabs, 'query', {
active: true,
currentWindow: true
})
return Driver.cache.tabs[id]
},
async getRobots(hostname, secure = false) {
if (!(await getOption('tracking', true))) {
return
}
if (typeof Driver.cache.robots[hostname] !== 'undefined') {
return Driver.cache.robots[hostname]
}
try {
Driver.cache.robots[hostname] = await Promise.race([
new Promise(async (resolve) => {
const response = await fetch(
`http${secure ? 's' : ''}://${hostname}/robots.txt`,
{
redirect: 'follow',
mode: 'no-cors'
}
)
if (!response.ok) {
Driver.error(new Error(response.statusText))
resolve('')
}
let agent
resolve(
(await response.text()).split('\n').reduce((disallows, line) => {
let matches = /^User-agent:\s*(.+)$/i.exec(line.trim())
if (matches) {
agent = matches[1].toLowerCase()
} else if (agent === '*' || agent === 'wappalyzer') {
matches = /^Disallow:\s*(.+)$/i.exec(line.trim())
if (matches) {
disallows.push(matches[1])
}
}
return disallows
}, [])
)
}),
new Promise((resolve) => setTimeout(() => resolve(''), 5000))
])
Driver.cache.robots = Object.keys(Driver.cache.robots)
.slice(-50)
.reduce(
(cache, hostname) => ({
...cache,
[hostname]: Driver.cache.robots[hostname]
}),
{}
)
await setOption('robots', Driver.cache.robots)
return Driver.cache.robots[hostname]
} catch (error) {
Driver.error(error)
}
},
async checkRobots(href) {
const url = new URL(href)
return resolve(Driver.cache.hostnames[url.hostname])
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
throw new Error('Invalid protocol')
}
}
;(async function() {
await Driver.loadTechnologies()
const robots = await Driver.getRobots(
url.hostname,
url.protocol === 'https:'
)
chrome.runtime.onConnect.addListener(Driver.onRuntimeConnect)
chrome.webRequest.onCompleted.addListener(
Driver.onWebRequestComplete,
{ urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] },
['responseHeaders']
if (robots.some((disallowed) => url.pathname.indexOf(disallowed) === 0)) {
throw new Error('Disallowed')
}
},
async ping() {
const tracking = await getOption('tracking', true)
const termsAccepted =
agent === 'chrome' || (await getOption('termsAccepted', false))
if (tracking && termsAccepted) {
const count = Object.keys(Driver.cache.hostnames).length
if (count && (count >= 50 || Driver.lastPing < Date.now() - 5000)) {
await Driver.post(
'https://api.wappalyzer.com/ping/v1/',
Object.keys(Driver.cache.hostnames).reduce((hostnames, hostname) => {
const { language, detections } = Driver.cache.hostnames[hostname]
hostnames[hostname] = hostnames[hostname] || {
applications: {},
meta: {
language
}
}
resolve(detections).forEach(({ name, confidence, version }) => {
if (confidence === 100) {
console.log(
name,
detections.find(
({ technology: { name: _name } }) => name === _name
)
)
})()
hostnames[hostname].applications[name] = {
version,
hits: detections.find(
({ technology: { name: _name } }) => name === _name
).pattern.hits
}
}
})
return hostnames
}, {})
)
await setOption('hostnames', (Driver.cache.hostnames = {}))
Driver.lastPing = Date.now()
}
if (Driver.cache.ads.length > 50) {
await Driver.post('https://ad.wappalyzer.com/log/wp/', Driver.cache.ads)
await setOption('ads', (Driver.cache.ads = []))
}
}
}
}
Driver.init()

@ -120,8 +120,8 @@ var exports = {};
return dict;
},
sendToBackground: function(message, event, responseMessage) {
if ( typeof browser !== 'undefined' || typeof chrome !== 'undefined' ) {
var port = browser.runtime.connect({name:"adparser"});
if ( typeof chrome !== 'undefined' ) {
var port = chrome.runtime.connect({name:"adparser"});
port.onMessage.addListener((message) => {
if ( message && message.tracking_enabled ) {
@ -1088,8 +1088,8 @@ var exports = {};
}
function addBackgroundListener(event, callback) {
if ( typeof browser !== 'undefined' || typeof chrome !== 'undefined' ) {
browser.runtime.onMessage.addListener(function(msg) {
if ( typeof chrome !== 'undefined' ) {
chrome.runtime.onMessage.addListener(function(msg) {
if ( msg.event === event ) {
callback(msg);
}
@ -1111,6 +1111,7 @@ var exports = {};
if ( origUrl.indexOf('google.com/_/chrome/newtab') === -1 ) {
var onBlockedRobotsMessage = function() {
return // TODO
var log;
log = _logGen.log('invalid-robotstxt', []);
log.doc.finalPageUrl = log.doc.url;
@ -1173,7 +1174,7 @@ if ( exports.utils.SCRIPT_IN_WINDOW_TOP ) {
})(window);
(function(adparser, pageUrl) {
function onAdFound(log) {
adparser.sendToBackground({ id: 'ad_log', subject: log }, 'ad_log', '', function(){});
adparser.sendToBackground({ func: 'onAd', args: [log] }, 'onAd', '', function(){});
}
if ( adparser && adparser.inWindowTop ) {

@ -1,63 +0,0 @@
jsonToDOM.namespaces = {
html: 'http://www.w3.org/1999/xhtml',
xul: 'http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul',
};
jsonToDOM.defaultNamespace = jsonToDOM.namespaces.html;
function jsonToDOM(jsonTemplate, doc, nodes) {
function namespace(name) {
const reElemNameParts = /^(?:(.*):)?(.*)$/.exec(name);
return { namespace: jsonToDOM.namespaces[reElemNameParts[1]], shortName: reElemNameParts[2] };
}
// Note that 'elemNameOrArray' is: either the full element name (eg. [html:]div) or an array of elements in JSON notation
function tag(elemNameOrArray, elemAttr) {
// Array of elements? Parse each one...
if (Array.isArray(elemNameOrArray)) {
const frag = doc.createDocumentFragment();
Array.prototype.forEach.call(arguments, (thisElem) => {
frag.appendChild(tag(...thisElem));
});
return frag;
}
// Single element? Parse element namespace prefix (if none exists, default to defaultNamespace), and create element
const elemNs = namespace(elemNameOrArray);
const elem = doc.createElementNS(elemNs.namespace || jsonToDOM.defaultNamespace, elemNs.shortName);
// Set element's attributes and/or callback functions (eg. onclick)
for (const key in elemAttr) {
const val = elemAttr[key];
if (nodes && key == 'key') {
nodes[val] = elem;
continue;
}
const attrNs = namespace(key);
if (typeof val === 'function') {
// Special case for function attributes; don't just add them as 'on...' attributes, but as events, using addEventListener
elem.addEventListener(key.replace(/^on/, ''), val, false);
} else {
// Note that the default namespace for XML attributes is, and should be, blank (ie. they're not in any namespace)
elem.setAttributeNS(attrNs.namespace || '', attrNs.shortName, val);
}
}
// Create and append this element's children
const childElems = Array.prototype.slice.call(arguments, 2);
childElems.forEach((childElem) => {
if (childElem != null) {
elem.appendChild(
childElem instanceof doc.defaultView.Node ? childElem
: Array.isArray(childElem) ? tag(...childElem)
: doc.createTextNode(childElem),
);
}
});
return elem;
}
return tag(...jsonTemplate);
}

@ -1,19 +1,6 @@
(function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i<t.length;i++)o(t[i]);return o}return r})()({1:[function(require,module,exports){
'use strict';
(function() {
function isChrome() {
return (typeof chrome !== 'undefined' &&
window.navigator.userAgent.match(/Chrom(?:e|ium)\/([0-9\.]+)/));
}
var requiredBrowserApis = [
browser.webNavigation,
browser.tabs,
browser.webRequest,
browser.runtime
];
(function() {
var MIN_FF_MAJOR_VERSION = 51;
var areListenersRegistered = false;
@ -61,7 +48,7 @@
'washingtonpost.com'
];
var robotsTxtAllows = wappalyzer.robotsTxtAllows.bind(wappalyzer);
var robotsTxtAllows = Driver.checkRobots;
if ( !String.prototype.endsWith ) {
String.prototype.endsWith = function(searchString, position) {
var subjectString = this.toString();
@ -76,34 +63,7 @@
}
function getFrame(getFrameDetails, callback) {
var gettingFrame = browser.webNavigation.getFrame(getFrameDetails);
gettingFrame.then(callback);
}
function ifBrowserValid(callback, elseCallback) {
if ( isChrome() ) {
callback();
} else if ( typeof browser !== 'undefined' ) {
try {
var gettingInfo = browser.runtime.getBrowserInfo();
gettingInfo.then(function(browserInfo) {
var browserVersion = parseInt(browserInfo.version.split('.')[0]);
if ( browserInfo.name === 'Firefox' &&
browserVersion >= MIN_FF_MAJOR_VERSION) {
callback();
} else {
elseCallback();
}
});
} catch (err) {
elseCallback();
}
} else {
elseCallback();
}
chrome.webNavigation.getFrame(getFrameDetails, callback);
}
function ifTrackingEnabled(details, ifCallback, elseCallback) {
@ -112,24 +72,19 @@
allowedByRobotsTxt(details, ifCallback, elseCallback);
};
browser.storage.local.get('tracking').then(function(item) {
if ( item.hasOwnProperty('tracking') ) {
if ( item.tracking ) {
Utils.getOption('tracking', true).then(function(tracking) {
if ( tracking ) {
fullIfCallback();
} else {
elseCallback();
}
} else {
fullIfCallback();
}
});
}
function allowedByRobotsTxt(details, ifCallback, elseCallback) {
if ( details.url && !details.url.startsWith('chrome://') ) {
robotsTxtAllows(details.url).then(ifCallback, elseCallback);
Driver.checkRobots(details.url, details.url.startsWith('https:')).then(ifCallback).catch(elseCallback);
} else {
elseCallback();
}
@ -268,7 +223,7 @@
PageNetworkTrafficCollector.prototype.sendLogMessageToTabConsole = function() {
var logMessage = Array.from(arguments).join(' ');
var message = {message: logMessage, event: 'console-log-message'};
browser.tabs.sendMessage(this.tabId, message);
chrome.tabs.sendMessage(this.tabId, message);
};
PageNetworkTrafficCollector.prototype.sendToTab = function(assetReq, reqs, curPageUrl, adTrackingEvent) {
@ -298,7 +253,7 @@
msg.origUrl = curPageUrl;
msg.displayAdFound = this.displayAdFound;
browser.tabs.sendMessage(this.tabId, msg);
chrome.tabs.sendMessage(this.tabId, msg);
};
PageNetworkTrafficCollector.prototype.getRedirKey = function(url, frameId) {
@ -615,7 +570,7 @@
var _this = this,
origPageUrl, msgAssetReq;
msgAssetReq = this.msgsBeingSent[msgKey];
browser.tabs.get(this.tabId).then(function(tab) {
chrome.tabs.get(this.tabId, function(tab) {
origPageUrl = tab.url;
});
@ -697,83 +652,74 @@
function registerListeners() {
browser.webRequest.onBeforeRequest.addListener(
chrome.webRequest.onBeforeRequest.addListener(
onBeforeRequestListener,
{urls: ['http://*/*', 'https://*/*']},
[]
);
browser.webRequest.onSendHeaders.addListener(
chrome.webRequest.onSendHeaders.addListener(
onSendHeadersListener,
{urls: ['http://*/*', 'https://*/*']},
['requestHeaders']
);
browser.webRequest.onHeadersReceived.addListener(
chrome.webRequest.onHeadersReceived.addListener(
onHeadersReceivedListener,
{urls: ['http://*/*', 'https://*/*']},
['responseHeaders']
);
browser.webRequest.onBeforeRedirect.addListener(
chrome.webRequest.onBeforeRedirect.addListener(
onBeforeRedirectListener,
{urls: ['http://*/*', 'https://*/*']},
[]
);
browser.webRequest.onResponseStarted.addListener(
chrome.webRequest.onResponseStarted.addListener(
onResponseStartedListener,
{urls: ['http://*/*', 'https://*/*']},
['responseHeaders']
);
browser.webNavigation.onCommitted.addListener(onCommittedListener);
browser.webNavigation.onCompleted.addListener(onCompletedListener);
browser.tabs.onRemoved.addListener(onRemovedListener);
browser.runtime.onMessage.addListener(onMessageListener);
chrome.webNavigation.onCommitted.addListener(onCommittedListener);
chrome.webNavigation.onCompleted.addListener(onCompletedListener);
chrome.tabs.onRemoved.addListener(onRemovedListener);
chrome.runtime.onMessage.addListener(onMessageListener);
areListenersRegistered = true;
}
function unregisterListeners() {
browser.webRequest.onBeforeRequest.removeListener(
chrome.webRequest.onBeforeRequest.removeListener(
onBeforeRequestListener
);
browser.webRequest.onSendHeaders.removeListener(
chrome.webRequest.onSendHeaders.removeListener(
onSendHeadersListener
);
browser.webRequest.onHeadersReceived.removeListener(
chrome.webRequest.onHeadersReceived.removeListener(
onHeadersReceivedListener
);
browser.webRequest.onBeforeRedirect.removeListener(
chrome.webRequest.onBeforeRedirect.removeListener(
onBeforeRedirectListener
);
browser.webRequest.onResponseStarted.removeListener(
chrome.webRequest.onResponseStarted.removeListener(
onResponseStartedListener
);
browser.webNavigation.onCommitted.removeListener(onCommittedListener);
browser.webNavigation.onCompleted.removeListener(onCompletedListener);
browser.tabs.onRemoved.removeListener(onRemovedListener);
browser.runtime.onMessage.removeListener(onMessageListener);
chrome.webNavigation.onCommitted.removeListener(onCommittedListener);
chrome.webNavigation.onCompleted.removeListener(onCompletedListener);
chrome.tabs.onRemoved.removeListener(onRemovedListener);
chrome.runtime.onMessage.removeListener(onMessageListener);
areListenersRegistered = false;
}
function areRequiredBrowserApisAvailable() {
return requiredBrowserApis.every(function(api) {
return typeof api !== 'undefined';
});
}
if ( areRequiredBrowserApisAvailable() ) {
ifBrowserValid(
function() {
browser.webNavigation.onBeforeNavigate.addListener(
chrome.webNavigation.onBeforeNavigate.addListener(
function(details) {
if ( details.frameId === 0 ) {
globalPageContainer.onNewNavigation(details);
@ -783,24 +729,8 @@
url: [{urlMatches: 'http://*/*'}, {urlMatches: 'https://*/*'}]
}
);
}, function() {
}
);
}
browser.runtime.onConnect.addListener((port) => {
port.onMessage.addListener((message) => {
if ( message === 'is_browser_valid' ) {
ifBrowserValid(
port.postMessage({'browser_valid': true}),
port.postMessage({'browser_valid': false})
);
}
});
});
browser.runtime.onConnect.addListener((port) => {
chrome.runtime.onConnect.addListener((port) => {
port.onMessage.addListener((message) => {
if ( message === 'is_tracking_enabled' ) {
ifTrackingEnabled(
@ -816,7 +746,4 @@
return true;
});
});
})();
},{}]},{},[1]);

@ -122,7 +122,9 @@ const Popup = {
})
)
technologies.forEach(({ name, slug, icon, website }) => {
technologies
.filter(({ confidence }) => confidence)
.forEach(({ name, slug, confidence, version, icon, website }) => {
const technologyNode = Popup.templates.technology.cloneNode(true)
const image = technologyNode.querySelector('.technology__icon')
@ -134,6 +136,26 @@ const Popup = {
link.href = `https://www.wappalyzer.com/technologies/${categorySlug}/${slug}`
link.textContent = name
const confidenceNode = technologyNode.querySelector(
'.technology__confidence'
)
if (confidence < 100) {
confidenceNode.textContent = `${confidence}% sure`
} else {
confidenceNode.remove()
}
const versionNode = technologyNode.querySelector(
'.technology__version'
)
if (version) {
versionNode.textContent = version
} else {
versionNode.remove()
}
categoryNode
.querySelector('.technologies')
.appendChild(technologyNode)

Loading…
Cancel
Save