Migrations
/
wappalyzer
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make 'requires' work with 'dom' and 'js' in WebExtension driver

Browse Source
main
Elbert Alias 4 years ago
parent 3695f771f2
commit 8b1f03e8da
4 changed files with 191 additions and 159 deletions
Show Stats Download Patch File Download Diff File

2
src/drivers/webextension/css/styles.css
Unescape View File

@ -616,7 +616,7 @@ body.dynamic-icon .category__heading:hover .category__pin {
color: var(--color-text);
border-radius: 3px;
font-size: .7rem;
padding: .3rem .3rem .1rem .3rem;
padding: .1rem .3rem;
margin-left: .3rem;
vertical-align: middle;
}

292
src/drivers/webextension/js/content.js
Unescape View File

@ -2,12 +2,129 @@
/* eslint-env browser */
/* globals chrome */
function getJs(technologies) {
return new Promise((resolve) => {
// Inject a script tag into the page to access methods of the window object
const script = document.createElement('script')
script.onload = () => {
const onMessage = ({ data }) => {
if (!data.wappalyzer || !data.wappalyzer.js) {
return
}
window.removeEventListener('message', onMessage)
resolve(data.wappalyzer.js)
script.remove()
}
window.addEventListener('message', onMessage)
window.postMessage({
wappalyzer: {
technologies: technologies
.filter(({ js }) => Object.keys(js).length)
.map(({ name, js }) => ({ name, chains: Object.keys(js) })),
},
})
}
script.setAttribute('src', chrome.extension.getURL('js/inject.js'))
document.body.appendChild(script)
})
}
function getDom(technologies) {
return technologies
.filter(({ dom }) => dom && dom.constructor === Object)
.map(({ name, dom }) => ({ name, dom }))
.reduce((technologies, { name, dom }) => {
const toScalar = (value) =>
typeof value === 'string' || typeof value === 'number' ? value : !!value
Object.keys(dom).forEach((selector) => {
let nodes = []
try {
nodes = document.querySelectorAll(selector)
} catch (error) {
Content.driver('error', error)
}
if (!nodes.length) {
return
}
dom[selector].forEach(({ exists, text, properties, attributes }) => {
nodes.forEach((node) => {
if (exists) {
technologies.push({
name,
selector,
exists: '',
})
}
if (text) {
const value = node.textContent.trim()
if (value) {
technologies.push({
name,
selector,
text: value,
})
}
}
if (properties) {
Object.keys(properties).forEach((property) => {
if (Object.prototype.hasOwnProperty.call(node, property)) {
const value = node[property]
if (typeof value !== 'undefined') {
technologies.push({
name,
selector,
property,
value: toScalar(value),
})
}
}
})
}
if (attributes) {
Object.keys(attributes).forEach((attribute) => {
if (node.hasAttribute(attribute)) {
const value = node.getAttribute(attribute)
technologies.push({
name,
selector,
attribute,
value: toScalar(value),
})
}
})
}
})
})
})
return technologies
}, [])
}
const Content = {
href: location.href,
cache: {},
language: '',
requiresAnalyzed: [],
analyzedRequires: [],
/**
* Initialise content script
@ -119,7 +236,7 @@ const Content = {
Content.cache = { html, css, scripts, meta }
Content.driver('onContentLoad', [
await Content.driver('onContentLoad', [
Content.href,
Content.cache,
Content.language,
@ -127,12 +244,12 @@ const Content = {
const technologies = await Content.driver('getTechnologies')
Content.onGetTechnologies(technologies)
await Content.onGetTechnologies(technologies)
// Delayed second pass to capture async JS
await new Promise((resolve) => setTimeout(resolve, 5000))
Content.onGetTechnologies(technologies)
await Content.onGetTechnologies(technologies)
} catch (error) {
Content.driver('error', error)
}
@ -165,7 +282,7 @@ const Content = {
},
driver(func, args) {
return new Promise((resolve, reject) => {
return new Promise((resolve) => {
chrome.runtime.sendMessage(
{
source: 'content.js',
@ -186,7 +303,9 @@ const Content = {
: Content.driver(
'error',
new Error(
`${chrome.runtime.lastError}: Driver.${func}(${args})`
`${
chrome.runtime.lastError.message
}: Driver.${func}(${JSON.stringify(args)})`
)
)
: resolve(response)
@ -195,147 +314,42 @@ const Content = {
})
},
analyzeRequires(requires) {
Object.keys(requires).forEach((name) => {
if (!Content.requiresAnalyzed.includes(name)) {
Content.requiresAnalyzed.push(name)
Content.driver('onContentLoad', [
Content.href,
Content.cache,
Content.language,
name,
])
Content.onGetTechnologies(requires[name].technologies)
}
})
async analyzeRequires(requires) {
await Promise.all(
Object.keys(requires).map(async (name) => {
if (!Content.analyzedRequires.includes(name)) {
Content.analyzedRequires.push(name)
const technologies = requires[name].technologies
await Promise.all([
Content.onGetTechnologies(technologies, name),
Content.driver('onContentLoad', [
Content.href,
Content.cache,
Content.language,
name,
]),
])
}
})
)
},
/**
* Callback for getTechnologies
* @param {Array} technologies
*/
onGetTechnologies(technologies = []) {
// Inject a script tag into the page to access methods of the window object
const script = document.createElement('script')
script.onload = () => {
const onMessage = ({ data }) => {
if (!data.wappalyzer || !data.wappalyzer.js) {
return
}
window.removeEventListener('message', onMessage)
chrome.runtime.sendMessage({
source: 'content.js',
func: 'analyzeJs',
args: [location.href.split('#')[0], data.wappalyzer.js],
})
script.remove()
}
window.addEventListener('message', onMessage)
window.postMessage({
wappalyzer: {
technologies: technologies
.filter(({ js }) => Object.keys(js).length)
.map(({ name, js }) => ({ name, chains: Object.keys(js) })),
},
})
}
script.setAttribute('src', chrome.extension.getURL('js/inject.js'))
document.body.appendChild(script)
// DOM
const dom = technologies
.filter(({ dom }) => dom && dom.constructor === Object)
.map(({ name, dom }) => ({ name, dom }))
.reduce((technologies, { name, dom }) => {
const toScalar = (value) =>
typeof value === 'string' || typeof value === 'number'
? value
: !!value
Object.keys(dom).forEach((selector) => {
let nodes = []
try {
nodes = document.querySelectorAll(selector)
} catch (error) {
Content.driver('error', error)
}
if (!nodes.length) {
return
}
dom[selector].forEach(({ exists, text, properties, attributes }) => {
nodes.forEach((node) => {
if (exists) {
technologies.push({
name,
selector,
exists: '',
})
}
if (text) {
const value = node.textContent.trim()
if (value) {
technologies.push({
name,
selector,
text: value,
})
}
}
if (properties) {
Object.keys(properties).forEach((property) => {
if (Object.prototype.hasOwnProperty.call(node, property)) {
const value = node[property]
if (typeof value !== 'undefined') {
technologies.push({
name,
selector,
property,
value: toScalar(value),
})
}
}
})
}
if (attributes) {
Object.keys(attributes).forEach((attribute) => {
if (node.hasAttribute(attribute)) {
const value = node.getAttribute(attribute)
technologies.push({
name,
selector,
attribute,
value: toScalar(value),
})
}
})
}
})
})
})
async onGetTechnologies(technologies = [], requires) {
const url = location.href.split('#')[0]
return technologies
}, [])
const js = await getJs(technologies)
const dom = getDom(technologies)
Content.driver('analyzeDom', [location.href, dom])
await Promise.all([
Content.driver('analyzeJs', [url, js, requires]),
Content.driver('analyzeDom', [url, dom, requires]),
])
},
}

49
src/drivers/webextension/js/driver.js
Unescape View File

@ -42,9 +42,7 @@ const Driver = {
pattern: { regex, confidence },
version,
}) => ({
technology: Wappalyzer.technologies.find(
({ name: _name }) => name === _name
),
technology: getTechnology(name, true),
pattern: {
regex: new RegExp(regex, 'i'),
confidence,
@ -121,7 +119,7 @@ const Driver = {
*/
log(message, source = 'driver', type = 'log') {
// eslint-disable-next-line no-console
console[type](`wappalyzer | ${source} |`, message)
console[type](message)
},
/**
@ -177,7 +175,11 @@ const Driver = {
* @param {String} url
* @param {Array} js
*/
async analyzeJs(url, js) {
async analyzeJs(url, js, requires) {
const technologies = requires
? Wappalyzer.requires[requires].technologies
: Wappalyzer.technologies
return Driver.onDetect(
url,
Array.prototype.concat.apply(
@ -187,7 +189,7 @@ const Driver = {
await next()
return analyzeManyToMany(
Wappalyzer.technologies.find(({ name: _name }) => name === _name),
technologies.find(({ name: _name }) => name === _name),
'js',
{ [chain]: [value] }
)
@ -202,7 +204,11 @@ const Driver = {
* @param {String} url
* @param {Array} dom
*/
async analyzeDom(url, dom) {
async analyzeDom(url, dom, requires) {
const technologies = requires
? Wappalyzer.requires[requires].technologies
: Wappalyzer.technologies
return Driver.onDetect(
url,
Array.prototype.concat.apply(
@ -215,7 +221,7 @@ const Driver = {
) => {
await next()
const technology = Wappalyzer.technologies.find(
const technology = technologies.find(
({ name: _name }) => name === _name
)
@ -283,7 +289,9 @@ const Driver = {
return
}
Driver.log({ source, func, args })
if (func !== 'log') {
Driver.log({ source, func, args })
}
if (!Driver[func]) {
Driver.error(new Error(`Method does not exist: Driver.${func}`))
@ -307,6 +315,10 @@ const Driver = {
return
}
if (tab.status !== 'complete') {
throw new Error(`Tab ${tab.id} not ready for sendMessage: ${tab.status}`)
}
return new Promise((resolve, reject) => {
chrome.tabs.sendMessage(
tab.id,
@ -321,7 +333,9 @@ const Driver = {
? resolve()
: Driver.error(
new Error(
`${chrome.runtime.lastError}: Driver.${func}(${args})`
`${
chrome.runtime.lastError.message
}: Driver.${func}(${JSON.stringify(args)})`
)
)
: resolve(response)
@ -567,14 +581,15 @@ const Driver = {
return detection
})
const requires = Wappalyzer.requires
.filter(({ name, technologies }) =>
resolved.some(({ name: _name }) => _name === name)
)
.map(({ technologies }) => technologies)
.flat()
const requires = Wappalyzer.requires.filter(({ name, technologies }) =>
resolved.some(({ name: _name }) => _name === name)
)
Driver.content(url, 'analyzeRequires', [requires])
try {
await Driver.content(url, 'analyzeRequires', [requires])
} catch (error) {
// Continue
}
await Driver.setIcon(url, resolved)

7
src/wappalyzer.js
Unescape View File

@ -11,7 +11,7 @@ function toArray(value) {
const Wappalyzer = {
technologies: [],
categories: [],
requires: {},
requires: [],
slugify: (string) =>
string
@ -21,7 +21,10 @@ const Wappalyzer = {
.replace(/(?:^-|-$)/g, ''),
getTechnology: (name) =>
Wappalyzer.technologies.find(({ name: _name }) => name === _name),
[
...Wappalyzer.technologies,
...Wappalyzer.requires.map(({ technologies }) => technologies).flat(),
].find(({ name: _name }) => name === _name),
getCategory: (id) => Wappalyzer.categories.find(({ id: _id }) => id === _id),