Migrations
/
wappalyzer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Non-blocking analyse

Browse Source
main
Elbert Alias 4 years ago
parent 0d6cfa64c4
commit 90f981000b
4 changed files with 169 additions and 115 deletions
Show Stats Download Patch File Download Diff File

10
src/README.md
Unescape View File

@ -25,16 +25,16 @@ const { technologies, categories } = JSON.parse(
Wappalyzer.setTechnologies(technologies)
Wappalyzer.setCategories(categories)
const detections = Wappalyzer.analyze({
Wappalyzer.analyze({
url: 'https://example.github.io/',
meta: { generator: ['WordPress'] },
headers: { server: ['Nginx'] },
scripts: ['jquery-3.0.0.js'],
cookies: { awselb: [''] },
html: '<div ng-app="">'
})
const results = Wappalyzer.resolve(detections)
}).then((detections) => {
const results = Wappalyzer.resolve(detections)
console.log(results)
console.log(results)
})
```

110
src/drivers/npm/driver.js
Unescape View File

@ -9,6 +9,10 @@ const Wappalyzer = require('./wappalyzer')
const { setTechnologies, setCategories, analyze, analyzeManyToMany, resolve } =
Wappalyzer
function next() {
return new Promise((resolve) => setImmediate(resolve))
}
const { AWS_LAMBDA_FUNCTION_NAME, CHROMIUM_BIN, CHROMIUM_DATA_DIR } =
process.env
@ -49,53 +53,75 @@ function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms))
}
function analyzeJs(js) {
async function analyzeJs(js) {
return Array.prototype.concat.apply(
[],
js.map(({ name, chain, value }) =>
analyzeManyToMany(
Wappalyzer.technologies.find(({ name: _name }) => name === _name),
'js',
{ [chain]: [value] }
)
await Promise.all(
js.map(async ({ name, chain, value }) => {
await next()
return analyzeManyToMany(
Wappalyzer.technologies.find(({ name: _name }) => name === _name),
'js',
{ [chain]: [value] }
)
})
)
)
}
function analyzeDom(dom) {
async function analyzeDom(dom) {
return Array.prototype.concat.apply(
[],
dom.map(({ name, selector, exists, text, property, attribute, value }) => {
const technology = Wappalyzer.technologies.find(
({ name: _name }) => name === _name
)
await Promise.all(
dom.map(
async ({
name,
selector,
exists,
text,
property,
attribute,
value,
}) => {
await next()
const technology = Wappalyzer.technologies.find(
({ name: _name }) => name === _name
)
if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''],
})
}
if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''],
})
}
if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}
if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}
if (typeof property !== 'undefined') {
return analyzeManyToMany(technology, `dom.properties.${property}`, {
[selector]: [value],
})
}
if (typeof property !== 'undefined') {
return analyzeManyToMany(technology, `dom.properties.${property}`, {
[selector]: [value],
})
}
if (typeof attribute !== 'undefined') {
return analyzeManyToMany(technology, `dom.attributes.${attribute}`, {
[selector]: [value],
})
}
if (typeof attribute !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.attributes.${attribute}`,
{
[selector]: [value],
}
)
}
return []
})
return []
}
)
)
)
}
@ -343,10 +369,10 @@ class Site {
if (!xhrDebounce.includes(hostname)) {
xhrDebounce.push(hostname)
setTimeout(() => {
setTimeout(async () => {
xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1)
this.onDetect(analyze({ xhr: hostname }))
this.onDetect(await analyze({ xhr: hostname }))
}, 1000)
}
}
@ -402,7 +428,7 @@ class Site {
? response.securityDetails().issuer()
: ''
this.onDetect(analyze({ headers, certIssuer }))
this.onDetect(await analyze({ headers, certIssuer }))
await this.emit('response', { page, response, headers, certIssuer })
}
@ -720,7 +746,7 @@ class Site {
return dns
}, {})
this.onDetect(analyze({ dns: this.dns }))
this.onDetect(await analyze({ dns: this.dns }))
}
// Validate response
@ -736,10 +762,10 @@ class Site {
throw new Error('No response from server')
}
this.onDetect(analyzeDom(dom))
this.onDetect(analyzeJs(js))
this.onDetect(await analyzeDom(dom))
this.onDetect(await analyzeJs(js))
this.onDetect(
analyze({
await analyze({
url,
cookies,
html,
@ -874,7 +900,7 @@ class Site {
this.log(`get ${path}: ok`)
this.onDetect(analyze({ [file]: body }))
this.onDetect(await analyze({ [file]: body }))
} catch (error) {
this.error(`get ${path}: ${error.message || error}`)
}

122
src/drivers/webextension/js/driver.js
Unescape View File

@ -1,6 +1,6 @@
'use strict'
/* eslint-env browser */
/* globals chrome, Wappalyzer, Utils */
/* globals chrome, Wappalyzer, Utils, next */
const {
setTechnologies,
@ -161,17 +161,21 @@ const Driver = {
* @param {String} url
* @param {Array} js
*/
analyzeJs(url, js) {
async analyzeJs(url, js) {
return Driver.onDetect(
url,
Array.prototype.concat.apply(
[],
js.map(({ name, chain, value }) =>
analyzeManyToMany(
Wappalyzer.technologies.find(({ name: _name }) => name === _name),
'js',
{ [chain]: [value] }
)
await Promise.all(
js.map(async ({ name, chain, value }) => {
await next()
return analyzeManyToMany(
Wappalyzer.technologies.find(({ name: _name }) => name === _name),
'js',
{ [chain]: [value] }
)
})
)
)
)
@ -182,54 +186,58 @@ const Driver = {
* @param {String} url
* @param {Array} dom
*/
analyzeDom(url, dom) {
async analyzeDom(url, dom) {
return Driver.onDetect(
url,
Array.prototype.concat.apply(
[],
dom.map(
(
{ name, selector, exists, text, property, attribute, value },
index
) => {
const technology = Wappalyzer.technologies.find(
({ name: _name }) => name === _name
)
await Promise.all(
dom.map(
async (
{ name, selector, exists, text, property, attribute, value },
index
) => {
await next()
const technology = Wappalyzer.technologies.find(
({ name: _name }) => name === _name
)
if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''],
})
}
if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''],
})
}
if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}
if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}
if (typeof property !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.properties.${property}`,
{
[selector]: [value],
}
)
}
if (typeof property !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.properties.${property}`,
{
[selector]: [value],
}
)
}
if (typeof attribute !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.attributes.${attribute}`,
{
[selector]: [value],
}
)
}
if (typeof attribute !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.attributes.${attribute}`,
{
[selector]: [value],
}
)
}
return []
}
return []
}
)
)
)
)
@ -304,7 +312,9 @@ const Driver = {
)
})
Driver.onDetect(request.url, analyze({ headers })).catch(Driver.error)
Driver.onDetect(request.url, await analyze({ headers })).catch(
Driver.error
)
}
} catch (error) {
Driver.error(error)
@ -332,12 +342,13 @@ const Driver = {
if (!xhrDebounce.includes(hostname)) {
xhrDebounce.push(hostname)
setTimeout(() => {
setTimeout(async () => {
xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1)
Driver.onDetect(request.originUrl, analyze({ xhr: hostname })).catch(
Driver.error
)
Driver.onDetect(
request.originUrl,
await analyze({ xhr: hostname })
).catch(Driver.error)
}, 1000)
}
},
@ -364,7 +375,12 @@ const Driver = {
{}
)
await Driver.onDetect(url, analyze({ url, ...items }), language, true)
await Driver.onDetect(
url,
await analyze({ url, ...items }),
language,
true
)
} catch (error) {
Driver.error(error)
}

42
src/wappalyzer.js
Unescape View File

@ -1,5 +1,13 @@
'use strict'
function next() {
return new Promise((resolve) =>
(typeof setImmediate !== 'undefined'
? setImmediate
: requestAnimationFrame)(resolve)
)
}
function toArray(value) {
return Array.isArray(value) ? value : [value]
}
@ -187,7 +195,7 @@ const Wappalyzer = {
* Initialize analyzation.
* @param {*} param0
*/
analyze({
async analyze({
url,
xhr,
html,
@ -208,20 +216,24 @@ const Wappalyzer = {
try {
const detections = flatten(
Wappalyzer.technologies.map((technology) =>
flatten([
oo(technology, 'url', url),
oo(technology, 'xhr', xhr),
oo(technology, 'html', html),
oo(technology, 'css', css),
oo(technology, 'robots', robots),
oo(technology, 'certIssuer', certIssuer),
om(technology, 'scripts', scripts),
mm(technology, 'cookies', cookies),
mm(technology, 'meta', meta),
mm(technology, 'headers', headers),
mm(technology, 'dns', dns),
])
await Promise.all(
Wappalyzer.technologies.map(async (technology) => {
await next()
return flatten([
oo(technology, 'url', url),
oo(technology, 'xhr', xhr),
oo(technology, 'html', html),
oo(technology, 'css', css),
oo(technology, 'robots', robots),
oo(technology, 'certIssuer', certIssuer),
om(technology, 'scripts', scripts),
mm(technology, 'cookies', cookies),
mm(technology, 'meta', meta),
mm(technology, 'headers', headers),
mm(technology, 'dns', dns),
])
})
)
).filter((technology) => technology)

Write Preview
Loading…
Cancel
Save