Migrations
/
wappalyzer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Non-blocking analyse

Browse Source
main
Elbert Alias 4 years ago
parent 0d6cfa64c4
commit 90f981000b
4 changed files with 169 additions and 115 deletions
Show Stats Download Patch File Download Diff File

10
src/README.md
Unescape View File

@ -25,16 +25,16 @@ const { technologies, categories } = JSON.parse(
Wappalyzer.setTechnologies(technologies) Wappalyzer.setTechnologies(technologies)
Wappalyzer.setCategories(categories) Wappalyzer.setCategories(categories)
const detections = Wappalyzer.analyze({ Wappalyzer.analyze({
url: 'https://example.github.io/', url: 'https://example.github.io/',
meta: { generator: ['WordPress'] }, meta: { generator: ['WordPress'] },
headers: { server: ['Nginx'] }, headers: { server: ['Nginx'] },
scripts: ['jquery-3.0.0.js'], scripts: ['jquery-3.0.0.js'],
cookies: { awselb: [''] }, cookies: { awselb: [''] },
html: '<div ng-app="">' html: '<div ng-app="">'
}) }).then((detections) => {
const results = Wappalyzer.resolve(detections)
const results = Wappalyzer.resolve(detections)
console.log(results) console.log(results)
})
``` ```

110
src/drivers/npm/driver.js
Unescape View File

@ -9,6 +9,10 @@ const Wappalyzer = require('./wappalyzer')
const { setTechnologies, setCategories, analyze, analyzeManyToMany, resolve } = const { setTechnologies, setCategories, analyze, analyzeManyToMany, resolve } =
Wappalyzer Wappalyzer
function next() {
return new Promise((resolve) => setImmediate(resolve))
}
const { AWS_LAMBDA_FUNCTION_NAME, CHROMIUM_BIN, CHROMIUM_DATA_DIR } = const { AWS_LAMBDA_FUNCTION_NAME, CHROMIUM_BIN, CHROMIUM_DATA_DIR } =
process.env process.env
@ -49,53 +53,75 @@ function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms)) return new Promise((resolve) => setTimeout(resolve, ms))
} }
function analyzeJs(js) { async function analyzeJs(js) {
return Array.prototype.concat.apply( return Array.prototype.concat.apply(
[], [],
js.map(({ name, chain, value }) => await Promise.all(
analyzeManyToMany( js.map(async ({ name, chain, value }) => {
Wappalyzer.technologies.find(({ name: _name }) => name === _name), await next()
'js',
{ [chain]: [value] } return analyzeManyToMany(
) Wappalyzer.technologies.find(({ name: _name }) => name === _name),
'js',
{ [chain]: [value] }
)
})
) )
) )
} }
function analyzeDom(dom) { async function analyzeDom(dom) {
return Array.prototype.concat.apply( return Array.prototype.concat.apply(
[], [],
dom.map(({ name, selector, exists, text, property, attribute, value }) => { await Promise.all(
const technology = Wappalyzer.technologies.find( dom.map(
({ name: _name }) => name === _name async ({
) name,
selector,
exists,
text,
property,
attribute,
value,
}) => {
await next()
const technology = Wappalyzer.technologies.find(
({ name: _name }) => name === _name
)
if (typeof exists !== 'undefined') { if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', { return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''], [selector]: [''],
}) })
} }
if (typeof text !== 'undefined') { if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', { return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text], [selector]: [text],
}) })
} }
if (typeof property !== 'undefined') { if (typeof property !== 'undefined') {
return analyzeManyToMany(technology, `dom.properties.${property}`, { return analyzeManyToMany(technology, `dom.properties.${property}`, {
[selector]: [value], [selector]: [value],
}) })
} }
if (typeof attribute !== 'undefined') { if (typeof attribute !== 'undefined') {
return analyzeManyToMany(technology, `dom.attributes.${attribute}`, { return analyzeManyToMany(
[selector]: [value], technology,
}) `dom.attributes.${attribute}`,
} {
[selector]: [value],
}
)
}
return [] return []
}) }
)
)
) )
} }
@ -343,10 +369,10 @@ class Site {
if (!xhrDebounce.includes(hostname)) { if (!xhrDebounce.includes(hostname)) {
xhrDebounce.push(hostname) xhrDebounce.push(hostname)
setTimeout(() => { setTimeout(async () => {
xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1) xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1)
this.onDetect(analyze({ xhr: hostname })) this.onDetect(await analyze({ xhr: hostname }))
}, 1000) }, 1000)
} }
} }
@ -402,7 +428,7 @@ class Site {
? response.securityDetails().issuer() ? response.securityDetails().issuer()
: '' : ''
this.onDetect(analyze({ headers, certIssuer })) this.onDetect(await analyze({ headers, certIssuer }))
await this.emit('response', { page, response, headers, certIssuer }) await this.emit('response', { page, response, headers, certIssuer })
} }
@ -720,7 +746,7 @@ class Site {
return dns return dns
}, {}) }, {})
this.onDetect(analyze({ dns: this.dns })) this.onDetect(await analyze({ dns: this.dns }))
} }
// Validate response // Validate response
@ -736,10 +762,10 @@ class Site {
throw new Error('No response from server') throw new Error('No response from server')
} }
this.onDetect(analyzeDom(dom)) this.onDetect(await analyzeDom(dom))
this.onDetect(analyzeJs(js)) this.onDetect(await analyzeJs(js))
this.onDetect( this.onDetect(
analyze({ await analyze({
url, url,
cookies, cookies,
html, html,
@ -874,7 +900,7 @@ class Site {
this.log(`get ${path}: ok`) this.log(`get ${path}: ok`)
this.onDetect(analyze({ [file]: body })) this.onDetect(await analyze({ [file]: body }))
} catch (error) { } catch (error) {
this.error(`get ${path}: ${error.message || error}`) this.error(`get ${path}: ${error.message || error}`)
} }

122
src/drivers/webextension/js/driver.js
Unescape View File

@ -1,6 +1,6 @@
'use strict' 'use strict'
/* eslint-env browser */ /* eslint-env browser */
/* globals chrome, Wappalyzer, Utils */ /* globals chrome, Wappalyzer, Utils, next */
const { const {
setTechnologies, setTechnologies,
@ -161,17 +161,21 @@ const Driver = {
* @param {String} url * @param {String} url
* @param {Array} js * @param {Array} js
*/ */
analyzeJs(url, js) { async analyzeJs(url, js) {
return Driver.onDetect( return Driver.onDetect(
url, url,
Array.prototype.concat.apply( Array.prototype.concat.apply(
[], [],
js.map(({ name, chain, value }) => await Promise.all(
analyzeManyToMany( js.map(async ({ name, chain, value }) => {
Wappalyzer.technologies.find(({ name: _name }) => name === _name), await next()
'js',
{ [chain]: [value] } return analyzeManyToMany(
) Wappalyzer.technologies.find(({ name: _name }) => name === _name),
'js',
{ [chain]: [value] }
)
})
) )
) )
) )
@ -182,54 +186,58 @@ const Driver = {
* @param {String} url * @param {String} url
* @param {Array} dom * @param {Array} dom
*/ */
analyzeDom(url, dom) { async analyzeDom(url, dom) {
return Driver.onDetect( return Driver.onDetect(
url, url,
Array.prototype.concat.apply( Array.prototype.concat.apply(
[], [],
dom.map( await Promise.all(
( dom.map(
{ name, selector, exists, text, property, attribute, value }, async (
index { name, selector, exists, text, property, attribute, value },
) => { index
const technology = Wappalyzer.technologies.find( ) => {
({ name: _name }) => name === _name await next()
)
const technology = Wappalyzer.technologies.find(
({ name: _name }) => name === _name
)
if (typeof exists !== 'undefined') { if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', { return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''], [selector]: [''],
}) })
} }
if (typeof text !== 'undefined') { if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', { return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text], [selector]: [text],
}) })
} }
if (typeof property !== 'undefined') { if (typeof property !== 'undefined') {
return analyzeManyToMany( return analyzeManyToMany(
technology, technology,
`dom.properties.${property}`, `dom.properties.${property}`,
{ {
[selector]: [value], [selector]: [value],
} }
) )
} }
if (typeof attribute !== 'undefined') { if (typeof attribute !== 'undefined') {
return analyzeManyToMany( return analyzeManyToMany(
technology, technology,
`dom.attributes.${attribute}`, `dom.attributes.${attribute}`,
{ {
[selector]: [value], [selector]: [value],
} }
) )
} }
return [] return []
} }
)
) )
) )
) )
@ -304,7 +312,9 @@ const Driver = {
) )
}) })
Driver.onDetect(request.url, analyze({ headers })).catch(Driver.error) Driver.onDetect(request.url, await analyze({ headers })).catch(
Driver.error
)
} }
} catch (error) { } catch (error) {
Driver.error(error) Driver.error(error)
@ -332,12 +342,13 @@ const Driver = {
if (!xhrDebounce.includes(hostname)) { if (!xhrDebounce.includes(hostname)) {
xhrDebounce.push(hostname) xhrDebounce.push(hostname)
setTimeout(() => { setTimeout(async () => {
xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1) xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1)
Driver.onDetect(request.originUrl, analyze({ xhr: hostname })).catch( Driver.onDetect(
Driver.error request.originUrl,
) await analyze({ xhr: hostname })
).catch(Driver.error)
}, 1000) }, 1000)
} }
}, },
@ -364,7 +375,12 @@ const Driver = {
{} {}
) )
await Driver.onDetect(url, analyze({ url, ...items }), language, true) await Driver.onDetect(
url,
await analyze({ url, ...items }),
language,
true
)
} catch (error) { } catch (error) {
Driver.error(error) Driver.error(error)
} }

42
src/wappalyzer.js
Unescape View File

@ -1,5 +1,13 @@
'use strict' 'use strict'
function next() {
return new Promise((resolve) =>
(typeof setImmediate !== 'undefined'
? setImmediate
: requestAnimationFrame)(resolve)
)
}
function toArray(value) { function toArray(value) {
return Array.isArray(value) ? value : [value] return Array.isArray(value) ? value : [value]
} }
@ -187,7 +195,7 @@ const Wappalyzer = {
* Initialize analyzation. * Initialize analyzation.
* @param {*} param0 * @param {*} param0
*/ */
analyze({ async analyze({
url, url,
xhr, xhr,
html, html,
@ -208,20 +216,24 @@ const Wappalyzer = {
try { try {
const detections = flatten( const detections = flatten(
Wappalyzer.technologies.map((technology) => await Promise.all(
flatten([ Wappalyzer.technologies.map(async (technology) => {
oo(technology, 'url', url), await next()
oo(technology, 'xhr', xhr),
oo(technology, 'html', html), return flatten([
oo(technology, 'css', css), oo(technology, 'url', url),
oo(technology, 'robots', robots), oo(technology, 'xhr', xhr),
oo(technology, 'certIssuer', certIssuer), oo(technology, 'html', html),
om(technology, 'scripts', scripts), oo(technology, 'css', css),
mm(technology, 'cookies', cookies), oo(technology, 'robots', robots),
mm(technology, 'meta', meta), oo(technology, 'certIssuer', certIssuer),
mm(technology, 'headers', headers), om(technology, 'scripts', scripts),
mm(technology, 'dns', dns), mm(technology, 'cookies', cookies),
]) mm(technology, 'meta', meta),
mm(technology, 'headers', headers),
mm(technology, 'dns', dns),
])
})
) )
).filter((technology) => technology) ).filter((technology) => technology)

Write Preview
Loading…
Cancel
Save