Migrations
/
wappalyzer
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Performance improvements

Browse Source
main
Elbert Alias 3 years ago
parent 791fe5e828
commit 460a97d5d9
4 changed files with 122 additions and 172 deletions
Show Stats Download Patch File Download Diff File

170
src/drivers/npm/driver.js
Unescape View File

@ -10,10 +10,6 @@ const Wappalyzer = require('./wappalyzer')
const { setTechnologies, setCategories, analyze, analyzeManyToMany, resolve } = const { setTechnologies, setCategories, analyze, analyzeManyToMany, resolve } =
Wappalyzer Wappalyzer
function next() {
return new Promise((resolve) => setImmediate(resolve))
}
const { CHROMIUM_BIN, CHROMIUM_DATA_DIR, CHROMIUM_WEBSOCKET } = process.env const { CHROMIUM_BIN, CHROMIUM_DATA_DIR, CHROMIUM_WEBSOCKET } = process.env
const chromiumArgs = [ const chromiumArgs = [
@ -98,21 +94,16 @@ function getJs(page, technologies = Wappalyzer.technologies) {
}, technologies) }, technologies)
} }
async function analyzeJs(js, technologies = Wappalyzer.technologies) { function analyzeJs(js, technologies = Wappalyzer.technologies) {
return Array.prototype.concat.apply( return js
[], .map(({ name, chain, value }) => {
await Promise.all( return analyzeManyToMany(
js.map(async ({ name, chain, value }) => { technologies.find(({ name: _name }) => name === _name),
await next() 'js',
{ [chain]: [value] }
return analyzeManyToMany( )
technologies.find(({ name: _name }) => name === _name), })
'js', .flat()
{ [chain]: [value] }
)
})
)
)
} }
function getDom(page, technologies = Wappalyzer.technologies) { function getDom(page, technologies = Wappalyzer.technologies) {
@ -200,59 +191,36 @@ function getDom(page, technologies = Wappalyzer.technologies) {
}, technologies) }, technologies)
} }
async function analyzeDom(dom, technologies = Wappalyzer.technologies) { function analyzeDom(dom, technologies = Wappalyzer.technologies) {
return Array.prototype.concat.apply( return dom
[], .map(({ name, selector, exists, text, property, attribute, value }) => {
await Promise.all( const technology = technologies.find(({ name: _name }) => name === _name)
dom.map(
async ({
name,
selector,
exists,
text,
property,
attribute,
value,
}) => {
await next()
const technology = technologies.find(
({ name: _name }) => name === _name
)
if (typeof exists !== 'undefined') { if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', { return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''], [selector]: [''],
}) })
} }
if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}
if (typeof property !== 'undefined') { if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, `dom.properties.${property}`, { return analyzeManyToMany(technology, 'dom.text', {
[selector]: [value], [selector]: [text],
}) })
} }
if (typeof attribute !== 'undefined') { if (typeof property !== 'undefined') {
return analyzeManyToMany( return analyzeManyToMany(technology, `dom.properties.${property}`, {
technology, [selector]: [value],
`dom.attributes.${attribute}`, })
{ }
[selector]: [value],
}
)
}
return [] if (typeof attribute !== 'undefined') {
} return analyzeManyToMany(technology, `dom.attributes.${attribute}`, {
) [selector]: [value],
) })
) }
})
.flat()
} }
function get(url, options = {}) { function get(url, options = {}) {
@ -860,22 +828,20 @@ class Site {
await this.onDetect( await this.onDetect(
url, url,
( [
await Promise.all([ analyzeDom(dom),
analyzeDom(dom), analyzeJs(js),
analyzeJs(js), analyze({
analyze({ url,
url, cookies,
cookies, html,
html, text,
text, css,
css, scripts,
scripts, scriptSrc,
scriptSrc, meta,
meta, }),
}), ].flat()
])
).flat()
) )
const reducedLinks = Array.prototype.reduce.call( const reducedLinks = Array.prototype.reduce.call(
@ -1189,25 +1155,23 @@ class Site {
await this.onDetect( await this.onDetect(
url, url,
( [
await Promise.all([ analyzeDom(dom, technologies),
analyzeDom(dom, technologies), analyzeJs(js, technologies),
analyzeJs(js, technologies), await analyze(
analyze( {
{ url,
url, cookies,
cookies, html,
html, text,
text, css,
css, scripts,
scripts, scriptSrc,
scriptSrc, meta,
meta, },
}, technologies
technologies ),
), ].flat()
])
).flat()
) )
} }
}) })

4
src/drivers/webextension/js/content.js
Unescape View File

@ -288,7 +288,9 @@ const Content = {
// Delayed second pass to capture async JS // Delayed second pass to capture async JS
await new Promise((resolve) => setTimeout(resolve, 5000)) await new Promise((resolve) => setTimeout(resolve, 5000))
await Content.onGetTechnologies(technologies) const js = await getJs(technologies)
await Content.driver('analyzeJs', [url, js])
} catch (error) { } catch (error) {
Content.driver('error', error) Content.driver('error', error)
} }

118
src/drivers/webextension/js/driver.js
Unescape View File

@ -1,6 +1,6 @@
'use strict' 'use strict'
/* eslint-env browser */ /* eslint-env browser */
/* globals chrome, Wappalyzer, Utils, next */ /* globals chrome, Wappalyzer, Utils */
const { const {
setTechnologies, setTechnologies,
@ -217,27 +217,22 @@ const Driver = {
* @param {String} url * @param {String} url
* @param {Array} js * @param {Array} js
*/ */
async analyzeJs(url, js, requires, categoryRequires) { analyzeJs(url, js, requires, categoryRequires) {
const technologies = const technologies =
getRequiredTechnologies(requires, categoryRequires) || getRequiredTechnologies(requires, categoryRequires) ||
Wappalyzer.technologies Wappalyzer.technologies
return Driver.onDetect( return Driver.onDetect(
url, url,
Array.prototype.concat.apply( js
[], .map(({ name, chain, value }) => {
await Promise.all( return analyzeManyToMany(
js.map(async ({ name, chain, value }) => { technologies.find(({ name: _name }) => name === _name),
await next() 'js',
{ [chain]: [value] }
return analyzeManyToMany( )
technologies.find(({ name: _name }) => name === _name), })
'js', .flat()
{ [chain]: [value] }
)
})
)
)
) )
}, },
@ -246,64 +241,57 @@ const Driver = {
* @param {String} url * @param {String} url
* @param {Array} dom * @param {Array} dom
*/ */
async analyzeDom(url, dom, requires, categoryRequires) { analyzeDom(url, dom, requires, categoryRequires) {
const technologies = const technologies =
getRequiredTechnologies(requires, categoryRequires) || getRequiredTechnologies(requires, categoryRequires) ||
Wappalyzer.technologies Wappalyzer.technologies
return Driver.onDetect( return Driver.onDetect(
url, url,
Array.prototype.concat.apply( dom
[], .map(
await Promise.all( (
dom.map( { name, selector, exists, text, property, attribute, value },
async ( index
{ name, selector, exists, text, property, attribute, value }, ) => {
index const technology = technologies.find(
) => { ({ name: _name }) => name === _name
await next() )
const technology = technologies.find(
({ name: _name }) => name === _name
)
if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''],
})
}
if (typeof text !== 'undefined') { if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', { return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [text], [selector]: [''],
}) })
} }
if (typeof property !== 'undefined') { if (typeof text !== 'undefined') {
return analyzeManyToMany( return analyzeManyToMany(technology, 'dom.text', {
technology, [selector]: [text],
`dom.properties.${property}`, })
{ }
[selector]: [value],
}
)
}
if (typeof attribute !== 'undefined') { if (typeof property !== 'undefined') {
return analyzeManyToMany( return analyzeManyToMany(
technology, technology,
`dom.attributes.${attribute}`, `dom.properties.${property}`,
{ {
[selector]: [value], [selector]: [value],
} }
) )
} }
return [] if (typeof attribute !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.attributes.${attribute}`,
{
[selector]: [value],
}
)
} }
) }
) )
) .flat()
) )
}, },
@ -673,11 +661,7 @@ const Driver = {
), ),
] ]
try { Driver.content(url, 'analyzeRequires', [url, requires])
await Driver.content(url, 'analyzeRequires', [url, requires])
} catch (error) {
// Continue
}
await Driver.setIcon(url, resolved) await Driver.setIcon(url, resolved)

2
src/technologies/l.json
Unescape View File

@ -1105,7 +1105,7 @@
], ],
"description": "LottieFiles is an open-source animation file format that's tiny, high quality, interactive, and can be manipulated at runtime.", "description": "LottieFiles is an open-source animation file format that's tiny, high quality, interactive, and can be manipulated at runtime.",
"icon": "LottieFiles.svg", "icon": "LottieFiles.svg",
"dom": "clipPath[id*='__lottie_element_']", "dom": "lottie-player, div[data-animation-type='lottie'], div[data-testid='lottie-player'], clipPath[id*='__lottie_element_']",
"scriptSrc": [ "scriptSrc": [
"/dist/lottie-player\\.js", "/dist/lottie-player\\.js",
"/dist/tgs-player\\.js" "/dist/tgs-player\\.js"