Migrations
/
wappalyzer
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Performance improvements

Browse Source
main
Elbert Alias 3 years ago
parent 791fe5e828
commit 460a97d5d9
4 changed files with 122 additions and 172 deletions
Show Stats Download Patch File Download Diff File

170
src/drivers/npm/driver.js
Unescape View File

@ -10,10 +10,6 @@ const Wappalyzer = require('./wappalyzer')
const { setTechnologies, setCategories, analyze, analyzeManyToMany, resolve } =
Wappalyzer
function next() {
return new Promise((resolve) => setImmediate(resolve))
}
const { CHROMIUM_BIN, CHROMIUM_DATA_DIR, CHROMIUM_WEBSOCKET } = process.env
const chromiumArgs = [
@ -98,21 +94,16 @@ function getJs(page, technologies = Wappalyzer.technologies) {
}, technologies)
}
async function analyzeJs(js, technologies = Wappalyzer.technologies) {
return Array.prototype.concat.apply(
[],
await Promise.all(
js.map(async ({ name, chain, value }) => {
await next()
return analyzeManyToMany(
technologies.find(({ name: _name }) => name === _name),
'js',
{ [chain]: [value] }
)
})
)
)
function analyzeJs(js, technologies = Wappalyzer.technologies) {
return js
.map(({ name, chain, value }) => {
return analyzeManyToMany(
technologies.find(({ name: _name }) => name === _name),
'js',
{ [chain]: [value] }
)
})
.flat()
}
function getDom(page, technologies = Wappalyzer.technologies) {
@ -200,59 +191,36 @@ function getDom(page, technologies = Wappalyzer.technologies) {
}, technologies)
}
async function analyzeDom(dom, technologies = Wappalyzer.technologies) {
return Array.prototype.concat.apply(
[],
await Promise.all(
dom.map(
async ({
name,
selector,
exists,
text,
property,
attribute,
value,
}) => {
await next()
const technology = technologies.find(
({ name: _name }) => name === _name
)
function analyzeDom(dom, technologies = Wappalyzer.technologies) {
return dom
.map(({ name, selector, exists, text, property, attribute, value }) => {
const technology = technologies.find(({ name: _name }) => name === _name)
if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''],
})
}
if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}
if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''],
})
}
if (typeof property !== 'undefined') {
return analyzeManyToMany(technology, `dom.properties.${property}`, {
[selector]: [value],
})
}
if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}
if (typeof attribute !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.attributes.${attribute}`,
{
[selector]: [value],
}
)
}
if (typeof property !== 'undefined') {
return analyzeManyToMany(technology, `dom.properties.${property}`, {
[selector]: [value],
})
}
return []
}
)
)
)
if (typeof attribute !== 'undefined') {
return analyzeManyToMany(technology, `dom.attributes.${attribute}`, {
[selector]: [value],
})
}
})
.flat()
}
function get(url, options = {}) {
@ -860,22 +828,20 @@ class Site {
await this.onDetect(
url,
(
await Promise.all([
analyzeDom(dom),
analyzeJs(js),
analyze({
url,
cookies,
html,
text,
css,
scripts,
scriptSrc,
meta,
}),
])
).flat()
[
analyzeDom(dom),
analyzeJs(js),
analyze({
url,
cookies,
html,
text,
css,
scripts,
scriptSrc,
meta,
}),
].flat()
)
const reducedLinks = Array.prototype.reduce.call(
@ -1189,25 +1155,23 @@ class Site {
await this.onDetect(
url,
(
await Promise.all([
analyzeDom(dom, technologies),
analyzeJs(js, technologies),
analyze(
{
url,
cookies,
html,
text,
css,
scripts,
scriptSrc,
meta,
},
technologies
),
])
).flat()
[
analyzeDom(dom, technologies),
analyzeJs(js, technologies),
await analyze(
{
url,
cookies,
html,
text,
css,
scripts,
scriptSrc,
meta,
},
technologies
),
].flat()
)
}
})

4
src/drivers/webextension/js/content.js
Unescape View File

@ -288,7 +288,9 @@ const Content = {
// Delayed second pass to capture async JS
await new Promise((resolve) => setTimeout(resolve, 5000))
await Content.onGetTechnologies(technologies)
const js = await getJs(technologies)
await Content.driver('analyzeJs', [url, js])
} catch (error) {
Content.driver('error', error)
}

118
src/drivers/webextension/js/driver.js
Unescape View File

@ -1,6 +1,6 @@
'use strict'
/* eslint-env browser */
/* globals chrome, Wappalyzer, Utils, next */
/* globals chrome, Wappalyzer, Utils */
const {
setTechnologies,
@ -217,27 +217,22 @@ const Driver = {
* @param {String} url
* @param {Array} js
*/
async analyzeJs(url, js, requires, categoryRequires) {
analyzeJs(url, js, requires, categoryRequires) {
const technologies =
getRequiredTechnologies(requires, categoryRequires) ||
Wappalyzer.technologies
return Driver.onDetect(
url,
Array.prototype.concat.apply(
[],
await Promise.all(
js.map(async ({ name, chain, value }) => {
await next()
return analyzeManyToMany(
technologies.find(({ name: _name }) => name === _name),
'js',
{ [chain]: [value] }
)
})
)
)
js
.map(({ name, chain, value }) => {
return analyzeManyToMany(
technologies.find(({ name: _name }) => name === _name),
'js',
{ [chain]: [value] }
)
})
.flat()
)
},
@ -246,64 +241,57 @@ const Driver = {
* @param {String} url
* @param {Array} dom
*/
async analyzeDom(url, dom, requires, categoryRequires) {
analyzeDom(url, dom, requires, categoryRequires) {
const technologies =
getRequiredTechnologies(requires, categoryRequires) ||
Wappalyzer.technologies
return Driver.onDetect(
url,
Array.prototype.concat.apply(
[],
await Promise.all(
dom.map(
async (
{ name, selector, exists, text, property, attribute, value },
index
) => {
await next()
const technology = technologies.find(
({ name: _name }) => name === _name
)
if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''],
})
}
dom
.map(
(
{ name, selector, exists, text, property, attribute, value },
index
) => {
const technology = technologies.find(
({ name: _name }) => name === _name
)
if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}
if (typeof exists !== 'undefined') {
return analyzeManyToMany(technology, 'dom.exists', {
[selector]: [''],
})
}
if (typeof property !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.properties.${property}`,
{
[selector]: [value],
}
)
}
if (typeof text !== 'undefined') {
return analyzeManyToMany(technology, 'dom.text', {
[selector]: [text],
})
}
if (typeof attribute !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.attributes.${attribute}`,
{
[selector]: [value],
}
)
}
if (typeof property !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.properties.${property}`,
{
[selector]: [value],
}
)
}
return []
if (typeof attribute !== 'undefined') {
return analyzeManyToMany(
technology,
`dom.attributes.${attribute}`,
{
[selector]: [value],
}
)
}
)
}
)
)
.flat()
)
},
@ -673,11 +661,7 @@ const Driver = {
),
]
try {
await Driver.content(url, 'analyzeRequires', [url, requires])
} catch (error) {
// Continue
}
Driver.content(url, 'analyzeRequires', [url, requires])
await Driver.setIcon(url, resolved)

2
src/technologies/l.json
Unescape View File

@ -1105,7 +1105,7 @@
],
"description": "LottieFiles is an open-source animation file format that's tiny, high quality, interactive, and can be manipulated at runtime.",
"icon": "LottieFiles.svg",
"dom": "clipPath[id*='__lottie_element_']",
"dom": "lottie-player, div[data-animation-type='lottie'], div[data-testid='lottie-player'], clipPath[id*='__lottie_element_']",
"scriptSrc": [
"/dist/lottie-player\\.js",
"/dist/tgs-player\\.js"