From c7417f1b89934d0a948df4365fdccb495a965458 Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Mon, 7 Nov 2022 09:23:28 +1100 Subject: [PATCH] Performance improvements --- src/drivers/npm/driver.js | 62 ++++++++++++++++-- src/drivers/npm/package.json | 2 +- src/drivers/webextension/js/content.js | 77 +++++++++++++++++++++-- src/drivers/webextension/js/popup.js | 2 - src/drivers/webextension/manifest-v2.json | 2 +- src/drivers/webextension/manifest-v3.json | 2 +- src/package.json | 2 +- src/technologies/g.json | 2 +- src/technologies/h.json | 2 +- src/technologies/o.json | 6 +- 10 files changed, 137 insertions(+), 22 deletions(-) diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index f6fe4650c..3137dcfc1 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -131,7 +131,20 @@ function getDom(page, technologies = Wappalyzer.technologies) { dom[selector].forEach(({ exists, text, properties, attributes }) => { nodes.forEach((node) => { - if (exists) { + if ( + technologies.filter(({ name: _name }) => _name === name) + .length >= 50 + ) { + return + } + + if ( + exists && + technologies.findIndex( + ({ name: _name, selector: _selector, exists }) => + name === _name && selector === _selector && exists === '' + ) === -1 + ) { technologies.push({ name, selector, @@ -140,9 +153,18 @@ function getDom(page, technologies = Wappalyzer.technologies) { } if (text) { - const value = node.textContent.trim() - - if (value) { + // eslint-disable-next-line unicorn/prefer-text-content + const value = ( + node.textContent ? node.textContent.trim() : '' + ).slice(0, 1000000) + + if ( + value && + technologies.findIndex( + ({ name: _name, selector: _selector, text }) => + name === _name && selector === _selector && text === value + ) === -1 + ) { technologies.push({ name, selector, @@ -153,7 +175,21 @@ function getDom(page, technologies = Wappalyzer.technologies) { if (properties) { Object.keys(properties).forEach((property) => { - if (Object.prototype.hasOwnProperty.call(node, property)) { + if ( + Object.prototype.hasOwnProperty.call(node, property) && + technologies.findIndex( + ({ + name: _name, + selector: _selector, + property: _property, + value, + }) => + name === _name && + selector === _selector && + property === _property && + value === toScalar(value) + ) === -1 + ) { const value = node[property] if (typeof value !== 'undefined') { @@ -170,7 +206,21 @@ function getDom(page, technologies = Wappalyzer.technologies) { if (attributes) { Object.keys(attributes).forEach((attribute) => { - if (node.hasAttribute(attribute)) { + if ( + node.hasAttribute(attribute) && + technologies.findIndex( + ({ + name: _name, + selector: _selector, + attribute: _atrribute, + value, + }) => + name === _name && + selector === _selector && + attribute === _atrribute && + value === toScalar(value) + ) === -1 + ) { const value = node.getAttribute(attribute) technologies.push({ diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index 5b643223b..3d2728d1f 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -13,7 +13,7 @@ "software" ], "homepage": "https://www.wappalyzer.com/", - "version": "6.10.45", + "version": "6.10.46", "author": "Wappalyzer", "license": "MIT", "repository": { diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js index 44511e07d..46798e0c8 100644 --- a/src/drivers/webextension/js/content.js +++ b/src/drivers/webextension/js/content.js @@ -71,9 +71,22 @@ async function getDom(technologies) { return } - dom[selector].forEach(({ exists, text, attributes }) => { + dom[selector].forEach(({ exists, text, properties, attributes }) => { nodes.forEach((node) => { - if (exists) { + if ( + technologies.filter(({ name: _name }) => _name === name).length >= + 50 + ) { + return + } + + if ( + exists && + technologies.findIndex( + ({ name: _name, selector: _selector, exists }) => + name === _name && selector === _selector && exists === '' + ) === -1 + ) { technologies.push({ name, selector, @@ -83,9 +96,18 @@ async function getDom(technologies) { if (text) { // eslint-disable-next-line unicorn/prefer-text-content - const value = node.innerText ? node.innerText.trim() : '' + const value = (node.innerText ? node.innerText.trim() : '').slice( + 0, + 1000000 + ) - if (value) { + if ( + value && + technologies.findIndex( + ({ name: _name, selector: _selector, text }) => + name === _name && selector === _selector && text === value + ) === -1 + ) { technologies.push({ name, selector, @@ -94,9 +116,54 @@ async function getDom(technologies) { } } + if (properties) { + Object.keys(properties).forEach((property) => { + if ( + Object.prototype.hasOwnProperty.call(node, property) && + technologies.findIndex( + ({ + name: _name, + selector: _selector, + property: _property, + value, + }) => + name === _name && + selector === _selector && + property === _property && + value === toScalar(value) + ) === -1 + ) { + const value = node[property] + + if (typeof value !== 'undefined') { + technologies.push({ + name, + selector, + property, + value: toScalar(value), + }) + } + } + }) + } + if (attributes) { Object.keys(attributes).forEach((attribute) => { - if (node.hasAttribute(attribute)) { + if ( + node.hasAttribute(attribute) && + technologies.findIndex( + ({ + name: _name, + selector: _selector, + attribute: _atrribute, + value, + }) => + name === _name && + selector === _selector && + attribute === _atrribute && + value === toScalar(value) + ) === -1 + ) { const value = node.getAttribute(attribute) technologies.push({ diff --git a/src/drivers/webextension/js/popup.js b/src/drivers/webextension/js/popup.js index ba8b753da..138b3307e 100644 --- a/src/drivers/webextension/js/popup.js +++ b/src/drivers/webextension/js/popup.js @@ -286,8 +286,6 @@ const Popup = { return templates }, {}) - Popup.onGetDetections() - // Disabled domains const dynamicIcon = await getOption('dynamicIcon', false) diff --git a/src/drivers/webextension/manifest-v2.json b/src/drivers/webextension/manifest-v2.json index 2b6c1dd37..6c95c2d7b 100644 --- a/src/drivers/webextension/manifest-v2.json +++ b/src/drivers/webextension/manifest-v2.json @@ -4,7 +4,7 @@ "author": "Wappalyzer", "homepage_url": "https://www.wappalyzer.com/", "description": "Identify web technologies", - "version": "6.10.45", + "version": "6.10.46", "default_locale": "en", "manifest_version": 2, "icons": { diff --git a/src/drivers/webextension/manifest-v3.json b/src/drivers/webextension/manifest-v3.json index 422d41b8a..f119a3244 100644 --- a/src/drivers/webextension/manifest-v3.json +++ b/src/drivers/webextension/manifest-v3.json @@ -4,7 +4,7 @@ "author": "Wappalyzer", "homepage_url": "https://www.wappalyzer.com/", "description": "Identify web technologies", - "version": "6.10.45", + "version": "6.10.46", "default_locale": "en", "manifest_version": 3, "icons": { diff --git a/src/package.json b/src/package.json index 8e042a773..57a13cac6 100644 --- a/src/package.json +++ b/src/package.json @@ -13,7 +13,7 @@ "software" ], "homepage": "https://www.wappalyzer.com/", - "version": "6.10.45", + "version": "6.10.46", "author": "Wappalyzer", "license": "MIT", "repository": { diff --git a/src/technologies/g.json b/src/technologies/g.json index 9be4fe53f..095a4de05 100644 --- a/src/technologies/g.json +++ b/src/technologies/g.json @@ -2307,4 +2307,4 @@ "implies": "Python", "website": "http://gunicorn.org" } -} +} \ No newline at end of file diff --git a/src/technologies/h.json b/src/technologies/h.json index 72e139b0c..e399094c9 100644 --- a/src/technologies/h.json +++ b/src/technologies/h.json @@ -1616,4 +1616,4 @@ "scriptSrc": "merchant\\.cdn\\.hoolah\\.co/", "website": "https://www.hoolah.co" } -} +} \ No newline at end of file diff --git a/src/technologies/o.json b/src/technologies/o.json index c044c7c11..56f334073 100644 --- a/src/technologies/o.json +++ b/src/technologies/o.json @@ -344,9 +344,9 @@ "js": { "OktaAuth": "", "isOktaEnabled": "", - "oktaCurrentSessionUrl": "", "okta.cdnUrlHostname": "", - "okta.locale": "" + "okta.locale": "", + "oktaCurrentSessionUrl": "" }, "pricing": [ "poa", @@ -1777,4 +1777,4 @@ }, "website": "https://owncloud.org" } -} +} \ No newline at end of file