From 670614082e1bd610df1e9848f1cac312c15a6d85 Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Wed, 9 Nov 2022 09:35:47 +1100 Subject: [PATCH] Track rootPath detections --- src/drivers/npm/driver.js | 11 ++++ src/drivers/npm/package.json | 2 +- src/drivers/webextension/js/driver.js | 73 +++++++++++++++-------- src/drivers/webextension/manifest-v2.json | 2 +- src/drivers/webextension/manifest-v3.json | 2 +- src/package.json | 2 +- src/technologies/c.json | 58 +++++++++--------- src/technologies/v.json | 26 ++++---- src/wappalyzer.js | 36 +++++++---- 9 files changed, 128 insertions(+), 84 deletions(-) diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 3137dcfc1..1ca4a49c3 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -1126,6 +1126,7 @@ class Site { website, cpe, categories, + rootPath, }) => ({ slug, name, @@ -1140,6 +1141,7 @@ class Site { slug, name, })), + rootPath, }) ), patterns, @@ -1272,6 +1274,15 @@ class Site { ) === index ) + // Track if technology was identified on website's root path + detections.forEach(({ technology: { name } }) => { + const detection = this.detections.find( + ({ technology: { name: _name } }) => name === _name + ) + + detection.rootPath = detection.rootPath || url.pathname === '/' + }) + if (this.cache[url.href]) { const resolved = resolve(this.detections) diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index 3d2728d1f..fec19983c 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -13,7 +13,7 @@ "software" ], "homepage": "https://www.wappalyzer.com/", - "version": "6.10.46", + "version": "6.10.47", "author": "Wappalyzer", "license": "MIT", "repository": { diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index e7499e7e1..c2782fa94 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -101,10 +101,12 @@ const Driver = { ) } } else if (version !== previous && upgradeMessage) { + /* open( `https://www.wappalyzer.com/upgraded/?utm_source=upgraded&utm_medium=extension&utm_campaign=wappalyzer`, false ) + */ } initDone() @@ -152,6 +154,13 @@ const Driver = { } } + Object.keys(technologies).forEach((name) => { + delete technologies[name].description + delete technologies[name].cpe + delete technologies[name].pricing + delete technologies[name].website + }) + setTechnologies(technologies) setCategories(categories) } catch (error) { @@ -560,7 +569,7 @@ const Driver = { url = url.split('#')[0] - const { hostname } = new URL(url) + const { hostname, pathname } = new URL(url) // Cache detections const cache = (Driver.cache.hostnames[hostname] = { @@ -605,6 +614,39 @@ const Driver = { return detection }) + // Track if technology was identified on website's root path + detections.forEach(({ technology: { name } }) => { + const detection = cache.detections.find( + ({ technology: { name: _name } }) => name === _name + ) + + detection.rootPath = detection.rootPath || pathname === '/' + }) + + const resolved = resolve(cache.detections).map((detection) => detection) + + // Look for technologies that require other technologies to be present on the page + const requires = [ + ...Wappalyzer.requires.filter(({ name }) => + resolved.some(({ name: _name }) => _name === name) + ), + ...Wappalyzer.categoryRequires.filter(({ categoryId }) => + resolved.some(({ categories }) => + categories.some(({ id }) => id === categoryId) + ) + ), + ] + + try { + await Driver.content(url, 'analyzeRequires', [url, requires]) + } catch (error) { + // Continue + } + + await Driver.setIcon(url, resolved) + + await Driver.ping() + cache.hits += incrementHits ? 1 : 0 cache.language = cache.language || language @@ -628,6 +670,7 @@ const Driver = { return hostnames }, {}) + // Save cache await setOption( 'hostnames', Object.keys(Driver.cache.hostnames).reduce( @@ -642,6 +685,7 @@ const Driver = { technology: { name: technology }, pattern: { regex, confidence }, version, + rootPath, lastUrl, }) => ({ technology, @@ -650,6 +694,7 @@ const Driver = { confidence, }, version, + rootPath, lastUrl, }) ), @@ -659,30 +704,7 @@ const Driver = { ) ) - const resolved = resolve(cache.detections).map((detection) => detection) - - const requires = [ - ...Wappalyzer.requires.filter(({ name }) => - resolved.some(({ name: _name }) => _name === name) - ), - ...Wappalyzer.categoryRequires.filter(({ categoryId }) => - resolved.some(({ categories }) => - categories.some(({ id }) => id === categoryId) - ) - ), - ] - - try { - await Driver.content(url, 'analyzeRequires', [url, requires]) - } catch (error) { - // Continue - } - - await Driver.setIcon(url, resolved) - Driver.log({ hostname, technologies: resolved }) - - await Driver.ping() }, /** @@ -930,11 +952,12 @@ const Driver = { if (!hostnameIgnoreList.test(hostname) && hits) { urls[url] = urls[url] || { technologies: resolve(detections).reduce( - (technologies, { name, confidence, version }) => { + (technologies, { name, confidence, version, rootPath }) => { if (confidence === 100) { technologies[name] = { version, hits, + rootPath, } } diff --git a/src/drivers/webextension/manifest-v2.json b/src/drivers/webextension/manifest-v2.json index 6c95c2d7b..f309835b9 100644 --- a/src/drivers/webextension/manifest-v2.json +++ b/src/drivers/webextension/manifest-v2.json @@ -4,7 +4,7 @@ "author": "Wappalyzer", "homepage_url": "https://www.wappalyzer.com/", "description": "Identify web technologies", - "version": "6.10.46", + "version": "6.10.47", "default_locale": "en", "manifest_version": 2, "icons": { diff --git a/src/drivers/webextension/manifest-v3.json b/src/drivers/webextension/manifest-v3.json index f119a3244..ce5335834 100644 --- a/src/drivers/webextension/manifest-v3.json +++ b/src/drivers/webextension/manifest-v3.json @@ -4,7 +4,7 @@ "author": "Wappalyzer", "homepage_url": "https://www.wappalyzer.com/", "description": "Identify web technologies", - "version": "6.10.46", + "version": "6.10.47", "default_locale": "en", "manifest_version": 3, "icons": { diff --git a/src/package.json b/src/package.json index 57a13cac6..2b560a508 100644 --- a/src/package.json +++ b/src/package.json @@ -13,7 +13,7 @@ "software" ], "homepage": "https://www.wappalyzer.com/", - "version": "6.10.46", + "version": "6.10.47", "author": "Wappalyzer", "license": "MIT", "repository": { diff --git a/src/technologies/c.json b/src/technologies/c.json index ee5274ca0..6dff46eac 100644 --- a/src/technologies/c.json +++ b/src/technologies/c.json @@ -147,6 +147,35 @@ }, "website": "http://dragonflycms.org" }, + "CRM+": { + "cats": [ + 53 + ], + "description": "CRM+ is a German CRM software product building on Vtiger with GDPR-compliant extensions and improvements.", + "dom": { + "div.footer > div.floatRight ": { + "text": "Powered by Brainformatik GmbH" + } + }, + "icon": "CRM+.png", + "implies": [ + "MariaDB", + "amCharts", + "Sentry", + "Vtiger" + ], + "pricing": [ + "mid", + "recurring", + "poa", + "freemium" + ], + "requires": [ + "Apache", + "PHP" + ], + "website": "https://www.brainformatik.com" + }, "CS Cart": { "cats": [ 6 @@ -3366,35 +3395,6 @@ ], "website": "http://criteo.com" }, - "CRM+": { - "cats": [ - 53 - ], - "description": "CRM+ is a German CRM software product building on Vtiger with GDPR-compliant extensions and improvements.", - "dom": { - "div.footer > div.floatRight ": { - "text": "Powered by Brainformatik GmbH" - } - }, - "icon": "CRM+.png", - "implies": [ - "MariaDB", - "amCharts", - "Sentry", - "Vtiger" - ], - "requires": [ - "Apache", - "PHP" - ], - "pricing": [ - "mid", - "recurring", - "poa", - "freemium" - ], - "website": "https://www.brainformatik.com" - }, "Crobox": { "cats": [ 5 diff --git a/src/technologies/v.json b/src/technologies/v.json index b137962d9..d3cc8bf29 100644 --- a/src/technologies/v.json +++ b/src/technologies/v.json @@ -140,17 +140,6 @@ "scriptSrc": "cdn\\.pushcrew\\.\\w+", "website": "https://vwo.com/engage" }, - "vxe-table": { - "cats": [ - 59 - ], - "description": "vxe-table is a Vue.js based PC form component, support add, delete, change, virtual scroll, lazy load, shortcut menu, data validation, tree structure, print export, form rendering, data paging, virtual list, modal window, custom template, renderer, flexible configuration items, extension interface.", - "icon": "vxe-table.png", - "requires": "Vue.js", - "dom": "div[class*='vxe-table']", - "oss": true, - "website": "https://vxetable.cn" - }, "Vaadin": { "cats": [ 18 @@ -770,8 +759,8 @@ } }, "icon": "Visx.svg", - "requires": "React", "oss": true, + "requires": "React", "website": "https://airbnb.io/visx/" }, "Vitals": { @@ -1108,5 +1097,16 @@ "generator": "vibecommerce" }, "website": "http://vibecommerce.com.br" + }, + "vxe-table": { + "cats": [ + 59 + ], + "description": "vxe-table is a Vue.js based PC form component, support add, delete, change, virtual scroll, lazy load, shortcut menu, data validation, tree structure, print export, form rendering, data paging, virtual list, modal window, custom template, renderer, flexible configuration items, extension interface.", + "dom": "div[class*='vxe-table']", + "icon": "vxe-table.png", + "oss": true, + "requires": "Vue.js", + "website": "https://vxetable.cn" } -} +} \ No newline at end of file diff --git a/src/wappalyzer.js b/src/wappalyzer.js index 286bb4c1c..38541fd26 100644 --- a/src/wappalyzer.js +++ b/src/wappalyzer.js @@ -104,24 +104,32 @@ const Wappalyzer = { ) { let version = '' let confidence = 0 + let rootPath detections - .filter(({ technology }) => technology) + .filter( + ({ technology: _technology }) => + _technology && _technology.name === technology.name + ) .forEach( - ({ technology: { name }, pattern, version: _version = '' }) => { - if (name === technology.name) { - confidence = Math.min(100, confidence + pattern.confidence) - version = - _version.length > version.length && - _version.length <= 15 && - (parseInt(_version, 10) || 0) < 10000 // Ignore long numeric strings like timestamps - ? _version - : version - } + ({ + technology: { name }, + pattern, + version: _version = '', + rootPath: _rootPath, + }) => { + confidence = Math.min(100, confidence + pattern.confidence) + version = + _version.length > version.length && + _version.length <= 15 && + (parseInt(_version, 10) || 0) < 10000 // Ignore long numeric strings like timestamps + ? _version + : version + rootPath = rootPath || _rootPath || undefined } ) - resolved.push({ technology, confidence, version, lastUrl }) + resolved.push({ technology, confidence, version, rootPath, lastUrl }) } return resolved @@ -152,6 +160,7 @@ const Wappalyzer = { }, confidence, version, + rootPath, lastUrl, }) => ({ name, @@ -164,6 +173,7 @@ const Wappalyzer = { website, pricing, cpe, + rootPath, lastUrl, }) ) @@ -360,7 +370,7 @@ const Wappalyzer = { technologies.push({ name, - description, + description: description || null, categories: cats || [], slug: Wappalyzer.slugify(name), url: transform(url),