From c5909d314ed8032c366524df560f2b00c3025765 Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Fri, 21 Jan 2022 08:54:21 +1100 Subject: [PATCH] Add option for extended output --- src/drivers/npm/cli.js | 2 + src/drivers/npm/driver.js | 29 +++++ src/drivers/npm/package.json | 2 +- src/drivers/webextension/manifest.json | 2 +- src/package.json | 2 +- src/technologies/_.json | 13 ++- src/technologies/a.json | 146 ++++++++++++------------- src/technologies/b.json | 6 +- src/technologies/d.json | 2 +- src/technologies/h.json | 31 +++--- src/technologies/i.json | 6 +- src/technologies/k.json | 6 +- src/technologies/m.json | 32 +++--- src/technologies/r.json | 36 +++--- src/technologies/v.json | 34 +++--- src/wappalyzer.js | 6 +- 16 files changed, 197 insertions(+), 158 deletions(-) diff --git a/src/drivers/npm/cli.js b/src/drivers/npm/cli.js index 423ab999a..26cad7556 100755 --- a/src/drivers/npm/cli.js +++ b/src/drivers/npm/cli.js @@ -23,6 +23,7 @@ const aliases = { w: 'maxWait', n: 'noScripts', N: 'noRedirect', + e: 'extended', } while (true) { @@ -77,6 +78,7 @@ Options: -a, --user-agent=... Set the user agent string -n, --no-scripts Disabled JavaScript on web pages -N, --no-redirect Disable cross-domain redirects + -e, --extended Output additional information `) process.exit(1) diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 14102ad02..b2475823a 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -280,6 +280,7 @@ class Driver { noScripts: false, userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36', + extended: false, ...options, } @@ -293,6 +294,7 @@ class Driver { this.options.htmlMaxCols = parseInt(this.options.htmlMaxCols, 10) this.options.htmlMaxRows = parseInt(this.options.htmlMaxRows, 10) this.options.noScripts = Boolean(+this.options.noScripts) + this.options.extended = Boolean(+this.options.extended) this.destroyed = false } @@ -990,6 +992,32 @@ class Site { this.error(error) } + const patterns = this.options.extended + ? this.detections.reduce( + ( + patterns, + { + technology: { name, implies, excludes }, + pattern: { regex, confidence, type, version }, + } + ) => { + patterns[name] = patterns[name] || [] + + patterns[name].push({ + type, + regex: regex.source, + confidence, + version, + implies: implies.map(({ name }) => name), + excludes: excludes.map(({ name }) => name), + }) + + return patterns + }, + {} + ) + : undefined + const results = { urls: this.analyzedUrls, technologies: resolve(this.detections).map( @@ -1017,6 +1045,7 @@ class Site { })), }) ), + patterns, } await this.emit('analyze', results) diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index 3f41db6ff..e5b7ca938 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -13,7 +13,7 @@ "software" ], "homepage": "https://www.wappalyzer.com/", - "version": "6.9.12", + "version": "6.9.13", "author": "Wappalyzer", "license": "MIT", "repository": { diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json index bb840fc02..5f689bf4f 100644 --- a/src/drivers/webextension/manifest.json +++ b/src/drivers/webextension/manifest.json @@ -4,7 +4,7 @@ "author": "Wappalyzer", "homepage_url": "https://www.wappalyzer.com/", "description": "Identify web technologies", - "version": "6.9.12", + "version": "6.9.13", "default_locale": "en", "manifest_version": 2, "icons": { diff --git a/src/package.json b/src/package.json index 54c2b2190..91529c926 100644 --- a/src/package.json +++ b/src/package.json @@ -13,7 +13,7 @@ "software" ], "homepage": "https://www.wappalyzer.com/", - "version": "6.9.12", + "version": "6.9.13", "author": "Wappalyzer", "license": "MIT", "repository": { diff --git a/src/technologies/_.json b/src/technologies/_.json index 3ccfb80e6..cd58c67ac 100644 --- a/src/technologies/_.json +++ b/src/technologies/_.json @@ -118,15 +118,20 @@ }, "8base": { "cats": [ - 3, 62 + 3, + 62 ], "description": "8base is a Backend-as-a-Service. It lets front-end developers focus more on what they love and backend developers expedite their workflow", "icon": "8base-logo.png", "pricing": [ - "low", "freemium", "recurring" + "low", + "freemium", + "recurring" + ], + "requires": [ + "GraphQL" ], "saas": true, - "requires": ["GraphQL"], "website": "https://8base.com", "xhr": "api\\.8base\\.com" }, @@ -140,4 +145,4 @@ }, "website": "https://github.com/sulu/web-js" } -} +} \ No newline at end of file diff --git a/src/technologies/a.json b/src/technologies/a.json index 2093d6b97..facbb3a45 100644 --- a/src/technologies/a.json +++ b/src/technologies/a.json @@ -77,6 +77,26 @@ "scriptSrc": "\\.ebis\\.ne\\.jp/", "website": "http://www.ebis.ne.jp" }, + "ADFOX": { + "cats": [ + 36 + ], + "description": "ADFOX is an advertising management platform for media publishers.", + "icon": "ADFOX.svg", + "js": { + "AdFox_getCodeScript": "", + "Site.adFoxParams": "", + "adFoxParams": "", + "adfoxAsyncParams": "", + "adfoxBiddersMap": "" + }, + "pricing": [ + "mid", + "recurring" + ], + "saas": true, + "website": "https://adfox.yandex.ru" + }, "AFThemes CoverNews": { "cats": [ 80 @@ -721,19 +741,19 @@ 36 ], "description": "AdRiver is a company which provide internet advertising management and audit software.", - "icon": "AdRiver.png", "dom": "link[href*='.adriver.ru'], img[src*='.adriver.ru'], iframe[src*='.adriver.ru']", + "icon": "AdRiver.png", "js": { - "adriver": "", "AdriverCounter": "", "AdriverPrebid": "", - "adfoxBiddersMap.adriver": "" + "adfoxBiddersMap.adriver": "", + "adriver": "" }, - "scriptSrc": "\\.adriver\\.ru/", - "saas": true, "pricing": [ "payg" ], + "saas": true, + "scriptSrc": "\\.adriver\\.ru/", "website": "http://adriver.ru" }, "AdRoll": { @@ -988,26 +1008,6 @@ "scriptSrc": "\\.adform\\.net/", "website": "https://site.adform.com" }, - "ADFOX": { - "cats": [ - 36 - ], - "description": "ADFOX is an advertising management platform for media publishers.", - "icon": "ADFOX.svg", - "js": { - "Site.adFoxParams": "", - "adfoxAsyncParams": "", - "AdFox_getCodeScript": "", - "adfoxBiddersMap": "", - "adFoxParams": "" - }, - "pricing": [ - "mid", - "recurring" - ], - "saas": true, - "website": "https://adfox.yandex.ru" - }, "Adloox": { "cats": [ 36 @@ -1094,11 +1094,11 @@ 36 ], "description": "Adnegah is a digital marketing and internet advertising agency.", - "icon": "Adnegah.png", "dom": "iframe[scr*='adnegah.net']", "headers": { "X-Advertising-By": "adnegah\\.net" }, + "icon": "Adnegah.png", "scriptSrc": "\\.adnegah\\.net/", "website": "https://adnegah.net" }, @@ -1349,13 +1349,13 @@ 36 ], "description": "Adverticum is the developer and operator of Hungary's market leading online ad serving solution, the Adverticum AdServer.", - "icon": "Adverticum.svg", "dom": "a[href*='ad.adverticum.net'], div.goAdverticum", - "scriptSrc": "\\.adverticum\\.net/", - "saas": true, + "icon": "Adverticum.svg", "pricing": [ "poa" ], + "saas": true, + "scriptSrc": "\\.adverticum\\.net/", "website": "http://adverticum.net" }, "Adyen": { @@ -1404,12 +1404,12 @@ ], "description": "Affilae is an affiliate marketing platform that enables brands to connect, collaborate with influencers and affiliates.", "icon": "Affilae.png", - "scriptSrc": "static\\.affilae\\.com/(?:.+v([\\d\\.]+)|.+)?\\.js\\;version:\\1", - "saas": true, "pricing": [ "mid", "recurring" ], + "saas": true, + "scriptSrc": "static\\.affilae\\.com/(?:.+v([\\d\\.]+)|.+)?\\.js\\;version:\\1", "website": "https://affilae.com" }, "Affiliate B": { @@ -1482,14 +1482,14 @@ "js": { "AfterbuyString": "" }, - "scriptSrc": "\\.afterbuy\\.de/", - "saas": true, "pricing": [ "freemium", "low", "recurring", "payg" ], + "saas": true, + "scriptSrc": "\\.afterbuy\\.de/", "website": "http://www.afterbuy.de" }, "Afterpay": { @@ -1591,8 +1591,8 @@ 19 ], "description": "Airform is a functional HTML forms for front-end developers.", - "icon": "Airform.svg", "dom": "form[action*='airform.io/']", + "icon": "Airform.svg", "oss": true, "pricing": [ "freemium" @@ -2665,12 +2665,12 @@ "APOS_DIALOGS.dialogAttributes": "", "apos.csrfCookieName": "" }, - "saas": true, "pricing": [ "freemium", "mid", "recurring" ], + "saas": true, "website": "https://apostrophecms.com" }, "AppDynamics": { @@ -2782,12 +2782,12 @@ } }, "icon": "Apple.svg", - "meta": { - "appleid-signin-client-id": "" - }, "js": { "AppleID": "" }, + "meta": { + "appleid-signin-client-id": "" + }, "scriptSrc": "appleid\\.auth\\.js", "website": "https://developer.apple.com/sign-in-with-apple/" }, @@ -2796,8 +2796,8 @@ 72 ], "description": "Appointy is a cloud-based scheduling solution that helps professionals and businesses to manage their appointment scheduling activities and routines.", + "dom": "a[href*='.appointy.com/'][target='_blank'], iframe[src*='.appointy.com/']", "icon": "Appointy.png", - "dom":"a[href*='.appointy.com/'][target='_blank'], iframe[src*='.appointy.com/']", "pricing": [ "low", "freemium", @@ -2878,19 +2878,19 @@ ], "cpe": "cpe:/a:arastta:ecommerce", "description": "Arastta is a free and open-source project with contributors from all over the world.", - "icon": "Arastta.svg", "excludes": "OpenCart", - "implies": "PHP", "headers": { "Arastta": "^(.+)$\\;version:\\1", "X-Arastta": "", "x-arastta": "" }, - "scriptSrc": "arastta\\.js", + "icon": "Arastta.svg", + "implies": "PHP", "oss": true, "pricing": [ "freemium" ], + "scriptSrc": "arastta\\.js", "website": "http://arastta.org" }, "Arc": { @@ -2998,6 +2998,16 @@ }, "website": "http://www.arvancloud.com" }, + "Asana": { + "cats": [ + 13 + ], + "description": "Asana is a web and mobile application designed to help teams organize, track, and manage their work.", + "dom": "a[href*='form.asana.com'], a[href*='app.asana.com']", + "icon": "Asana.svg", + "saas": true, + "website": "https://asana.com" + }, "AsciiDoc": { "cats": [ 1, @@ -3061,26 +3071,6 @@ "scriptSrc": "themes/astra\\S*\\.js(?:\\?ver=([0-9.]+))?\\;version:\\1", "website": "https://wpastra.com/" }, - "Astro": { - "cats": [ - 57, - 12 - ], - "description": "Astro is a new JavaScript-based static site builder.", - "icon": "Astro.svg", - "dom": { - "link[href*='/_astro/']": { - "attributes": { - "href": "/_astro/(?:common|index)-[\\d\\w]+\\.css" - } - } - }, - "oss": true, - "pricing": [ - "freemium" - ], - "website": "https://astro.build" - }, "Astra Widgets": { "cats": [ 87, @@ -3104,6 +3094,26 @@ ], "website": "https://wpastra.com/did-you-know-astra-is-widget-ready" }, + "Astro": { + "cats": [ + 57, + 12 + ], + "description": "Astro is a new JavaScript-based static site builder.", + "dom": { + "link[href*='/_astro/']": { + "attributes": { + "href": "/_astro/(?:common|index)-[\\d\\w]+\\.css" + } + } + }, + "icon": "Astro.svg", + "oss": true, + "pricing": [ + "freemium" + ], + "website": "https://astro.build" + }, "Astute Solutions": { "cats": [ 53 @@ -3768,15 +3778,5 @@ "icon": "augmented-ui.png", "oss": true, "website": "http://augmented-ui.com" - }, - "Asana": { - "cats": [ - 13 - ], - "description": "Asana is a web and mobile application designed to help teams organize, track, and manage their work.", - "dom": "a[href*='form.asana.com'], a[href*='app.asana.com']", - "icon": "Asana.svg", - "saas": true, - "website": "https://asana.com" } -} +} \ No newline at end of file diff --git a/src/technologies/b.json b/src/technologies/b.json index 3e54c0d99..936be616c 100644 --- a/src/technologies/b.json +++ b/src/technologies/b.json @@ -1021,16 +1021,16 @@ ], "description": "Boost Commerce provides beautiful and advanced product filter and smart site search for Shopify stores to boost sales.", "icon": "Boost Commerce.png", - "requires": "Shopify", "js": { "bcSfFilterConfig.api.filterUrl": "services\\.mybcapps\\.com/", "boostPFSAppConfig.api.filterUrl": "services\\.mybcapps\\.com/" }, - "saas": true, "pricing": [ "payg", "recurring" ], + "requires": "Shopify", + "saas": true, "website": "https://boostcommerce.net" }, "Bootstrap": { @@ -1801,4 +1801,4 @@ }, "website": "https://addyosmani.github.io/basket.js/" } -} +} \ No newline at end of file diff --git a/src/technologies/d.json b/src/technologies/d.json index 90ca52195..8462763c6 100644 --- a/src/technologies/d.json +++ b/src/technologies/d.json @@ -1339,4 +1339,4 @@ "scriptSrc": "deepMiner\\.js", "website": "https://github.com/deepwn/deepMiner" } -} +} \ No newline at end of file diff --git a/src/technologies/h.json b/src/technologies/h.json index 3e30ce2a0..7f3b3fb5c 100644 --- a/src/technologies/h.json +++ b/src/technologies/h.json @@ -82,6 +82,18 @@ "scriptSrc": "haddock-util\\.js", "website": "http://www.haskell.org/haddock/" }, + "Halo": { + "cats": [ + 1, + 11 + ], + "icon": "Halo.png", + "implies": "Java", + "meta": { + "generator": "Halo ([\\d.]+)?\\;version:\\1" + }, + "website": "https://halo.run" + }, "Hamechio": { "cats": [ 18 @@ -98,18 +110,6 @@ ], "website": "https://hamech.io" }, - "Halo": { - "cats": [ - 1, - 11 - ], - "icon": "Halo.png", - "implies": "Java", - "meta": { - "generator": "Halo ([\\d.]+)?\\;version:\\1" - }, - "website": "https://halo.run" - }, "Hammer.js": { "cats": [ 59 @@ -833,7 +833,10 @@ "_hsq": "", "hubspot": "" }, - "pricing": ["recurring", "high"], + "pricing": [ + "recurring", + "high" + ], "saas": true, "website": "https://www.hubspot.com" }, @@ -1140,4 +1143,4 @@ "scriptSrc": "merchant\\.cdn\\.hoolah\\.co/", "website": "https://www.hoolah.co" } -} +} \ No newline at end of file diff --git a/src/technologies/i.json b/src/technologies/i.json index 1fe28b33a..127e66736 100644 --- a/src/technologies/i.json +++ b/src/technologies/i.json @@ -215,16 +215,16 @@ "description": "Imber is an all-in-one marketing automation platform built for customer support (live chat), sales, and marketing.", "icon": "Imber.png", "js": { - "IMBER_ID": "", "$imber.getVisitor": "", + "IMBER_ID": "", "IMBER_SOCKET": "" }, - "saas": true, "pricing": [ "freemium", "low", "recurring" ], + "saas": true, "website": "https://imber.live" }, "Immutable.js": { @@ -1080,4 +1080,4 @@ ], "website": "https://www.iubenda.com/" } -} +} \ No newline at end of file diff --git a/src/technologies/k.json b/src/technologies/k.json index f7f937082..136a74b73 100644 --- a/src/technologies/k.json +++ b/src/technologies/k.json @@ -335,17 +335,17 @@ 36 ], "description": "Kevel (formerly Adzerk) is a developer of ad-serving APIs to help developers build server-side ad platforms.", - "icon": "Kevel.svg", "dom": "iframe[src*='adzerk.net'], link[href*='adzerk.net']", + "icon": "Kevel.svg", "js": { "ados": "", "adosResults": "" }, - "scriptSrc": "adzerk\\.net/", - "saas": true, "pricing": [ "poa" ], + "saas": true, + "scriptSrc": "adzerk\\.net/", "website": "https://www.kevel.com" }, "KeyCDN": { diff --git a/src/technologies/m.json b/src/technologies/m.json index 13c1efb65..5b28b03f1 100644 --- a/src/technologies/m.json +++ b/src/technologies/m.json @@ -30,6 +30,22 @@ "saas": true, "website": "https://ma-jin.jp" }, + "MDUI": { + "cats": [ + 66 + ], + "description": "MDUI is a CSS Framework based on material design.", + "icon": "MDUI.png", + "js": { + "_mduiEventId": "", + "mdui.Drawer": "" + }, + "oss": true, + "pricing": [ + "freemium" + ], + "website": "https://www.mdui.org" + }, "MGID": { "cats": [ 36 @@ -710,22 +726,6 @@ }, "website": "https://maxemail.xtremepush.com" }, - "MDUI": { - "cats": [ - 66 - ], - "description": "MDUI is a CSS Framework based on material design.", - "icon": "MDUI.png", - "js": { - "mdui.Drawer": "", - "_mduiEventId": "" - }, - "oss": true, - "pricing": [ - "freemium" - ], - "website": "https://www.mdui.org" - }, "Medallia": { "cats": [ 10, diff --git a/src/technologies/r.json b/src/technologies/r.json index ae3a09edb..59eca5e4a 100644 --- a/src/technologies/r.json +++ b/src/technologies/r.json @@ -52,6 +52,23 @@ }, "website": "https://github.com/ruby/rdoc" }, + "REDAXO": { + "cats": [ + 1 + ], + "description": "REDAXO is a content management system that provides business optimisation through web projects and output codes.", + "icon": "REDAXO.png", + "implies": "PHP", + "js": { + "redaxo": "\\;confidence:50" + }, + "oss": true, + "pricing": [ + "freemium" + ], + "url": "^https?://(?:www\\.)?[\\d\\w\\-]+\\.[\\w]+/redaxo/\\;confidence:50", + "website": "https://redaxo.org" + }, "REG.RU": { "cats": [ 88 @@ -581,23 +598,6 @@ "scriptSrc": "js\\.recurly\\.com", "website": "https://recurly.com" }, - "REDAXO": { - "cats": [ - 1 - ], - "description": "REDAXO is a content management system that provides business optimisation through web projects and output codes.", - "icon": "REDAXO.png", - "implies": "PHP", - "js": { - "redaxo": "\\;confidence:50" - }, - "url": "^https?://(?:www\\.)?[\\d\\w\\-]+\\.[\\w]+\/redaxo/\\;confidence:50", - "oss": true, - "pricing": [ - "freemium" - ], - "website": "https://redaxo.org" - }, "Red Hat": { "cats": [ 28 @@ -1386,11 +1386,11 @@ "js": { "Rotic.setting": "" }, - "saas": true, "pricing": [ "low", "recurring" ], + "saas": true, "website": "https://rotic.ir" }, "RoundCube": { diff --git a/src/technologies/v.json b/src/technologies/v.json index 04edbf165..3b3567c43 100644 --- a/src/technologies/v.json +++ b/src/technologies/v.json @@ -1,4 +1,21 @@ { + "VAPTCHA": { + "cats": [ + 16 + ], + "description": "VAPTCHA is the abbreviation of (Variation Analysis based Public Turing Test to Tell Computers and Humans Apart), also known as gesture verification code, is a human-machine verification solution based on artificial intelligence and big data.", + "icon": "VAPTCHA.svg", + "js": { + "vaptcha": "" + }, + "pricing": [ + "poa", + "freemium" + ], + "saas": true, + "scriptSrc": "\\.vaptcha\\.com/v([\\d\\.]+)\\.js\\;version:\\1", + "website": "https://en.vaptcha.com" + }, "VDX.tv": { "cats": [ 36 @@ -159,23 +176,6 @@ "implies": "PHP", "website": "http://vanillaforums.org" }, - "VAPTCHA": { - "cats": [ - 16 - ], - "description": "VAPTCHA is the abbreviation of (Variation Analysis based Public Turing Test to Tell Computers and Humans Apart), also known as gesture verification code, is a human-machine verification solution based on artificial intelligence and big data.", - "icon": "VAPTCHA.svg", - "js": { - "vaptcha": "" - }, - "scriptSrc": "\\.vaptcha\\.com/v([\\d\\.]+)\\.js\\;version:\\1", - "saas": true, - "pricing": [ - "poa", - "freemium" - ], - "website": "https://en.vaptcha.com" - }, "Varbase": { "cats": [ 1 diff --git a/src/wappalyzer.js b/src/wappalyzer.js index 65bbae6c0..22db3e680 100644 --- a/src/wappalyzer.js +++ b/src/wappalyzer.js @@ -557,7 +557,7 @@ const Wappalyzer = { if (pattern.regex.test(value)) { technologies.push({ technology, - pattern, + pattern: { ...pattern, type }, version: Wappalyzer.resolveVersion(pattern, value), }) } @@ -584,7 +584,7 @@ const Wappalyzer = { if (pattern.regex.test(value)) { technologies.push({ technology, - pattern, + pattern: { ...pattern, type }, version: Wappalyzer.resolveVersion(pattern, value), }) } @@ -621,7 +621,7 @@ const Wappalyzer = { if (pattern.regex.test(value)) { technologies.push({ technology, - pattern, + pattern: { ...pattern, type }, version: Wappalyzer.resolveVersion(pattern, value), }) }