From 27ce68e9111edb8c2004db1fc6f00f44eeaa181c Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Sat, 29 May 2021 09:29:50 +1000 Subject: [PATCH] Fix probe for recursive crawls --- schema.json | 13 ++ src/drivers/npm/driver.js | 6 +- src/drivers/npm/package.json | 2 +- src/drivers/webextension/manifest.json | 2 +- src/package.json | 2 +- src/technologies.json | 223 ++++++++++++++----------- 6 files changed, 150 insertions(+), 98 deletions(-) diff --git a/schema.json b/schema.json index 92e4b9cff..b9bab602a 100644 --- a/schema.json +++ b/schema.json @@ -165,6 +165,19 @@ } ] }, + "magento": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/definitions/non-empty-non-blank-string" + } + }, + { + "$ref": "#/definitions/non-empty-non-blank-string" + } + ] + }, "certIssuer": { "oneOf": [ { diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 2ecb3efda..fe34e1442 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -265,6 +265,8 @@ class Site { this.dnsChecked = false this.dns = [] + + this.probed = false } log(message, source = 'driver', type = 'log') { @@ -839,8 +841,10 @@ class Site { await this.batch(links.slice(0, this.options.maxUrls), depth + 1) } - if (this.options.probe) { + if (this.options.probe && !this.probed) { await this.probe(url) + + this.probed = true } } catch (error) { this.analyzedUrls[url.href] = { diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index 14e890817..b1cbd5ef7 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -13,7 +13,7 @@ "software" ], "homepage": "https://www.wappalyzer.com/", - "version": "6.7.0", + "version": "6.7.2", "author": "Wappalyzer", "license": "MIT", "repository": { diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json index 46728eee4..cd01b56e9 100644 --- a/src/drivers/webextension/manifest.json +++ b/src/drivers/webextension/manifest.json @@ -4,7 +4,7 @@ "author": "Wappalyzer", "homepage_url": "https://www.wappalyzer.com/", "description": "Identify web technologies", - "version": "6.7.0", + "version": "6.7.2", "default_locale": "en", "manifest_version": 2, "icons": { diff --git a/src/package.json b/src/package.json index ef8eac9f1..93fab96a1 100644 --- a/src/package.json +++ b/src/package.json @@ -13,7 +13,7 @@ "software" ], "homepage": "https://www.wappalyzer.com/", - "version": "6.7.0", + "version": "6.7.2", "author": "Wappalyzer", "license": "MIT", "repository": { diff --git a/src/technologies.json b/src/technologies.json index 8266e719f..deb017292 100644 --- a/src/technologies.json +++ b/src/technologies.json @@ -4561,13 +4561,17 @@ 6 ], "description": "Cococart is an ecommerce platform.", - "icon": "Cococart.png", "dom": [ "meta[property='og:image'][content*='static.cococart.co']", "div[style*='static.cococart.co']" ], + "icon": "Cococart.png", + "pricing": [ + "freemium", + "recurring", + "payg" + ], "saas": true, - "pricing": ["freemium", "recurring", "payg"], "website": "https://www.cococart.co" }, "CoconutSoftware": { @@ -5208,9 +5212,12 @@ "$czater": "", "$czaterMethods": "" }, - "scripts": "\\.czater\\.pl", + "pricing": [ + "low", + "recurring" + ], "saas": true, - "pricing": ["low", "recurring"], + "scripts": "\\.czater\\.pl", "website": "https://www.czater.pl" }, "D3": { @@ -5490,21 +5497,6 @@ "icon": "Detectify.svg", "website": "https://detectify.com/" }, - "Shuttle": { - "cats": [ - 1 - ], - "description": "Shuttle is a website development platform.", - "icon": "Devisto.svg", - "implies":["Laravel", "PHP", "Amazon Web Services"], - "js": { - "Shuttle.FrontApp": "" - }, - "scripts": "shuttle(?:-assets-new|-storage)\\.s3\\.amazonaws\\.com", - "saas": true, - "pricing": ["low", "recurring"], - "website": "https://www.devisto.com" - }, "Didomi": { "cats": [ 67 @@ -5630,9 +5622,13 @@ "js": { "DIVI": "" }, - "scripts": "Divi/js/custom\\.(?:min|unified)\\.js\\?ver=([\\d.]+)\\;version:\\1", + "pricing": [ + "low", + "recurring", + "onetime" + ], "saas": true, - "pricing": ["low", "recurring", "onetime"], + "scripts": "Divi/js/custom\\.(?:min|unified)\\.js\\?ver=([\\d.]+)\\;version:\\1", "website": "https://www.elegantthemes.com/gallery/divi" }, "Django": { @@ -6024,18 +6020,6 @@ ], "website": "http://www.ec-cube.net" }, - "eClass": { - "cats": [ - 1 - ], - "description": "eClass is an online learning platform.", - "icon": "eClass.png", - "js": { - "fe_eclass": "\\;confidence:50", - "fe_eclass_guest": "\\;confidence:50" - }, - "website": "https://www.eclass.com.hk" - }, "EKM": { "cats": [ 6 @@ -9920,6 +9904,27 @@ }, "website": "https://www.kissmetrics.com" }, + "KQS.store": { + "cats": [ + 6 + ], + "description": "KQS.store is an ecommerce software.", + "dom": [ + "a[href*='kqsdesign.pl'][target='_blank']", + "a[href*='kqs.pl'][target='_blank']", + "#kqs-box,kqs-cookie" + ], + "icon": "KQS.store.png", + "js": { + "kqs_box": "\\;confidence:50", + "kqs_off": "\\;confidence:50" + }, + "pricing": [ + "onetime" + ], + "saas": false, + "website": "https://www.kqs.pl" + }, "KaTeX": { "cats": [ 25 @@ -10447,25 +10452,6 @@ ], "website": "https://kount.com/" }, - "KQS.store": { - "cats": [ - 6 - ], - "description": "KQS.store is an ecommerce software.", - "icon": "KQS.store.png", - "dom": [ - "a[href*='kqsdesign.pl'][target='_blank']", - "a[href*='kqs.pl'][target='_blank']", - "#kqs-box,kqs-cookie" - ], - "js": { - "kqs_box": "\\;confidence:50", - "kqs_off": "\\;confidence:50" - }, - "saas": false, - "pricing": ["onetime"], - "website": "https://www.kqs.pl" - }, "Kubernetes Dashboard": { "cats": [ 47 @@ -12710,23 +12696,6 @@ }, "website": "https://www.digital.nsw.gov.au/digital-design-system" }, - "Nukeviet CMS": { - "cats": [ - 1 - ], - "description": "NukeViet CMS is a Vietnamese content management system.", - "icon": "Nukeviet CMS.png", - "meta": { - "generator": "NukeViet v([\\d.]+)\\;version:\\1" - }, - "js": { - "nv_DigitalClock": "\\;confidence:50", - "nv_is_change_act_confirm": "\\;confidence:50" - }, - "oss": true, - "pricing": ["freemium"], - "website": "https://nukeviet.vn/en/" - }, "NTLM": { "cats": [ 16 @@ -12978,6 +12947,25 @@ "scripts": "connect\\.nosto\\.\\w+/", "website": "https://www.nosto.com" }, + "Nukeviet CMS": { + "cats": [ + 1 + ], + "description": "NukeViet CMS is a Vietnamese content management system.", + "icon": "Nukeviet CMS.png", + "js": { + "nv_DigitalClock": "\\;confidence:50", + "nv_is_change_act_confirm": "\\;confidence:50" + }, + "meta": { + "generator": "NukeViet v([\\d.]+)\\;version:\\1" + }, + "oss": true, + "pricing": [ + "freemium" + ], + "website": "https://nukeviet.vn/en/" + }, "Nuvemshop": { "cats": [ 6 @@ -15007,13 +14995,17 @@ }, "Provide Support": { "cats": [ - 52 + 52 ], "description": "Provide Support is a SaaS for customer service that includes live chat, real-time website monitoring, chat statistics.", "icon": "Provide Support.svg", - "scripts": "\\.providesupport\\.com", + "pricing": [ + "low", + "recurring", + "payg" + ], "saas": true, - "pricing": ["low", "recurring", "payg"], + "scripts": "\\.providesupport\\.com", "website": "https://www.providesupport.com" }, "Proximis": { @@ -17465,6 +17457,28 @@ ], "website": "https://www.shopware.com" }, + "Shuttle": { + "cats": [ + 1 + ], + "description": "Shuttle is a website development platform.", + "icon": "Devisto.svg", + "implies": [ + "Laravel", + "PHP", + "Amazon Web Services" + ], + "js": { + "Shuttle.FrontApp": "" + }, + "pricing": [ + "low", + "recurring" + ], + "saas": true, + "scripts": "shuttle(?:-assets-new|-storage)\\.s3\\.amazonaws\\.com", + "website": "https://www.devisto.com" + }, "Sift": { "cats": [ 10 @@ -17493,17 +17507,20 @@ }, "Signifyd": { "cats": [ - 10, - 16 + 10, + 16 ], "description": "Signifyd is a provider of an enterprise-grade fraud technology solution for ecommerce stores.", "icon": "Signifyd.svg", "js": { "SIGNIFYD_GLOBAL": "" }, - "scripts": "\\.signifyd\\.com", + "pricing": [ + "high", + "recurring" + ], "saas": true, - "pricing": ["high", "recurring"], + "scripts": "\\.signifyd\\.com", "website": "https://www.signifyd.com" }, "SilverStripe": { @@ -19186,13 +19203,16 @@ 6 ], "description": "TakeDrop is an ecommerce platform.", - "icon": "TakeDrop.png", "dom": "img[src*='main.takedropstorage.com']", + "icon": "TakeDrop.png", "js": { "webpackJsonptakedrop-react": "" }, + "pricing": [ + "low", + "recurring" + ], "saas": true, - "pricing": ["low", "recurring"], "website": "https://takedrop.pl" }, "Tamago": { @@ -21015,6 +21035,24 @@ "oss": true, "website": "https://webassembly.org/" }, + "WebEngage": { + "cats": [ + 32, + 76 + ], + "description": "WebEngage is a customer data platform and marketing automation suite.", + "icon": "WebEngage.png", + "js": { + "webengage.__v": "([\\d.]+)\\;version:\\1" + }, + "pricing": [ + "mid", + "recurring" + ], + "saas": true, + "scripts": "\\.webengage\\.co(?:m)?/", + "website": "https://webengage.com" + }, "WebGUI": { "cats": [ 1 @@ -22499,6 +22537,18 @@ "scripts": "[^a-z\\d]e107\\.js", "website": "http://e107.org" }, + "eClass": { + "cats": [ + 1 + ], + "description": "eClass is an online learning platform.", + "icon": "eClass.png", + "js": { + "fe_eclass": "\\;confidence:50", + "fe_eclass_guest": "\\;confidence:50" + }, + "website": "https://www.eclass.com.hk" + }, "eSyndiCat": { "cats": [ 1 @@ -23675,21 +23725,6 @@ }, "website": "http://webedition.de/en" }, - "WebEngage": { - "cats": [ - 32, - 76 - ], - "description": "WebEngage is a customer data platform and marketing automation suite.", - "icon": "WebEngage.png", - "js": { - "webengage.__v": "([\\d.]+)\\;version:\\1" - }, - "scripts": "\\.webengage\\.co(?:m)?/", - "saas": true, - "pricing": ["mid", "recurring"], - "website": "https://webengage.com" - }, "webpack": { "cats": [ 19 @@ -23775,4 +23810,4 @@ "website": "https://www.xt-commerce.com" } } -} +} \ No newline at end of file