From 94c275b195591af7114c0ff1558613b27131d57a Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Fri, 31 Jul 2020 13:40:10 +1000 Subject: [PATCH] Add option for custom request headers in NPM --- src/apps.json | 264 +++++++++++++++++++------------------- src/drivers/npm/README.md | 6 +- src/drivers/npm/driver.js | 18 ++- 3 files changed, 151 insertions(+), 137 deletions(-) diff --git a/src/apps.json b/src/apps.json index 8ea57d7ed..ea49872d0 100644 --- a/src/apps.json +++ b/src/apps.json @@ -86,26 +86,6 @@ ], "website": "https://www.adplan7.com/" }, - "Afterpay": { - "cats": [ - 41 - ], - "cpe": "cpe:/a:afterpay:afterpay", - "icon": "afterpay.png", - "script": [ - "portal\\.afterpay\\.com", - "static\\.afterpay\\.com" - ], - "website": "https://www.afterpay.com/" - }, - "Amex Checkout": { - "cats": [ - 41 - ], - "icon": "amex.png", - "script": "aexp-static\\.com", - "website": "https://www.americanexpress.com/us/express-checkout/" - }, "AMP": { "cats": [ 12 @@ -512,6 +492,18 @@ "script": "shop-static\\.afterbuy\\.de", "website": "http://www.afterbuy.de" }, + "Afterpay": { + "cats": [ + 41 + ], + "cpe": "cpe:/a:afterpay:afterpay", + "icon": "afterpay.png", + "script": [ + "portal\\.afterpay\\.com", + "static\\.afterpay\\.com" + ], + "website": "https://www.afterpay.com/" + }, "Ahoy": { "cats": [ 10 @@ -794,6 +786,14 @@ "script": "ametys\\.js", "website": "http://ametys.org" }, + "Amex Checkout": { + "cats": [ + 41 + ], + "icon": "amex.png", + "script": "aexp-static\\.com", + "website": "https://www.americanexpress.com/us/express-checkout/" + }, "Amiro.CMS": { "cats": [ 1 @@ -1053,8 +1053,8 @@ "]*appleid-signin-client-id", "]*appleid\\.apple\\.com/auth/authorize" ], - "script": "appleid.auth.js", "icon": "Apple.svg", + "script": "appleid.auth.js", "website": "https://developer.apple.com/sign-in-with-apple/" }, "Arastta": { @@ -3764,15 +3764,6 @@ ], "website": "https://etherpad.org" }, - "eWAY Payments": { - "cats": [ - 41 - ], - "html": "]*src=\"[^/]*//[^/]*eway\\.com", - "icon": "eway.png", - "script": "secure\\.ewaypayments\\.com", - "website": "https://www.eway.com.au/" - }, "Exhibit": { "cats": [ 25 @@ -3886,10 +3877,10 @@ "cats": [ 69 ], + "icon": "Facebook.svg", "js": { - "FB.getLoginStatus":"" + "FB.getLoginStatus": "" }, - "icon": "Facebook.svg", "website": "https://developers.facebook.com/docs/facebook-login/" }, "Facil-iti": { @@ -3965,17 +3956,6 @@ "icon": "fastspring.png", "website": "https://fastspring.com" }, - "Fat-Free Framework": { - "cats": [ - 18 - ], - "headers": { - "X-Powered-By": "^Fat-Free Framework$" - }, - "icon": "Fat-Free Framework.png", - "implies": "PHP", - "website": "http://fatfreeframework.com" - }, "Fat Zebra": { "cats": [ 41 @@ -3988,6 +3968,17 @@ "script": "paynow\\.pmnts\\.io", "website": "https://www.fatzebra.com/" }, + "Fat-Free Framework": { + "cats": [ + 18 + ], + "headers": { + "X-Powered-By": "^Fat-Free Framework$" + }, + "icon": "Fat-Free Framework.png", + "implies": "PHP", + "website": "http://fatfreeframework.com" + }, "Fbits": { "cats": [ 6 @@ -4848,23 +4839,6 @@ "script": "googleapis\\.com/.+webfont", "website": "http://google.com/fonts" }, - "Google Sign-in": { - "cats": [ - 69 - ], - "html": [ - "]*google-signin-client_id", - "]*google-signin-scope", - "]*accounts\\.google\\.com/o/oauth2", - "]*accounts\\.google\\.com/o/oauth2" - ], - "script": [ - "apis\\.google\\.com/js/platform\\.js", - "accounts\\.google\\.com/gsi/client" - ], - "icon": "Google.svg", - "website": "https://developers.google.com/identity/sign-in/web" - }, "Google Maps": { "cats": [ 35 @@ -4904,6 +4878,23 @@ "script": "apis\\.google\\.com/js/[a-z]*\\.js", "website": "http://plus.google.com" }, + "Google Sign-in": { + "cats": [ + 69 + ], + "html": [ + "]*google-signin-client_id", + "]*google-signin-scope", + "]*accounts\\.google\\.com/o/oauth2", + "]*accounts\\.google\\.com/o/oauth2" + ], + "icon": "Google.svg", + "script": [ + "apis\\.google\\.com/js/platform\\.js", + "accounts\\.google\\.com/gsi/client" + ], + "website": "https://developers.google.com/identity/sign-in/web" + }, "Google Sites": { "cats": [ 1 @@ -5071,6 +5062,20 @@ "script": "assets\\.growingio\\.com/([\\d.]+)/gio\\.js\\;version:\\1", "website": "https://www.growingio.com/" }, + "HCL Digital Experience": { + "cats": [ + 1 + ], + "cpe": "cpe:/a:ibm:websphere_portal", + "headers": { + "IBM-Web2-Location": "", + "Itx-Generated-Timestamp": "" + }, + "icon": "IBM.svg", + "implies": "Java", + "url": "/wps/", + "website": "https://www.hcltechsw.com/products/dx" + }, "HERE": { "cats": [ 35 @@ -5428,20 +5433,6 @@ "url": "/wcs/", "website": "http://ibm.com/software/genservers/commerceproductline" }, - "HCL Digital Experience": { - "cats": [ - 1 - ], - "cpe": "cpe:/a:ibm:websphere_portal", - "headers": { - "IBM-Web2-Location": "", - "Itx-Generated-Timestamp": "" - }, - "icon": "IBM.svg", - "implies": "Java", - "url": "/wps/", - "website": "https://www.hcltechsw.com/products/dx" - }, "IIS": { "cats": [ 22 @@ -7870,6 +7861,17 @@ "script": "mustache(?:\\.min)?\\.js", "website": "https://mustache.github.io" }, + "My Food Link": { + "cats": [ + 6 + ], + "html": [ + "
", + "", - "]*src=\"[^/]*//[^/]*eway\\.com", + "icon": "zip_pay.png", + "script": [ + "static\\.zipmoney\\.com\\.au", + "zip\\.co" + ], + "website": "https://www.zip.co/" + }, "Zipkin": { "cats": [ 10 @@ -14037,6 +14040,15 @@ }, "website": "http://esyndicat.com" }, + "eWAY Payments": { + "cats": [ + 41 + ], + "html": "]*src=\"[^/]*//[^/]*eway\\.com", + "icon": "eway.png", + "script": "secure\\.ewaypayments\\.com", + "website": "https://www.eway.com.au/" + }, "eZ Platform": { "cats": [ 1, @@ -15190,18 +15202,6 @@ "generator": "xt:Commerce" }, "website": "https://www.xt-commerce.com" - }, - "Zip Pay / Zip Money": { - "cats": [ - 41 - ], - "html": "]*src=\"[^/]*//[^/]*eway\\.com", - "icon": "zip_pay.png", - "script": [ - "static\\.zipmoney\\.com\\.au", - "zip\\.co" - ], - "website": "https://www.zip.co/" } }, "categories": { @@ -15474,4 +15474,4 @@ "priority": 6 } } -} +} \ No newline at end of file diff --git a/src/drivers/npm/README.md b/src/drivers/npm/README.md index 3d287495a..a2d83e0b3 100644 --- a/src/drivers/npm/README.md +++ b/src/drivers/npm/README.md @@ -54,6 +54,7 @@ const url = 'https://www.wappalyzer.com'; const options = { debug: false, delay: 500, + headers: {}, maxDepth: 3, maxUrls: 10, maxWait: 5000, @@ -69,7 +70,10 @@ const options = { try { await wappalyzer.init() - const site = await wappalyzer.open(url) + // Optionally set additional request headers + const headers = {} + + const site = await wappalyzer.open(url, headers) // Optionally capture and output errors site.on('error', console.error) diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 853c8ba41..93c04a85d 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -130,8 +130,8 @@ class Driver { } } - open(url) { - return new Site(url.split('#')[0], this) + open(url, headers = {}) { + return new Site(url.split('#')[0], headers, this) } log(message, source = 'driver') { @@ -143,9 +143,14 @@ class Driver { } class Site { - constructor(url, driver) { + constructor(url, headers = {}, driver) { ;({ options: this.options, browser: this.browser } = driver) + this.options.headers = { + ...this.options.headers, + ...headers + } + this.driver = driver try { @@ -232,7 +237,12 @@ class Site { ) { request.abort('blockedbyclient') } else { - request.continue() + const headers = { + ...request.headers(), + ...this.options.headers + } + + request.continue({ headers }) } } catch (error) { this.error(error)