diff --git a/README.md b/README.md
index 0c0b6fb43..7694a4e59 100644
--- a/README.md
+++ b/README.md
@@ -95,7 +95,8 @@ Patterns (regular expressions) are kept in [`src/technologies.json`](https://git
"generator": "(?:Example|Another Example)"
},
"script": "example-([0-9.]+)\\.js\\;confidence:50\\;version:\\1",
- "url": ".+\\.example\\.com",
+ "url": "example\\.com",
+ "xhr": "example\\.com",
"oss": true,
"saas": true,
"pricing": ["medium", "freemium", "recurring"],
@@ -354,6 +355,12 @@ Plus any of:
| Full URL of the page. |
"^https?//.+\\.wordpress\\.com" |
+
+ xhr |
+ String |
+ Hostnames of XHR requests. |
+ "cdn\\.netlify\\.com" |
+
meta |
Object |
diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js
index c94c2c626..9404e727d 100644
--- a/src/drivers/npm/driver.js
+++ b/src/drivers/npm/driver.js
@@ -51,6 +51,8 @@ const { technologies, categories } = JSON.parse(
setTechnologies(technologies)
setCategories(categories)
+const xhrDebounce = []
+
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms))
}
@@ -328,6 +330,26 @@ class Site {
page.on('request', async (request) => {
try {
+ if (request.resourceType() === 'xhr') {
+ let hostname
+
+ try {
+ ;({ hostname } = new URL(request.url()))
+ } catch (error) {
+ return
+ }
+
+ if (!xhrDebounce.includes(hostname)) {
+ xhrDebounce.push(hostname)
+
+ setTimeout(() => {
+ xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1)
+
+ this.onDetect(analyze({ xhr: hostname }))
+ }, 1000)
+ }
+ }
+
if (
(responseReceived && request.isNavigationRequest()) ||
request.frame() !== page.mainFrame() ||
diff --git a/src/drivers/webextension/images/icons/commercelayer.svg b/src/drivers/webextension/images/icons/commercelayer.svg
new file mode 100644
index 000000000..85fb6ccc5
--- /dev/null
+++ b/src/drivers/webextension/images/icons/commercelayer.svg
@@ -0,0 +1,10 @@
+
+
+
\ No newline at end of file
diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js
index ba01cb039..8074f8912 100644
--- a/src/drivers/webextension/js/driver.js
+++ b/src/drivers/webextension/js/driver.js
@@ -16,6 +16,8 @@ const expiry = 1000 * 60 * 60 * 24
const hostnameIgnoreList = /((local|dev(elop(ment)?)?|stag(e|ing)?|preprod|preview|test(ing)?|demo(shop)?|admin|cache)[.-]|localhost|wappalyzer|google|facebook|twitter|reddit|yahoo|wikipedia|amazon|youtube|\/admin|\.local|\.test|\.dev|^[0-9.]$)/
+const xhrDebounce = []
+
const Driver = {
lastPing: Date.now(),
@@ -66,6 +68,11 @@ const Driver = {
['responseHeaders']
)
+ chrome.webRequest.onCompleted.addListener(Driver.onXhrRequestComplete, {
+ urls: ['http://*/*', 'https://*/*'],
+ types: ['xmlhttprequest'],
+ })
+
chrome.tabs.onRemoved.addListener((id) => (Driver.cache.tabs[id] = null))
// Enable messaging between scripts
@@ -80,10 +87,10 @@ const Driver = {
'https://www.wappalyzer.com/installed/?utm_source=installed&utm_medium=extension&utm_campaign=wappalyzer'
)
} else if (version !== previous && upgradeMessage) {
- // open(
- // `https://www.wappalyzer.com/upgraded/?utm_source=upgraded&utm_medium=extension&utm_campaign=wappalyzer`,
- // false
- // )
+ open(
+ `https://www.wappalyzer.com/upgraded/?utm_source=upgraded&utm_medium=extension&utm_campaign=wappalyzer`,
+ false
+ )
}
await setOption('version', version)
@@ -256,11 +263,11 @@ const Driver = {
* @param {Object} request
*/
async onWebRequestComplete(request) {
- if (await Driver.isDisabledDomain(request.url)) {
- return
- }
-
if (request.responseHeaders) {
+ if (await Driver.isDisabledDomain(request.url)) {
+ return
+ }
+
const headers = {}
try {
@@ -281,7 +288,7 @@ const Driver = {
)
})
- await Driver.onDetect(request.url, analyze({ headers }))
+ Driver.onDetect(request.url, analyze({ headers })).catch(Driver.error)
}
} catch (error) {
Driver.error(error)
@@ -289,6 +296,36 @@ const Driver = {
}
},
+ /**
+ * Analyse XHR request hostnames
+ * @param {Object} request
+ */
+ async onXhrRequestComplete(request) {
+ if (await Driver.isDisabledDomain(request.url)) {
+ return
+ }
+
+ let hostname
+
+ try {
+ ;({ hostname } = new URL(request.url))
+ } catch (error) {
+ return
+ }
+
+ if (!xhrDebounce.includes(hostname)) {
+ xhrDebounce.push(hostname)
+
+ setTimeout(() => {
+ xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1)
+
+ Driver.onDetect(request.originUrl, analyze({ xhr: hostname })).catch(
+ Driver.error
+ )
+ }, 1000)
+ }
+ },
+
/**
* Process return values from content.js
* @param {String} url
diff --git a/src/technologies.json b/src/technologies.json
index 55120703d..ebd7cf438 100644
--- a/src/technologies.json
+++ b/src/technologies.json
@@ -4216,6 +4216,10 @@
"description": "Contentful is an API-first content management platform to create, manage and publish content on any digital channel.",
"html": "<[^>]+(?:https?:)?//(?:assets|downloads|images|videos)\\.(?:ct?fassets\\.net|contentful\\.com)",
"icon": "Contentful.svg",
+ "headers": {
+ "x-contentful-request-id": ""
+ },
+ "xhr": "cdn\\.contentful\\.com",
"pricing": [
"mid",
"freemium",
@@ -15483,6 +15487,19 @@
},
"website": "https://shopfa.com"
},
+ "commercelayer": {
+ "cats": [
+ 6
+ ],
+ "pricing": [
+ "medium",
+ "recurring"
+ ],
+ "saas": true,
+ "xhr": "\\.commercelayer\\.io",
+ "icon": "commercelayer.svg",
+ "website": "https://commercelayer.io"
+ },
"Shopify": {
"cats": [
6
@@ -21178,4 +21195,4 @@
"website": "https://www.xt-commerce.com"
}
}
-}
\ No newline at end of file
+}
diff --git a/src/wappalyzer.js b/src/wappalyzer.js
index ac55b1c1e..c4da0fea4 100644
--- a/src/wappalyzer.js
+++ b/src/wappalyzer.js
@@ -189,6 +189,7 @@ const Wappalyzer = {
*/
analyze({
url,
+ xhr,
html,
css,
robots,
@@ -210,6 +211,7 @@ const Wappalyzer = {
Wappalyzer.technologies.map((technology) =>
flatten([
oo(technology, 'url', url),
+ oo(technology, 'xhr', xhr),
oo(technology, 'html', html),
oo(technology, 'css', css),
oo(technology, 'robots', robots),
@@ -240,6 +242,7 @@ const Wappalyzer = {
const {
cats,
url,
+ xhr,
dom,
html,
css,
@@ -263,6 +266,7 @@ const Wappalyzer = {
categories: cats || [],
slug: Wappalyzer.slugify(name),
url: transform(url),
+ xhr: transform(xhr),
headers: transform(headers),
dns: transform(dns),
cookies: transform(cookies),