Migrations
/
wappalyzer
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add XHR request hostname inspection, add commercelayer detection

Browse Source
main
Elbert Alias 4 years ago
parent e8794aa791
commit c2561be751
6 changed files with 108 additions and 11 deletions
Show Stats Download Patch File Download Diff File

9
README.md
Unescape View File

@ -95,7 +95,8 @@ Patterns (regular expressions) are kept in [`src/technologies.json`](https://git
"generator": "(?:Example|Another Example)" "generator": "(?:Example|Another Example)"
}, },
"script": "example-([0-9.]+)\\.js\\;confidence:50\\;version:\\1", "script": "example-([0-9.]+)\\.js\\;confidence:50\\;version:\\1",
"url": ".+\\.example\\.com", "url": "example\\.com",
"xhr": "example\\.com",
"oss": true, "oss": true,
"saas": true, "saas": true,
"pricing": ["medium", "freemium", "recurring"], "pricing": ["medium", "freemium", "recurring"],
@ -354,6 +355,12 @@ Plus any of:
<td>Full URL of the page.</td> <td>Full URL of the page.</td>
<td><code>"^https?//.+\\.wordpress\\.com"</code></td> <td><code>"^https?//.+\\.wordpress\\.com"</code></td>
</tr> </tr>
<tr>
<td><code>xhr</code></td>
<td>String</td>
<td>Hostnames of XHR requests.</td>
<td><code>"cdn\\.netlify\\.com"</code></td>
</tr>
<tr> <tr>
<td><code>meta</code></td> <td><code>meta</code></td>
<td>Object</td> <td>Object</td>

22
src/drivers/npm/driver.js
Unescape View File

@ -51,6 +51,8 @@ const { technologies, categories } = JSON.parse(
setTechnologies(technologies) setTechnologies(technologies)
setCategories(categories) setCategories(categories)
const xhrDebounce = []
function sleep(ms) { function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms)) return new Promise((resolve) => setTimeout(resolve, ms))
} }
@ -328,6 +330,26 @@ class Site {
page.on('request', async (request) => { page.on('request', async (request) => {
try { try {
if (request.resourceType() === 'xhr') {
let hostname
try {
;({ hostname } = new URL(request.url()))
} catch (error) {
return
}
if (!xhrDebounce.includes(hostname)) {
xhrDebounce.push(hostname)
setTimeout(() => {
xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1)
this.onDetect(analyze({ xhr: hostname }))
}, 1000)
}
}
if ( if (
(responseReceived && request.isNavigationRequest()) || (responseReceived && request.isNavigationRequest()) ||
request.frame() !== page.mainFrame() || request.frame() !== page.mainFrame() ||

10
src/drivers/webextension/images/icons/commercelayer.svg
Unescape View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Generator: Adobe Illustrator 23.0.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="commercelayer-glyph-2019" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 260 260" style="enable-background:new 0 0 260 260;" xml:space="preserve">
<g id="glyph-black">
<path id="bottom-bracket" d="M228.5,157.6V195c0,2.6-1.1,5.2-3.1,6.9c-2,1.8-4.7,2.9-7.4,2.8h-57.6c-4.8-0.1-9.6,0.7-14.1,2.3 c-3.3,1.2-6.4,3.1-9,5.4c-2.9,2.5-5.4,5.6-7.2,9c-1.8-3.4-4.2-6.4-7.1-8.9c-2.7-2.3-5.7-4.1-9-5.4c-4.5-1.6-9.3-2.4-14.2-2.2H42.1 c-2.7,0-5.4-1-7.4-2.9c-2-1.8-3.1-4.3-3.1-7v-37.4H0V195c-0.1,5.7,1,11.4,3.1,16.7c1.9,5,4.9,9.6,8.8,13.3 c3.8,3.7,8.4,6.6,13.3,8.5c5.3,2,10.9,3,16.6,2.8h57.6c5.2,0,10.8,0.2,12.1,2.6c2,3.7,2.9,9.4,2.9,17.1v3.9h31.5v-3.9 c0-7.7,1-13.5,2.9-17.1c1.2-2.4,6.9-2.6,12.1-2.6h57.6c5.6,0.1,11.2-0.8,16.5-2.8c4.9-2,9.5-4.9,13.3-8.5c3.9-3.7,6.9-8.3,8.8-13.3 c2.1-5.3,3.1-11,3.1-16.7v-37.4H228.5z"/>
<rect id="right-eye" x="149.7" y="102.4" width="55.2" height="55.2"/>
<rect id="left-eye" x="55.2" y="102.4" width="55.2" height="55.2"/>
<path id="top-bracket" d="M256.9,48.3L256.9,48.3c-1.9-5-4.9-9.5-8.8-13.3c-3.8-3.7-8.4-6.6-13.3-8.5c-5.3-2-10.9-3-16.6-2.8h-57.6 c-5.2,0-10.8-0.2-12.1-2.6c-1.9-3.7-2.9-9.4-2.9-17.1V0h-31.5v3.9c0,7.7-1,13.5-2.9,17.1c-1.2,2.4-6.9,2.6-12,2.6H41.7 c-5.6-0.1-11.2,0.8-16.5,2.8c-5,2-9.5,4.8-13.4,8.5C8,38.7,5,43.3,3.1,48.3C1,53.6-0.1,59.3,0,65v37.4h31.5V65 c0-2.6,1.1-5.2,3.1-6.9c2-1.9,4.7-2.9,7.4-3h57.6c4.8,0.1,9.6-0.6,14.1-2.2c3.3-1.2,6.4-3.1,9-5.4c2.9-2.5,5.3-5.5,7.1-8.9 c1.8,3.4,4.2,6.4,7.1,8.9c2.7,2.3,5.7,4.1,9,5.4c4.5,1.6,9.3,2.4,14.1,2.2h57.6c2.7,0,5.4,1,7.4,2.9c2,1.8,3.2,4.3,3.2,7v37.4H260 V65C260.1,59.3,259,53.6,256.9,48.3z"/>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 1.8 KiB

55
src/drivers/webextension/js/driver.js
Unescape View File

@ -16,6 +16,8 @@ const expiry = 1000 * 60 * 60 * 24
const hostnameIgnoreList = /((local|dev(elop(ment)?)?|stag(e|ing)?|preprod|preview|test(ing)?|demo(shop)?|admin|cache)[.-]|localhost|wappalyzer|google|facebook|twitter|reddit|yahoo|wikipedia|amazon|youtube|\/admin|\.local|\.test|\.dev|^[0-9.]$)/ const hostnameIgnoreList = /((local|dev(elop(ment)?)?|stag(e|ing)?|preprod|preview|test(ing)?|demo(shop)?|admin|cache)[.-]|localhost|wappalyzer|google|facebook|twitter|reddit|yahoo|wikipedia|amazon|youtube|\/admin|\.local|\.test|\.dev|^[0-9.]$)/
const xhrDebounce = []
const Driver = { const Driver = {
lastPing: Date.now(), lastPing: Date.now(),
@ -66,6 +68,11 @@ const Driver = {
['responseHeaders'] ['responseHeaders']
) )
chrome.webRequest.onCompleted.addListener(Driver.onXhrRequestComplete, {
urls: ['http://*/*', 'https://*/*'],
types: ['xmlhttprequest'],
})
chrome.tabs.onRemoved.addListener((id) => (Driver.cache.tabs[id] = null)) chrome.tabs.onRemoved.addListener((id) => (Driver.cache.tabs[id] = null))
// Enable messaging between scripts // Enable messaging between scripts
@ -80,10 +87,10 @@ const Driver = {
'https://www.wappalyzer.com/installed/?utm_source=installed&utm_medium=extension&utm_campaign=wappalyzer' 'https://www.wappalyzer.com/installed/?utm_source=installed&utm_medium=extension&utm_campaign=wappalyzer'
) )
} else if (version !== previous && upgradeMessage) { } else if (version !== previous && upgradeMessage) {
// open( open(
// `https://www.wappalyzer.com/upgraded/?utm_source=upgraded&utm_medium=extension&utm_campaign=wappalyzer`, `https://www.wappalyzer.com/upgraded/?utm_source=upgraded&utm_medium=extension&utm_campaign=wappalyzer`,
// false false
// ) )
} }
await setOption('version', version) await setOption('version', version)
@ -256,11 +263,11 @@ const Driver = {
* @param {Object} request * @param {Object} request
*/ */
async onWebRequestComplete(request) { async onWebRequestComplete(request) {
if (await Driver.isDisabledDomain(request.url)) {
return
}
if (request.responseHeaders) { if (request.responseHeaders) {
if (await Driver.isDisabledDomain(request.url)) {
return
}
const headers = {} const headers = {}
try { try {
@ -281,7 +288,7 @@ const Driver = {
) )
}) })
await Driver.onDetect(request.url, analyze({ headers })) Driver.onDetect(request.url, analyze({ headers })).catch(Driver.error)
} }
} catch (error) { } catch (error) {
Driver.error(error) Driver.error(error)
@ -289,6 +296,36 @@ const Driver = {
} }
}, },
/**
* Analyse XHR request hostnames
* @param {Object} request
*/
async onXhrRequestComplete(request) {
if (await Driver.isDisabledDomain(request.url)) {
return
}
let hostname
try {
;({ hostname } = new URL(request.url))
} catch (error) {
return
}
if (!xhrDebounce.includes(hostname)) {
xhrDebounce.push(hostname)
setTimeout(() => {
xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1)
Driver.onDetect(request.originUrl, analyze({ xhr: hostname })).catch(
Driver.error
)
}, 1000)
}
},
/** /**
* Process return values from content.js * Process return values from content.js
* @param {String} url * @param {String} url

17
src/technologies.json
Unescape View File

@ -4216,6 +4216,10 @@
"description": "Contentful is an API-first content management platform to create, manage and publish content on any digital channel.", "description": "Contentful is an API-first content management platform to create, manage and publish content on any digital channel.",
"html": "<[^>]+(?:https?:)?//(?:assets|downloads|images|videos)\\.(?:ct?fassets\\.net|contentful\\.com)", "html": "<[^>]+(?:https?:)?//(?:assets|downloads|images|videos)\\.(?:ct?fassets\\.net|contentful\\.com)",
"icon": "Contentful.svg", "icon": "Contentful.svg",
"headers": {
"x-contentful-request-id": ""
},
"xhr": "cdn\\.contentful\\.com",
"pricing": [ "pricing": [
"mid", "mid",
"freemium", "freemium",
@ -15483,6 +15487,19 @@
}, },
"website": "https://shopfa.com" "website": "https://shopfa.com"
}, },
"commercelayer": {
"cats": [
6
],
"pricing": [
"medium",
"recurring"
],
"saas": true,
"xhr": "\\.commercelayer\\.io",
"icon": "commercelayer.svg",
"website": "https://commercelayer.io"
},
"Shopify": { "Shopify": {
"cats": [ "cats": [
6 6

4
src/wappalyzer.js
Unescape View File

@ -189,6 +189,7 @@ const Wappalyzer = {
*/ */
analyze({ analyze({
url, url,
xhr,
html, html,
css, css,
robots, robots,
@ -210,6 +211,7 @@ const Wappalyzer = {
Wappalyzer.technologies.map((technology) => Wappalyzer.technologies.map((technology) =>
flatten([ flatten([
oo(technology, 'url', url), oo(technology, 'url', url),
oo(technology, 'xhr', xhr),
oo(technology, 'html', html), oo(technology, 'html', html),
oo(technology, 'css', css), oo(technology, 'css', css),
oo(technology, 'robots', robots), oo(technology, 'robots', robots),
@ -240,6 +242,7 @@ const Wappalyzer = {
const { const {
cats, cats,
url, url,
xhr,
dom, dom,
html, html,
css, css,
@ -263,6 +266,7 @@ const Wappalyzer = {
categories: cats || [], categories: cats || [],
slug: Wappalyzer.slugify(name), slug: Wappalyzer.slugify(name),
url: transform(url), url: transform(url),
xhr: transform(xhr),
headers: transform(headers), headers: transform(headers),
dns: transform(dns), dns: transform(dns),
cookies: transform(cookies), cookies: transform(cookies),