From fe4092b74bdc1a4fab6d45ce79f16e49c6bde70d Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Fri, 15 Oct 2021 11:47:17 +1100 Subject: [PATCH] Inspect JavaScript files. Renamed `scripts` fields to `srciptSrc`, repurposed `scripts` field for in-file matches. --- README.md | 6 +- src/README.md | 2 +- src/drivers/npm/README.md | 2 +- src/drivers/npm/driver.js | 44 +++- src/drivers/webextension/js/content.js | 13 +- src/drivers/webextension/js/driver.js | 31 +++ src/technologies/_.json | 8 +- src/technologies/a.json | 220 +++++++++---------- src/technologies/b.json | 96 ++++----- src/technologies/c.json | 182 ++++++++-------- src/technologies/d.json | 72 +++---- src/technologies/e.json | 64 +++--- src/technologies/f.json | 86 ++++---- src/technologies/g.json | 89 ++++---- src/technologies/h.json | 66 +++--- src/technologies/i.json | 56 ++--- src/technologies/j.json | 48 ++--- src/technologies/k.json | 48 ++--- src/technologies/l.json | 74 +++---- src/technologies/m.json | 128 +++++------ src/technologies/n.json | 30 +-- src/technologies/o.json | 78 +++---- src/technologies/p.json | 148 ++++++------- src/technologies/q.json | 16 +- src/technologies/r.json | 114 +++++----- src/technologies/s.json | 284 ++++++++++++------------- src/technologies/t.json | 128 +++++------ src/technologies/u.json | 40 ++-- src/technologies/v.json | 44 ++-- src/technologies/w.json | 58 ++--- src/technologies/x.json | 16 +- src/technologies/y.json | 26 +-- src/technologies/z.json | 32 +-- src/wappalyzer.js | 12 +- 34 files changed, 1213 insertions(+), 1148 deletions(-) diff --git a/README.md b/README.md index 47ba6295a..3ef34b85a 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ Patterns (regular expressions) are kept in [`src/technologies/`](https://github. "meta": { "generator": "(?:Example|Another Example)" }, - "script": "example-([0-9.]+)\\.js\\;confidence:50\\;version:\\1", + "scriptSrc": "example-([0-9.]+)\\.js\\;confidence:50\\;version:\\1", "url": "example\\.com", "xhr": "example\\.com", "oss": true, @@ -379,7 +379,7 @@ Plus any of: { "generator": "^WordPress$" } - scripts + scriptSrc String | Array URLs of JavaScript files included on the page. @@ -432,7 +432,7 @@ Tags (a non-standard syntax) can be appended to patterns (and implies and exclud syntax. - "scripts": "jquery-([0-9.]+)\.js\\;version:\\1" + "scriptSrc": "jquery-([0-9.]+)\.js\\;version:\\1" diff --git a/src/README.md b/src/README.md index d95e7b381..13b1a2442 100644 --- a/src/README.md +++ b/src/README.md @@ -44,7 +44,7 @@ Wappalyzer.analyze({ url: 'https://example.github.io/', meta: { generator: ['WordPress'] }, headers: { server: ['Nginx'] }, - scripts: ['jquery-3.0.0.js'], + scriptSrc: ['jquery-3.0.0.js'], cookies: { awselb: [''] }, html: '
' }).then((detections) => { diff --git a/src/drivers/npm/README.md b/src/drivers/npm/README.md index f3c476e38..0e2e5e022 100644 --- a/src/drivers/npm/README.md +++ b/src/drivers/npm/README.md @@ -132,5 +132,5 @@ Listen to events with `site.on(eventName, callback)`. Use the `page` parameter t | `error` | `message`, `source` | Error messages | | `request` | `page`, `request` | Emitted at the start of a request | | `response` | `page`, `request` | Emitted upon receiving a server response | -| `goto` | `page`, `url`, `html`, `cookies`, `scripts`, `meta`, `js`, `language` `links` | Emitted after a page has been analysed | +| `goto` | `page`, `url`, `html`, `cookies`, `scriptsSrc`, `scripts`, `meta`, `js`, `language` `links` | Emitted after a page has been analysed | | `analyze` | `urls`, `technologies`, `meta` | Emitted when the site has been analysed | diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 1cd10c7db..2c42a58c4 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -511,7 +511,6 @@ class Site { if (!this.browser) { await this.initDriver() - if (!this.browser) { throw new Error('Browser closed') } @@ -580,6 +579,15 @@ class Site { page.on('response', async (response) => { try { + if ( + response.frame().url() === url.href && + response.request().resourceType() === 'script' + ) { + const scripts = await response.text() + + await this.onDetect(response.url(), await analyze({ scripts })) + } + if (response.url() === url.href) { this.analyzedUrls[url.href] = { status: response.status(), @@ -673,6 +681,7 @@ class Site { let links = [] let css = '' + let scriptSrc = [] let scripts = [] let meta = [] let js = [] @@ -741,20 +750,32 @@ class Site { ) // Script tags - scripts = await this.promiseTimeout( + ;[scripts, scriptSrc] = await this.promiseTimeout( ( await this.promiseTimeout( - page.evaluateHandle(() => - Array.from(document.getElementsByTagName('script')) - .map(({ src }) => src) - .filter((src) => src) - ), + page.evaluateHandle(() => { + const nodes = Array.from( + document.getElementsByTagName('script') + ) + + return [ + nodes + .map( + ({ src }) => + src && !src.startsWith('data:text/javascript;') + ) + .filter((src) => src), + nodes + .map((node) => node.textContent) + .filter((script) => script), + ] + }), { jsonValue: () => [] }, - 'Timeout (scripts)' + 'Timeout (scriptSrc)' ) ).jsonValue(), [], - 'Timeout (scripts)' + 'Timeout (scriptSrc)' ) // Meta tags @@ -798,6 +819,7 @@ class Site { html, cookies, scripts, + scriptSrc, meta, } @@ -813,6 +835,7 @@ class Site { html, css, scripts, + scriptSrc, meta, }), ]) @@ -1099,7 +1122,7 @@ class Site { if (!this.analyzedRequires[url.href].includes(name)) { this.analyzedRequires[url.href].push(name) - const { page, cookies, html, css, scripts, meta } = + const { page, cookies, html, css, scripts, scriptSrc, meta } = this.cache[url.href] const js = await this.promiseTimeout( @@ -1126,6 +1149,7 @@ class Site { html, css, scripts, + scriptSrc, meta, }, technologies diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js index 7b7ab6a23..00974ae6f 100644 --- a/src/drivers/webextension/js/content.js +++ b/src/drivers/webextension/js/content.js @@ -196,10 +196,15 @@ const Content = { css = css.join('\n') // Script tags - const scripts = Array.from(document.scripts) - .filter(({ src }) => src) + const scriptNodes = Array.from(document.scripts) + + const scriptSrc = scriptNodes + .filter(({ src }) => src && !src.startsWith('data:text/javascript;')) .map(({ src }) => src) - .filter((script) => script.indexOf('data:text/javascript;') !== 0) + + const scripts = scriptNodes + .map((node) => node.textContent) + .filter((script) => script) // Meta tags const meta = Array.from(document.querySelectorAll('meta')).reduce( @@ -264,7 +269,7 @@ const Content = { } } - Content.cache = { html, css, scripts, meta, cookies } + Content.cache = { html, css, scriptSrc, scripts, meta, cookies } await Content.driver('onContentLoad', [ url, diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index a06e31bc5..c8d9c043e 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -19,6 +19,8 @@ const hostnameIgnoreList = const xhrDebounce = [] +const scriptsPending = [] + const Driver = { lastPing: Date.now(), @@ -67,6 +69,11 @@ const Driver = { ['responseHeaders'] ) + chrome.webRequest.onCompleted.addListener(Driver.onScriptRequestComplete, { + urls: ['http://*/*', 'https://*/*'], + types: ['script'], + }) + chrome.webRequest.onCompleted.addListener(Driver.onXhrRequestComplete, { urls: ['http://*/*', 'https://*/*'], types: ['xmlhttprequest'], @@ -400,6 +407,30 @@ const Driver = { } }, + /** + * Analyse scripts + * @param {Object} request + */ + async onScriptRequestComplete(request) { + if (await Driver.isDisabledDomain(request.url)) { + return + } + + if (scriptsPending.includes(request.url)) { + scriptsPending.splice(scriptsPending.indexOf(request.url), 1) + } else if (request.statusCode === 200) { + scriptsPending.push(request.url) + + const response = await fetch(request.url) + + const scripts = await response.text() + + Driver.onDetect(request.documentUrl, await analyze({ scripts })).catch( + Driver.error + ) + } + }, + /** * Analyse XHR request hostnames * @param {Object} request diff --git a/src/technologies/_.json b/src/technologies/_.json index 14ae1a1dd..4ba53a908 100644 --- a/src/technologies/_.json +++ b/src/technologies/_.json @@ -22,7 +22,7 @@ "recurring" ], "saas": true, - "scripts": "bitrix(?:\\.info/|/js/main/core)", + "scriptSrc": "bitrix(?:\\.info/|/js/main/core)", "website": "http://www.1c-bitrix.ru" }, "2B Advice": { @@ -35,7 +35,7 @@ "BBCookieControler": "" }, "saas": true, - "scripts": "2badvice-cdn\\.azureedge\\.net", + "scriptSrc": "2badvice-cdn\\.azureedge\\.net", "website": "https://www.2b-advice.com/en/data-privacy-software/cookie-consent-plugin/" }, "33Across": { @@ -61,7 +61,7 @@ "X-Powered-By": "3DCART" }, "icon": "3dCart.png", - "scripts": "(?:twlh(?:track)?\\.asp|3d_upsell\\.js)", + "scriptSrc": "(?:twlh(?:track)?\\.asp|3d_upsell\\.js)", "website": "http://www.3dcart.com" }, "4-Tell": { @@ -82,7 +82,7 @@ "poa" ], "saas": true, - "scripts": "4tellcdn\\.azureedge\\.net", + "scriptSrc": "4tellcdn\\.azureedge\\.net", "website": "https://4-tell.com" }, "@sulu/web": { diff --git a/src/technologies/a.json b/src/technologies/a.json index 7dff16e3c..9896ca78e 100644 --- a/src/technologies/a.json +++ b/src/technologies/a.json @@ -9,7 +9,7 @@ "js": { "AFRAME.version": "^(.+)$\\;version:\\1" }, - "scripts": "/?([\\d.]+)?/aframe(?:\\.min)?\\.js\\;version:\\1", + "scriptSrc": "/?([\\d.]+)?/aframe(?:\\.min)?\\.js\\;version:\\1", "website": "https://aframe.io" }, "A8.net": { @@ -24,7 +24,7 @@ "a8sales": "", "map_A8": "" }, - "scripts": "statics\\.a8\\.net", + "scriptSrc": "statics\\.a8\\.net", "website": "https://www.a8.net" }, "AB Tasty": { @@ -42,7 +42,7 @@ "poa" ], "saas": true, - "scripts": "try\\.abtasty\\.com", + "scriptSrc": "try\\.abtasty\\.com", "website": "https://www.abtasty.com" }, "ABOUT YOU Commerce Suite": { @@ -74,7 +74,7 @@ "payg" ], "saas": true, - "scripts": "\\.ebis\\.ne\\.jp/", + "scriptSrc": "\\.ebis\\.ne\\.jp/", "website": "http://www.ebis.ne.jp" }, "AMP": { @@ -127,7 +127,7 @@ "AOS.refreshHard": "\\;confidence:50" }, "oss": true, - "scripts": [ + "scriptSrc": [ "unpkg\\.com/aos@([\\d\\.]+)/dist/aos\\.js\\;version:\\1", "/typo3conf/ext/udem_vendor/Resources/Public/aos-([\\d\\.]+)\\;version:\\1" ], @@ -152,7 +152,7 @@ "js": { "xt_click": "" }, - "scripts": "xiti\\.com/hit\\.xiti", + "scriptSrc": "xiti\\.com/hit\\.xiti", "website": "http://atinternet.com/en" }, "ATSHOP": { @@ -167,7 +167,7 @@ "recurring" ], "saas": true, - "scripts": "\\.atshop\\.io", + "scriptSrc": "\\.atshop\\.io", "website": "https://atshop.io" }, "AWIN": { @@ -187,7 +187,7 @@ "payg" ], "saas": true, - "scripts": "dwin1\\.com", + "scriptSrc": "dwin1\\.com", "website": "https://www.awin.com" }, "AWS Certificate Manager": { @@ -256,7 +256,7 @@ "recurring" ], "saas": true, - "scripts": "acsbapp?\\.com/.*/acsb\\.js", + "scriptSrc": "acsbapp?\\.com/.*/acsb\\.js", "website": "https://accessibe.com" }, "Accessibility Toolbar Plugin": { @@ -282,7 +282,7 @@ "poa" ], "saas": true, - "scripts": [ + "scriptSrc": [ "/npm/@accessible360/accessible-slick@([\\d\\.]+)/\\;version:\\1", "/accessible360/accessible-slick/slick/slick\\.min\\.js\\?v=([\\d\\.]+)\\;version:\\1" ], @@ -299,7 +299,7 @@ "recurring" ], "saas": true, - "scripts": "accessibly\\.onthemapmarketing\\.com", + "scriptSrc": "accessibly\\.onthemapmarketing\\.com", "website": "https://www.onthemapmarketing.com/accessibly/" }, "Accesso": { @@ -312,7 +312,7 @@ "js": { "accesso": "" }, - "scripts": "/embed/accesso\\.js", + "scriptSrc": "/embed/accesso\\.js", "website": "https://accesso.com/" }, "Ackee": { @@ -338,7 +338,7 @@ "Mautic" ], "saas": true, - "scripts": [ + "scriptSrc": [ "mautic\\.net", "maestro\\.mautic\\.com" ], @@ -354,7 +354,7 @@ "Acquia Cloud Platform" ], "saas": true, - "scripts": [ + "scriptSrc": [ "https?:\\/\\/.+\\.web\\.ahdev\\.cloud" ], "url": "https:?\\/\\/.+\\.web\\.ahdev\\.cloud", @@ -393,7 +393,7 @@ "Acquia Cloud Platform" ], "saas": true, - "scripts": [ + "scriptSrc": [ "sites\\/g\\/files" ], "website": "https://www.acquia.com/products/drupal-cloud/site-factory" @@ -414,7 +414,7 @@ "Acquia Cloud Platform" ], "saas": true, - "scripts": [ + "scriptSrc": [ "content-hub\\.acquia\\.com" ], "url": "https?:\\/\\/.+\\.content-hub\\.acquia\\.com", @@ -436,7 +436,7 @@ "agiloneObject": "" }, "saas": true, - "scripts": [ + "scriptSrc": [ "^https?:\\/\\/.+\\.agilone\\.com", "^https?:\\/\\/scripts\\.agilone\\.com\\/latest\\/a1.js" ], @@ -460,7 +460,7 @@ "_tcaq": "" }, "saas": true, - "scripts": [ + "scriptSrc": [ "lift\\.acquia\\.com" ], "website": "https://www.acquia.com/products/marketing-cloud/personalization", @@ -479,7 +479,7 @@ "Acquia Cloud Platform" ], "saas": true, - "scripts": [ + "scriptSrc": [ "sites/\\w*/files/cohesion" ], "website": "https://www.acquia.com/products/drupal-cloud/site-studio" @@ -498,7 +498,7 @@ "recurring" ], "saas": true, - "scripts": "/cdnr/\\d+/acton/bn/tracker/\\d+", + "scriptSrc": "/cdnr/\\d+/acton/bn/tracker/\\d+", "website": "http://act-on.com" }, "Actito": { @@ -520,7 +520,7 @@ "recurring" ], "saas": true, - "scripts": [ + "scriptSrc": [ "cdn\\.actito\\.be", "\\.advisor\\.smartfocus\\.com" ], @@ -541,7 +541,7 @@ "recurring" ], "saas": true, - "scripts": [ + "scriptSrc": [ "plugins/activecampaign-subscription-forms/site_tracking\\.js", "\\.activehosted\\.com/" ], @@ -562,7 +562,7 @@ "recurring" ], "saas": true, - "scripts": "\\.acuityscheduling\\.com", + "scriptSrc": "\\.acuityscheduling\\.com", "website": "https://acuityscheduling.com" }, "Ad Lightning": { @@ -575,7 +575,7 @@ "poa" ], "saas": true, - "scripts": "\\.adlightning\\.com", + "scriptSrc": "\\.adlightning\\.com", "website": "https://www.adlightning.com" }, "AdInfinity": { @@ -583,7 +583,7 @@ 36 ], "icon": "AdInfinity.png", - "scripts": "adinfinity\\.com\\.au", + "scriptSrc": "adinfinity\\.com\\.au", "website": "http://adinfinity.com.au" }, "AdOcean": { @@ -597,7 +597,7 @@ "ado.placement": "", "ado.slave": "" }, - "scripts": [ + "scriptSrc": [ "adocean\\.pl/files/js/ado\\.js", "adocean\\.pl\\;confidence:80" ], @@ -612,7 +612,7 @@ "js": { "adriver": "" }, - "scripts": "(?:adriver\\.core\\.\\d\\.js|https?://(?:content|ad|masterh\\d)\\.adriver\\.ru/)", + "scriptSrc": "(?:adriver\\.core\\.\\d\\.js|https?://(?:content|ad|masterh\\d)\\.adriver\\.ru/)", "website": "http://adriver.ru" }, "AdRoll": { @@ -631,7 +631,7 @@ "recurring" ], "saas": true, - "scripts": "(?:a|s)\\.adroll\\.com", + "scriptSrc": "(?:a|s)\\.adroll\\.com", "website": "http://adroll.com" }, "AdRoll CMP System": { @@ -668,7 +668,7 @@ "payg" ], "saas": true, - "scripts": "\\.adscale\\.com/", + "scriptSrc": "\\.adscale\\.com/", "website": "https://www.adscale.com" }, "AdThrive": { @@ -682,7 +682,7 @@ "adthriveVideosInjected": "" }, "saas": true, - "scripts": "ads\\.adthrive\\.com", + "scriptSrc": "ads\\.adthrive\\.com", "website": "https://www.adthrive.com" }, "Ada": { @@ -699,7 +699,7 @@ "poa" ], "saas": true, - "scripts": "\\.ada\\.support", + "scriptSrc": "\\.ada\\.support", "website": "https://www.ada.cx" }, "Adabra": { @@ -718,7 +718,7 @@ "recurring" ], "saas": true, - "scripts": "track\\.adabra\\.com", + "scriptSrc": "track\\.adabra\\.com", "website": "https://www.adabra.com", "xhr": "my\\.adabra\\.com" }, @@ -727,7 +727,7 @@ 68 ], "icon": "Adally.png", - "scripts": "cloudfront\\.net/.*/adally\\.js", + "scriptSrc": "cloudfront\\.net/.*/adally\\.js", "website": "https://adally.com/" }, "Adalyser": { @@ -739,7 +739,7 @@ "js": { "adalyserModules": "" }, - "scripts": "c5\\.adalyser\\.com", + "scriptSrc": "c5\\.adalyser\\.com", "website": "https://adalyser.com/" }, "Adcash": { @@ -756,7 +756,7 @@ "ct_siteunder": "", "ct_tag": "" }, - "scripts": "^[^\\/]*//(?:[^\\/]+\\.)?adcash\\.com/(?:script|ad)/", + "scriptSrc": "^[^\\/]*//(?:[^\\/]+\\.)?adcash\\.com/(?:script|ad)/", "url": "^https?://(?:[^\\/]+\\.)?adcash\\.com/script/pop_", "website": "http://adcash.com" }, @@ -771,7 +771,7 @@ "poa" ], "saas": true, - "scripts": "(?:cdn\\.)?shop\\.pe/widget/", + "scriptSrc": "(?:cdn\\.)?shop\\.pe/widget/", "website": "http://www.addshoppers.com" }, "AddThis": { @@ -783,7 +783,7 @@ "js": { "addthis": "" }, - "scripts": "addthis\\.com/js/", + "scriptSrc": "addthis\\.com/js/", "website": "http://www.addthis.com" }, "AddToAny": { @@ -795,7 +795,7 @@ "js": { "a2apage_init": "" }, - "scripts": "addtoany\\.com/menu/page\\.js", + "scriptSrc": "addtoany\\.com/menu/page\\.js", "website": "http://www.addtoany.com" }, "Adminer": { @@ -823,7 +823,7 @@ "pricing": [ "payg" ], - "scripts": [ + "scriptSrc": [ "artfut\\.com/static/(?:tracking|crossdevice)\\.min\\.js", "cdn\\.admitad\\.com" ], @@ -838,7 +838,7 @@ }, "html": "