diff --git a/README.md b/README.md
index 47ba6295a..3ef34b85a 100644
--- a/README.md
+++ b/README.md
@@ -98,7 +98,7 @@ Patterns (regular expressions) are kept in [`src/technologies/`](https://github.
"meta": {
"generator": "(?:Example|Another Example)"
},
- "script": "example-([0-9.]+)\\.js\\;confidence:50\\;version:\\1",
+ "scriptSrc": "example-([0-9.]+)\\.js\\;confidence:50\\;version:\\1",
"url": "example\\.com",
"xhr": "example\\.com",
"oss": true,
@@ -379,7 +379,7 @@ Plus any of:
'
}).then((detections) => {
diff --git a/src/drivers/npm/README.md b/src/drivers/npm/README.md
index f3c476e38..0e2e5e022 100644
--- a/src/drivers/npm/README.md
+++ b/src/drivers/npm/README.md
@@ -132,5 +132,5 @@ Listen to events with `site.on(eventName, callback)`. Use the `page` parameter t
| `error` | `message`, `source` | Error messages |
| `request` | `page`, `request` | Emitted at the start of a request |
| `response` | `page`, `request` | Emitted upon receiving a server response |
-| `goto` | `page`, `url`, `html`, `cookies`, `scripts`, `meta`, `js`, `language` `links` | Emitted after a page has been analysed |
+| `goto` | `page`, `url`, `html`, `cookies`, `scriptsSrc`, `scripts`, `meta`, `js`, `language` `links` | Emitted after a page has been analysed |
| `analyze` | `urls`, `technologies`, `meta` | Emitted when the site has been analysed |
diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js
index 1cd10c7db..2c42a58c4 100644
--- a/src/drivers/npm/driver.js
+++ b/src/drivers/npm/driver.js
@@ -511,7 +511,6 @@ class Site {
if (!this.browser) {
await this.initDriver()
-
if (!this.browser) {
throw new Error('Browser closed')
}
@@ -580,6 +579,15 @@ class Site {
page.on('response', async (response) => {
try {
+ if (
+ response.frame().url() === url.href &&
+ response.request().resourceType() === 'script'
+ ) {
+ const scripts = await response.text()
+
+ await this.onDetect(response.url(), await analyze({ scripts }))
+ }
+
if (response.url() === url.href) {
this.analyzedUrls[url.href] = {
status: response.status(),
@@ -673,6 +681,7 @@ class Site {
let links = []
let css = ''
+ let scriptSrc = []
let scripts = []
let meta = []
let js = []
@@ -741,20 +750,32 @@ class Site {
)
// Script tags
- scripts = await this.promiseTimeout(
+ ;[scripts, scriptSrc] = await this.promiseTimeout(
(
await this.promiseTimeout(
- page.evaluateHandle(() =>
- Array.from(document.getElementsByTagName('script'))
- .map(({ src }) => src)
- .filter((src) => src)
- ),
+ page.evaluateHandle(() => {
+ const nodes = Array.from(
+ document.getElementsByTagName('script')
+ )
+
+ return [
+ nodes
+ .map(
+ ({ src }) =>
+ src && !src.startsWith('data:text/javascript;')
+ )
+ .filter((src) => src),
+ nodes
+ .map((node) => node.textContent)
+ .filter((script) => script),
+ ]
+ }),
{ jsonValue: () => [] },
- 'Timeout (scripts)'
+ 'Timeout (scriptSrc)'
)
).jsonValue(),
[],
- 'Timeout (scripts)'
+ 'Timeout (scriptSrc)'
)
// Meta tags
@@ -798,6 +819,7 @@ class Site {
html,
cookies,
scripts,
+ scriptSrc,
meta,
}
@@ -813,6 +835,7 @@ class Site {
html,
css,
scripts,
+ scriptSrc,
meta,
}),
])
@@ -1099,7 +1122,7 @@ class Site {
if (!this.analyzedRequires[url.href].includes(name)) {
this.analyzedRequires[url.href].push(name)
- const { page, cookies, html, css, scripts, meta } =
+ const { page, cookies, html, css, scripts, scriptSrc, meta } =
this.cache[url.href]
const js = await this.promiseTimeout(
@@ -1126,6 +1149,7 @@ class Site {
html,
css,
scripts,
+ scriptSrc,
meta,
},
technologies
diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js
index 7b7ab6a23..00974ae6f 100644
--- a/src/drivers/webextension/js/content.js
+++ b/src/drivers/webextension/js/content.js
@@ -196,10 +196,15 @@ const Content = {
css = css.join('\n')
// Script tags
- const scripts = Array.from(document.scripts)
- .filter(({ src }) => src)
+ const scriptNodes = Array.from(document.scripts)
+
+ const scriptSrc = scriptNodes
+ .filter(({ src }) => src && !src.startsWith('data:text/javascript;'))
.map(({ src }) => src)
- .filter((script) => script.indexOf('data:text/javascript;') !== 0)
+
+ const scripts = scriptNodes
+ .map((node) => node.textContent)
+ .filter((script) => script)
// Meta tags
const meta = Array.from(document.querySelectorAll('meta')).reduce(
@@ -264,7 +269,7 @@ const Content = {
}
}
- Content.cache = { html, css, scripts, meta, cookies }
+ Content.cache = { html, css, scriptSrc, scripts, meta, cookies }
await Content.driver('onContentLoad', [
url,
diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js
index a06e31bc5..c8d9c043e 100644
--- a/src/drivers/webextension/js/driver.js
+++ b/src/drivers/webextension/js/driver.js
@@ -19,6 +19,8 @@ const hostnameIgnoreList =
const xhrDebounce = []
+const scriptsPending = []
+
const Driver = {
lastPing: Date.now(),
@@ -67,6 +69,11 @@ const Driver = {
['responseHeaders']
)
+ chrome.webRequest.onCompleted.addListener(Driver.onScriptRequestComplete, {
+ urls: ['http://*/*', 'https://*/*'],
+ types: ['script'],
+ })
+
chrome.webRequest.onCompleted.addListener(Driver.onXhrRequestComplete, {
urls: ['http://*/*', 'https://*/*'],
types: ['xmlhttprequest'],
@@ -400,6 +407,30 @@ const Driver = {
}
},
+ /**
+ * Analyse scripts
+ * @param {Object} request
+ */
+ async onScriptRequestComplete(request) {
+ if (await Driver.isDisabledDomain(request.url)) {
+ return
+ }
+
+ if (scriptsPending.includes(request.url)) {
+ scriptsPending.splice(scriptsPending.indexOf(request.url), 1)
+ } else if (request.statusCode === 200) {
+ scriptsPending.push(request.url)
+
+ const response = await fetch(request.url)
+
+ const scripts = await response.text()
+
+ Driver.onDetect(request.documentUrl, await analyze({ scripts })).catch(
+ Driver.error
+ )
+ }
+ },
+
/**
* Analyse XHR request hostnames
* @param {Object} request
diff --git a/src/technologies/_.json b/src/technologies/_.json
index 14ae1a1dd..4ba53a908 100644
--- a/src/technologies/_.json
+++ b/src/technologies/_.json
@@ -22,7 +22,7 @@
"recurring"
],
"saas": true,
- "scripts": "bitrix(?:\\.info/|/js/main/core)",
+ "scriptSrc": "bitrix(?:\\.info/|/js/main/core)",
"website": "http://www.1c-bitrix.ru"
},
"2B Advice": {
@@ -35,7 +35,7 @@
"BBCookieControler": ""
},
"saas": true,
- "scripts": "2badvice-cdn\\.azureedge\\.net",
+ "scriptSrc": "2badvice-cdn\\.azureedge\\.net",
"website": "https://www.2b-advice.com/en/data-privacy-software/cookie-consent-plugin/"
},
"33Across": {
@@ -61,7 +61,7 @@
"X-Powered-By": "3DCART"
},
"icon": "3dCart.png",
- "scripts": "(?:twlh(?:track)?\\.asp|3d_upsell\\.js)",
+ "scriptSrc": "(?:twlh(?:track)?\\.asp|3d_upsell\\.js)",
"website": "http://www.3dcart.com"
},
"4-Tell": {
@@ -82,7 +82,7 @@
"poa"
],
"saas": true,
- "scripts": "4tellcdn\\.azureedge\\.net",
+ "scriptSrc": "4tellcdn\\.azureedge\\.net",
"website": "https://4-tell.com"
},
"@sulu/web": {
diff --git a/src/technologies/a.json b/src/technologies/a.json
index 7dff16e3c..9896ca78e 100644
--- a/src/technologies/a.json
+++ b/src/technologies/a.json
@@ -9,7 +9,7 @@
"js": {
"AFRAME.version": "^(.+)$\\;version:\\1"
},
- "scripts": "/?([\\d.]+)?/aframe(?:\\.min)?\\.js\\;version:\\1",
+ "scriptSrc": "/?([\\d.]+)?/aframe(?:\\.min)?\\.js\\;version:\\1",
"website": "https://aframe.io"
},
"A8.net": {
@@ -24,7 +24,7 @@
"a8sales": "",
"map_A8": ""
},
- "scripts": "statics\\.a8\\.net",
+ "scriptSrc": "statics\\.a8\\.net",
"website": "https://www.a8.net"
},
"AB Tasty": {
@@ -42,7 +42,7 @@
"poa"
],
"saas": true,
- "scripts": "try\\.abtasty\\.com",
+ "scriptSrc": "try\\.abtasty\\.com",
"website": "https://www.abtasty.com"
},
"ABOUT YOU Commerce Suite": {
@@ -74,7 +74,7 @@
"payg"
],
"saas": true,
- "scripts": "\\.ebis\\.ne\\.jp/",
+ "scriptSrc": "\\.ebis\\.ne\\.jp/",
"website": "http://www.ebis.ne.jp"
},
"AMP": {
@@ -127,7 +127,7 @@
"AOS.refreshHard": "\\;confidence:50"
},
"oss": true,
- "scripts": [
+ "scriptSrc": [
"unpkg\\.com/aos@([\\d\\.]+)/dist/aos\\.js\\;version:\\1",
"/typo3conf/ext/udem_vendor/Resources/Public/aos-([\\d\\.]+)\\;version:\\1"
],
@@ -152,7 +152,7 @@
"js": {
"xt_click": ""
},
- "scripts": "xiti\\.com/hit\\.xiti",
+ "scriptSrc": "xiti\\.com/hit\\.xiti",
"website": "http://atinternet.com/en"
},
"ATSHOP": {
@@ -167,7 +167,7 @@
"recurring"
],
"saas": true,
- "scripts": "\\.atshop\\.io",
+ "scriptSrc": "\\.atshop\\.io",
"website": "https://atshop.io"
},
"AWIN": {
@@ -187,7 +187,7 @@
"payg"
],
"saas": true,
- "scripts": "dwin1\\.com",
+ "scriptSrc": "dwin1\\.com",
"website": "https://www.awin.com"
},
"AWS Certificate Manager": {
@@ -256,7 +256,7 @@
"recurring"
],
"saas": true,
- "scripts": "acsbapp?\\.com/.*/acsb\\.js",
+ "scriptSrc": "acsbapp?\\.com/.*/acsb\\.js",
"website": "https://accessibe.com"
},
"Accessibility Toolbar Plugin": {
@@ -282,7 +282,7 @@
"poa"
],
"saas": true,
- "scripts": [
+ "scriptSrc": [
"/npm/@accessible360/accessible-slick@([\\d\\.]+)/\\;version:\\1",
"/accessible360/accessible-slick/slick/slick\\.min\\.js\\?v=([\\d\\.]+)\\;version:\\1"
],
@@ -299,7 +299,7 @@
"recurring"
],
"saas": true,
- "scripts": "accessibly\\.onthemapmarketing\\.com",
+ "scriptSrc": "accessibly\\.onthemapmarketing\\.com",
"website": "https://www.onthemapmarketing.com/accessibly/"
},
"Accesso": {
@@ -312,7 +312,7 @@
"js": {
"accesso": ""
},
- "scripts": "/embed/accesso\\.js",
+ "scriptSrc": "/embed/accesso\\.js",
"website": "https://accesso.com/"
},
"Ackee": {
@@ -338,7 +338,7 @@
"Mautic"
],
"saas": true,
- "scripts": [
+ "scriptSrc": [
"mautic\\.net",
"maestro\\.mautic\\.com"
],
@@ -354,7 +354,7 @@
"Acquia Cloud Platform"
],
"saas": true,
- "scripts": [
+ "scriptSrc": [
"https?:\\/\\/.+\\.web\\.ahdev\\.cloud"
],
"url": "https:?\\/\\/.+\\.web\\.ahdev\\.cloud",
@@ -393,7 +393,7 @@
"Acquia Cloud Platform"
],
"saas": true,
- "scripts": [
+ "scriptSrc": [
"sites\\/g\\/files"
],
"website": "https://www.acquia.com/products/drupal-cloud/site-factory"
@@ -414,7 +414,7 @@
"Acquia Cloud Platform"
],
"saas": true,
- "scripts": [
+ "scriptSrc": [
"content-hub\\.acquia\\.com"
],
"url": "https?:\\/\\/.+\\.content-hub\\.acquia\\.com",
@@ -436,7 +436,7 @@
"agiloneObject": ""
},
"saas": true,
- "scripts": [
+ "scriptSrc": [
"^https?:\\/\\/.+\\.agilone\\.com",
"^https?:\\/\\/scripts\\.agilone\\.com\\/latest\\/a1.js"
],
@@ -460,7 +460,7 @@
"_tcaq": ""
},
"saas": true,
- "scripts": [
+ "scriptSrc": [
"lift\\.acquia\\.com"
],
"website": "https://www.acquia.com/products/marketing-cloud/personalization",
@@ -479,7 +479,7 @@
"Acquia Cloud Platform"
],
"saas": true,
- "scripts": [
+ "scriptSrc": [
"sites/\\w*/files/cohesion"
],
"website": "https://www.acquia.com/products/drupal-cloud/site-studio"
@@ -498,7 +498,7 @@
"recurring"
],
"saas": true,
- "scripts": "/cdnr/\\d+/acton/bn/tracker/\\d+",
+ "scriptSrc": "/cdnr/\\d+/acton/bn/tracker/\\d+",
"website": "http://act-on.com"
},
"Actito": {
@@ -520,7 +520,7 @@
"recurring"
],
"saas": true,
- "scripts": [
+ "scriptSrc": [
"cdn\\.actito\\.be",
"\\.advisor\\.smartfocus\\.com"
],
@@ -541,7 +541,7 @@
"recurring"
],
"saas": true,
- "scripts": [
+ "scriptSrc": [
"plugins/activecampaign-subscription-forms/site_tracking\\.js",
"\\.activehosted\\.com/"
],
@@ -562,7 +562,7 @@
"recurring"
],
"saas": true,
- "scripts": "\\.acuityscheduling\\.com",
+ "scriptSrc": "\\.acuityscheduling\\.com",
"website": "https://acuityscheduling.com"
},
"Ad Lightning": {
@@ -575,7 +575,7 @@
"poa"
],
"saas": true,
- "scripts": "\\.adlightning\\.com",
+ "scriptSrc": "\\.adlightning\\.com",
"website": "https://www.adlightning.com"
},
"AdInfinity": {
@@ -583,7 +583,7 @@
36
],
"icon": "AdInfinity.png",
- "scripts": "adinfinity\\.com\\.au",
+ "scriptSrc": "adinfinity\\.com\\.au",
"website": "http://adinfinity.com.au"
},
"AdOcean": {
@@ -597,7 +597,7 @@
"ado.placement": "",
"ado.slave": ""
},
- "scripts": [
+ "scriptSrc": [
"adocean\\.pl/files/js/ado\\.js",
"adocean\\.pl\\;confidence:80"
],
@@ -612,7 +612,7 @@
"js": {
"adriver": ""
},
- "scripts": "(?:adriver\\.core\\.\\d\\.js|https?://(?:content|ad|masterh\\d)\\.adriver\\.ru/)",
+ "scriptSrc": "(?:adriver\\.core\\.\\d\\.js|https?://(?:content|ad|masterh\\d)\\.adriver\\.ru/)",
"website": "http://adriver.ru"
},
"AdRoll": {
@@ -631,7 +631,7 @@
"recurring"
],
"saas": true,
- "scripts": "(?:a|s)\\.adroll\\.com",
+ "scriptSrc": "(?:a|s)\\.adroll\\.com",
"website": "http://adroll.com"
},
"AdRoll CMP System": {
@@ -668,7 +668,7 @@
"payg"
],
"saas": true,
- "scripts": "\\.adscale\\.com/",
+ "scriptSrc": "\\.adscale\\.com/",
"website": "https://www.adscale.com"
},
"AdThrive": {
@@ -682,7 +682,7 @@
"adthriveVideosInjected": ""
},
"saas": true,
- "scripts": "ads\\.adthrive\\.com",
+ "scriptSrc": "ads\\.adthrive\\.com",
"website": "https://www.adthrive.com"
},
"Ada": {
@@ -699,7 +699,7 @@
"poa"
],
"saas": true,
- "scripts": "\\.ada\\.support",
+ "scriptSrc": "\\.ada\\.support",
"website": "https://www.ada.cx"
},
"Adabra": {
@@ -718,7 +718,7 @@
"recurring"
],
"saas": true,
- "scripts": "track\\.adabra\\.com",
+ "scriptSrc": "track\\.adabra\\.com",
"website": "https://www.adabra.com",
"xhr": "my\\.adabra\\.com"
},
@@ -727,7 +727,7 @@
68
],
"icon": "Adally.png",
- "scripts": "cloudfront\\.net/.*/adally\\.js",
+ "scriptSrc": "cloudfront\\.net/.*/adally\\.js",
"website": "https://adally.com/"
},
"Adalyser": {
@@ -739,7 +739,7 @@
"js": {
"adalyserModules": ""
},
- "scripts": "c5\\.adalyser\\.com",
+ "scriptSrc": "c5\\.adalyser\\.com",
"website": "https://adalyser.com/"
},
"Adcash": {
@@ -756,7 +756,7 @@
"ct_siteunder": "",
"ct_tag": ""
},
- "scripts": "^[^\\/]*//(?:[^\\/]+\\.)?adcash\\.com/(?:script|ad)/",
+ "scriptSrc": "^[^\\/]*//(?:[^\\/]+\\.)?adcash\\.com/(?:script|ad)/",
"url": "^https?://(?:[^\\/]+\\.)?adcash\\.com/script/pop_",
"website": "http://adcash.com"
},
@@ -771,7 +771,7 @@
"poa"
],
"saas": true,
- "scripts": "(?:cdn\\.)?shop\\.pe/widget/",
+ "scriptSrc": "(?:cdn\\.)?shop\\.pe/widget/",
"website": "http://www.addshoppers.com"
},
"AddThis": {
@@ -783,7 +783,7 @@
"js": {
"addthis": ""
},
- "scripts": "addthis\\.com/js/",
+ "scriptSrc": "addthis\\.com/js/",
"website": "http://www.addthis.com"
},
"AddToAny": {
@@ -795,7 +795,7 @@
"js": {
"a2apage_init": ""
},
- "scripts": "addtoany\\.com/menu/page\\.js",
+ "scriptSrc": "addtoany\\.com/menu/page\\.js",
"website": "http://www.addtoany.com"
},
"Adminer": {
@@ -823,7 +823,7 @@
"pricing": [
"payg"
],
- "scripts": [
+ "scriptSrc": [
"artfut\\.com/static/(?:tracking|crossdevice)\\.min\\.js",
"cdn\\.admitad\\.com"
],
@@ -838,7 +838,7 @@
},
"html": "