diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index 930cbe835..a75f7c8fb 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -721,7 +721,7 @@ class Site { await this.promiseTimeout( page.evaluateHandle(() => // eslint-disable-next-line unicorn/prefer-text-content - document.body.innerText.replace(/\s+/g, ' ') + document.body.innerText.replace(/\s+/g, ' ').slice(0, 25000) ), { jsonValue: () => '' }, 'Timeout (text)' diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js index f2876972f..58c9d19bb 100644 --- a/src/drivers/webextension/js/content.js +++ b/src/drivers/webextension/js/content.js @@ -178,7 +178,7 @@ const Content = { // Text // eslint-disable-next-line unicorn/prefer-text-content - const text = document.body.innerText.replace(/\s+/g, ' ') + const text = document.body.innerText.replace(/\s+/g, ' ').slice(0, 25000) // CSS rules let css = [] diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index b2a484c5a..908805692 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -19,6 +19,8 @@ const hostnameIgnoreList = const xhrDebounce = [] +let xhrAnalyzed = {} + const scriptsPending = [] function getRequiredTechnologies(name, categoryId) { @@ -458,16 +460,34 @@ const Driver = { return } + let originHostname + + try { + ;({ hostname: originHostname } = new URL(request.originUrl)) + } catch (error) { + return + } + if (!xhrDebounce.includes(hostname)) { xhrDebounce.push(hostname) setTimeout(async () => { xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1) - Driver.onDetect( - request.originUrl || request.initiator, - await analyze({ xhr: hostname }) - ).catch(Driver.error) + xhrAnalyzed[originHostname] = xhrAnalyzed[originHostname] || [] + + if (!xhrAnalyzed[originHostname].includes(hostname)) { + xhrAnalyzed[originHostname].push(hostname) + + if (Object.keys(xhrAnalyzed).length > 500) { + xhrAnalyzed = {} + } + + Driver.onDetect( + request.originUrl || request.initiator, + await analyze({ xhr: hostname }) + ).catch(Driver.error) + } }, 1000) } }, @@ -886,6 +906,8 @@ const Driver = { Driver.cache.hostnames = {} Driver.cache.tabs = {} + xhrAnalyzed = {} + await setOption('hostnames', {}) },