From 7a3ed8add4cf05a5abe717e7c566c98326db6532 Mon Sep 17 00:00:00 2001
From: Elbert Alias <77259+AliasIO@users.noreply.github.com>
Date: Tue, 2 Nov 2021 13:15:47 +1100
Subject: [PATCH] Prevent repeat analysis of the identical XHR requests

---
 src/drivers/npm/driver.js              |  2 +-
 src/drivers/webextension/js/content.js |  2 +-
 src/drivers/webextension/js/driver.js  | 30 ++++++++++++++++++++++----
 3 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js
index 930cbe835..a75f7c8fb 100644
--- a/src/drivers/npm/driver.js
+++ b/src/drivers/npm/driver.js
@@ -721,7 +721,7 @@ class Site {
             await this.promiseTimeout(
               page.evaluateHandle(() =>
                 // eslint-disable-next-line unicorn/prefer-text-content
-                document.body.innerText.replace(/\s+/g, ' ')
+                document.body.innerText.replace(/\s+/g, ' ').slice(0, 25000)
               ),
               { jsonValue: () => '' },
               'Timeout (text)'
diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js
index f2876972f..58c9d19bb 100644
--- a/src/drivers/webextension/js/content.js
+++ b/src/drivers/webextension/js/content.js
@@ -178,7 +178,7 @@ const Content = {
 
       // Text
       // eslint-disable-next-line unicorn/prefer-text-content
-      const text = document.body.innerText.replace(/\s+/g, ' ')
+      const text = document.body.innerText.replace(/\s+/g, ' ').slice(0, 25000)
 
       // CSS rules
       let css = []
diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js
index b2a484c5a..908805692 100644
--- a/src/drivers/webextension/js/driver.js
+++ b/src/drivers/webextension/js/driver.js
@@ -19,6 +19,8 @@ const hostnameIgnoreList =
 
 const xhrDebounce = []
 
+let xhrAnalyzed = {}
+
 const scriptsPending = []
 
 function getRequiredTechnologies(name, categoryId) {
@@ -458,16 +460,34 @@ const Driver = {
       return
     }
 
+    let originHostname
+
+    try {
+      ;({ hostname: originHostname } = new URL(request.originUrl))
+    } catch (error) {
+      return
+    }
+
     if (!xhrDebounce.includes(hostname)) {
       xhrDebounce.push(hostname)
 
       setTimeout(async () => {
         xhrDebounce.splice(xhrDebounce.indexOf(hostname), 1)
 
-        Driver.onDetect(
-          request.originUrl || request.initiator,
-          await analyze({ xhr: hostname })
-        ).catch(Driver.error)
+        xhrAnalyzed[originHostname] = xhrAnalyzed[originHostname] || []
+
+        if (!xhrAnalyzed[originHostname].includes(hostname)) {
+          xhrAnalyzed[originHostname].push(hostname)
+
+          if (Object.keys(xhrAnalyzed).length > 500) {
+            xhrAnalyzed = {}
+          }
+
+          Driver.onDetect(
+            request.originUrl || request.initiator,
+            await analyze({ xhr: hostname })
+          ).catch(Driver.error)
+        }
       }, 1000)
     }
   },
@@ -886,6 +906,8 @@ const Driver = {
     Driver.cache.hostnames = {}
     Driver.cache.tabs = {}
 
+    xhrAnalyzed = {}
+
     await setOption('hostnames', {})
   },