Enable multiple headers with the same name & Get scripts via DOM (#1819)

* Enable multiple header with same name

Enable multiple header with same name for WebExtension driver

*  Enable multiple header with same name

Enable multiple header with same name for Bookmarklet driver

*  Enable multiple header with same name

Enable multiple header with same name for the NPM driver

* Enable multiple headers with the same name

* Get scripts via DOM (WebExtension)

* Get scripts via DOM (Bookmarklet)

* Get scripts via DOM

* Get scripts via DOM (NPM)

src/drivers/bookmarklet/driver.js

@@ -18,8 +18,8 @@
     console.log('[wappalyzer ' + type + ']', '[' + source + ']', message);
   };
 
-  function getEnvironmentVars() {
+  function getPageContent() {
-    wappalyzer.log('func: getEnvironmentVars');
+    wappalyzer.log('func: getPageContent');
 
     var env = [];
 
@@ -27,9 +27,15 @@
       env.push(i);
     }
 
+    var scripts = Array.prototype.slice
+      .apply(document.scripts)
+      .filter(s => s.src)
+      .map(s => s.src);
+
     wappalyzer.analyze(domain, url, {
       html: document.documentElement.innerHTML,
-      env: env
+      env: env,
+      scripts: scripts
     });
   }
 
@@ -56,7 +62,10 @@
               name  = line.substring(0, line.indexOf(': '));
               value = line.substring(line.indexOf(': ') + 2, line.length - 1);
 
-              responseHeaders[name.toLowerCase()] = value;
+              if ( !responseHeaders[name.toLowerCase()] ){
+                responseHeaders[name.toLowerCase()] = []
+              }
+              responseHeaders[name.toLowerCase()].push(value);
             }
           });
 
@@ -139,6 +148,6 @@
     return string.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/--+/g, '-').replace(/(?:^-|-$)/, '');
   }
 
-  getEnvironmentVars();
+  getPageContent();
   getResponseHeaders();
 })();

src/drivers/npm/driver.js

@@ -72,18 +72,26 @@ const driver = options => {
               const headers = {};
 
               browser.resources['0'].response.headers._headers.forEach(header => {
-                headers[header[0]] = header[1];
+                if ( !headers[header[0]] ){
+                  headers[header[0]] = [];
+                }
+                headers[header[0]].push(header[1]);
               });
 
               const vars = Object.getOwnPropertyNames(browser.window);
               const html = browser.html();
+              const scripts = Array.prototype.slice
+                .apply(browser.document.scripts)
+                .filter(s => s.src)
+                .map(s => s.src);
 
             const hostname = wappalyzer.parseUrl(url).hostname;
 
               wappalyzer.analyze(hostname, url, {
                 headers,
                 html,
-                env: vars
+                env: vars,
+                scripts
               });
             });
         });

src/drivers/webextension/js/content.js

@@ -7,6 +7,11 @@ if ( typeof browser !== 'undefined' && typeof document.body !== 'undefined' ) {
     html = html.substring(0, 25000) + html.substring(html.length - 25000, html.length);
   }
 
+  var scripts = Array.prototype.slice
+      .apply(document.scripts)
+      .filter(s => s.src)
+      .map(s => s.src);
+
   try {
     browser.runtime.sendMessage({
       id: 'analyze',
@@ -14,6 +19,12 @@ if ( typeof browser !== 'undefined' && typeof document.body !== 'undefined' ) {
       source: 'content.js'
     });
 
+    browser.runtime.sendMessage({
+      id: 'analyze',
+      subject: { scripts },
+      source: 'content.js'
+    });
+
     var container = document.createElement('wappalyzerData');
 
     container.setAttribute('id',    'wappalyzerData');

src/drivers/webextension/js/driver.js

@@ -138,20 +138,23 @@ browser.webRequest.onCompleted.addListener(request => {
     var url = wappalyzer.parseUrl(request.url);
 
     request.responseHeaders.forEach(function(header) {
-      responseHeaders[header.name.toLowerCase()] = header.value || '' + header.binaryValue;
+      if ( !responseHeaders[header.name.toLowerCase()] ) {
+        responseHeaders[header.name.toLowerCase()] = []
+      }
+      responseHeaders[header.name.toLowerCase()].push(header.value || '' + header.binaryValue);
     });
 
     if ( headersCache.length > 50 ) {
       headersCache = {};
     }
 
-    if ( /text\/html/.test(responseHeaders['content-type']) ) {
+    if ( /text\/html/.test(responseHeaders['content-type'][0]) ) {
       if ( headersCache[url.canonical] === undefined ) {
         headersCache[url.canonical] = {};
       }
 
       Object.keys(responseHeaders).forEach(header => {
-        headersCache[url.canonical][header] = responseHeaders[header];
+        headersCache[url.canonical][header] = responseHeaders[header].slice();
       });
     }
   }

src/wappalyzer.js

@@ -62,10 +62,13 @@ class Wappalyzer {
 
       if ( data.html ) {
         this.analyzeHtml(app, data.html);
-        this.analyzeScript(app, data.html);
         this.analyzeMeta(app, data.html);
       }
 
+      if ( data.scripts ) {
+        this.analyzeScripts(app, data.scripts);
+      }
+      
       if ( data.headers ) {
         this.analyzeHeaders(app, data.headers);
       }
@@ -396,20 +399,19 @@ class Wappalyzer {
   /**
    * Analyze script tag
    */
-  analyzeScript(app, html) {
+  analyzeScripts(app, scripts) {
-    var regex = new RegExp('<script[^>]+src=("|\')([^"\']+)', 'ig');
     var patterns = this.parsePatterns(app.props.script);
 
     if ( patterns.length ) {
       patterns.forEach(pattern => {
         var match;
 
-        while ( ( match = regex.exec(html) ) ) {
+        scripts.forEach(uri => {
-          if ( pattern.regex.test(match[2]) ) {
+          if ( pattern.regex.test(uri) ) {
-            this.addDetected(app, pattern, 'script', match[2]);
+            this.addDetected(app, pattern, 'script', uri);
-          }
           }
         });
+      });
     }
   }
 
@@ -444,12 +446,16 @@ class Wappalyzer {
     var patterns = this.parsePatterns(app.props.headers);
 
     if ( headers ) {
-      Object.keys(patterns).forEach(header => {
+      Object.keys(patterns).forEach(headerName => {
-        patterns[header].forEach(pattern => {
+        patterns[headerName].forEach(pattern => {
-          header = header.toLowerCase();
+          headerName = headerName.toLowerCase();
 
-          if ( header in headers && pattern.regex.test(headers[header]) ) {
+          if ( headerName in headers ) {
-            this.addDetected(app, pattern, 'headers', headers[header], header);
+            headers[headerName].forEach(headerValue => {
+              if ( pattern.regex.test(headerValue) ) {
+                this.addDetected(app, pattern, 'headers', headerValue, headerName);
+              }
+            });
           }
         });
       });