Asynchronous regex execution

src/drivers/npm/driver.js

@@ -131,8 +131,8 @@ class Driver {
         html,
         scripts,
         js
-      });
+      })
-
+        .then(() => {
           const links = Array.from(browser.document.getElementsByTagName('a'))
             .filter(link => link.protocol === 'http:' || link.protocol === 'https:')
             .filter(link => link.hostname === this.origPageUrl.hostname)
@@ -141,6 +141,7 @@ class Driver {
 
           return resolve(links);
         });
+    });
   }
 
   responseOk(browser, pageUrl) {

src/drivers/webextension/js/content.js

@@ -3,11 +3,12 @@
 
 if ( typeof browser !== 'undefined' && typeof document.body !== 'undefined' ) {
   try {
-    var html = new XMLSerializer().serializeToString(document);
+    var html = new XMLSerializer().serializeToString(document).split('\n');
 
-    if ( html.length > 100 * 1024 ) {
+    html = html
-      html = html.substring(0, 50 * 1024) + html.substring(html.length - 50 * 1024, html.length);
+      .slice(0, 1000).concat(html.slice(html.length - 1000))
-    }
+      .map(line => line.substring(0, 1000))
+      .join('\n');
 
     const scripts = Array.prototype.slice
       .apply(document.scripts)

src/drivers/webextension/js/driver.js

@@ -103,10 +103,10 @@ getOption('version')
       getOption('upgradeMessage', true)
         .then(upgradeMessage => {
           if ( upgradeMessage ) {
-            // openTab({
+            openTab({
-            //   url: wappalyzer.config.websiteURL + 'upgraded?v' + version,
+              url: wappalyzer.config.websiteURL + 'upgraded?v' + version,
-            //   background: true
+              background: true
-            // });
+            });
           }
         });
     }

src/drivers/webextension/js/inject.js

@@ -20,7 +20,7 @@
               for ( let index in patterns[appName][chain] ) {
                 const value = detectJs(chain);
 
-                if ( value ) {
+                if ( value && patterns[appName][chain].hasOwnProperty(index) ) {
                   js[appName][chain][index] = value;
                 }
               }

src/wappalyzer.js

@@ -38,7 +38,13 @@ class Wappalyzer {
     this.driver.log(message, source || '', type || 'debug');
   }
 
+  asyncForEach(iterable, iterator) {
+    return Promise.all(iterable.map(item => new Promise(resolve => setTimeout(() => resolve(iterator(item)), 1))));
+  }
+
   analyze(url, data, context) {
+    const promises = [];
+
     var apps = {};
 
     if ( typeof data.html !== 'string' ) {
@@ -62,33 +68,32 @@ class Wappalyzer {
       this.analyzeUrl(app, url);
 
       if ( data.html ) {
-        this.analyzeHtml(app, data.html);
+        promises.push(this.analyzeHtml(app, data.html));
-        this.analyzeMeta(app, data.html);
+        promises.push(this.analyzeMeta(app, data.html));
       }
 
       if ( data.scripts ) {
-        this.analyzeScripts(app, data.scripts);
+        promises.push(this.analyzeScripts(app, data.scripts));
       }
 
       if ( data.headers ) {
-        this.analyzeHeaders(app, data.headers);
+        promises.push(this.analyzeHeaders(app, data.headers));
       }
 
       if ( data.env ) {
-        this.analyzeEnv(app, data.env);
+        promises.push(this.analyzeEnv(app, data.env));
-      }
-
-      if ( data.robotsTxt ) {
-        this.analyzeRobotsTxt(app, data.robotsTxt);
       }
     })
 
     if ( data.js ) {
       Object.keys(data.js).forEach(appName => {
-        this.analyzeJs(apps[appName], data.js[appName]);
+        promises.push(this.analyzeJs(apps[appName], data.js[appName]));
       });
     }
 
+    return new Promise(resolve => {
+      Promise.all(promises)
+        .then(() => {
           Object.keys(apps).forEach(appName => {
             var app = apps[appName];
 
@@ -108,6 +113,10 @@ class Wappalyzer {
           }
 
           this.driver.displayApps(this.detected[url.canonical], { language }, context);
+
+          resolve();
+        });
+    });
   }
 
   /**
@@ -393,14 +402,16 @@ class Wappalyzer {
   analyzeUrl(app, url) {
     var patterns = this.parsePatterns(app.props.url);
 
-    if ( patterns.length ) {
+    if ( !patterns.length ) {
-      patterns.forEach(pattern => {
+      return Promise.resolve();
+    }
+
+    return this.asyncForEach(patterns, pattern => {
       if ( pattern.regex.test(url.canonical) ) {
         this.addDetected(app, pattern, 'url', url.canonical);
       }
     });
   }
-  }
 
   /**
    * Analyze HTML
@@ -408,14 +419,16 @@ class Wappalyzer {
   analyzeHtml(app, html) {
     var patterns = this.parsePatterns(app.props.html);
 
-    if ( patterns.length ) {
+    if ( !patterns.length ) {
-      patterns.forEach(pattern => {
+      return Promise.resolve();
+    }
+
+    return this.asyncForEach(patterns, pattern => {
       if ( pattern.regex.test(html) ) {
         this.addDetected(app, pattern, 'html', html);
       }
     });
   }
-  }
 
   /**
    * Analyze script tag
@@ -423,8 +436,11 @@ class Wappalyzer {
   analyzeScripts(app, scripts) {
     var patterns = this.parsePatterns(app.props.script);
 
-    if ( patterns.length ) {
+    if ( !patterns.length ) {
-      patterns.forEach(pattern => {
+      return Promise.resolve();
+    }
+
+    return this.asyncForEach(patterns, pattern => {
       var match;
 
       scripts.forEach(uri => {
@@ -434,56 +450,61 @@ class Wappalyzer {
       });
     });
   }
-  }
 
   /**
    * Analyze meta tag
    */
   analyzeMeta(app, html) {
-    var regex = /<meta[^>]+>/ig;
+    const regex = /<meta[^>]+>/ig;
-    var patterns = this.parsePatterns(app.props.meta);
+    const patterns = this.parsePatterns(app.props.meta);
+    const promises = [];
+
     var content = '';
     var matches = [];
 
     while ( patterns && ( matches = regex.exec(html) ) ) {
       for ( var meta in patterns ) {
-
         const r = new RegExp('(?:name|property)=["\']' + meta + '["\']', 'i');
 
-        if ( new RegExp('(?:name|property)=["\']' + meta + '["\']', 'i').test(matches[0]) ) {
+        if ( r.test(matches[0]) ) {
           content = matches[0].match(/content=("|')([^"']+)("|')/i);
 
-          patterns[meta].forEach(pattern => {
+          promises.push(this.asyncForEach(patterns[meta], pattern => {
             if ( content && content.length === 4 && pattern.regex.test(content[2]) ) {
               this.addDetected(app, pattern, 'meta', content[2], meta);
             }
-          });
+          }));
         }
       }
     }
+
+    return promises ? Promise.all(promises) : Promise.resolve();
   }
 
   /**
    * analyze response headers
    */
   analyzeHeaders(app, headers) {
-    var patterns = this.parsePatterns(app.props.headers);
+    const patterns = this.parsePatterns(app.props.headers);
+    const promises = [];
 
     if ( headers ) {
       Object.keys(patterns).forEach(headerName => {
-        patterns[headerName].forEach(pattern => {
+        this.asyncForEach(patterns[headerName], pattern => {
           headerName = headerName.toLowerCase();
 
           if ( headerName in headers ) {
-            headers[headerName].forEach(headerValue => {
+            promises.push(headers[headerName].forEach(headerValue => {
               if ( pattern.regex.test(headerValue) ) {
                 this.addDetected(app, pattern, 'headers', headerValue, headerName);
               }
-            });
+            }));
           }
         });
       });
     }
+
+    return promises ? Promise.all(promises) : Promise.resolve();
   }
 
   /**
@@ -493,7 +514,10 @@ class Wappalyzer {
     var patterns = this.parsePatterns(app.props.env);
 
     if ( patterns.length ) {
-      patterns.forEach(pattern => {
+      return Promise.resolve();
+    }
+
+    return this.asyncForEach(patterns, pattern => {
       Object.keys(envs).forEach(env => {
         if ( pattern.regex.test(envs[env]) ) {
           this.addDetected(app, pattern, 'env', envs[env]);
@@ -501,37 +525,25 @@ class Wappalyzer {
       })
     });
   }
-  }
 
   /**
    * Analyze JavaScript variables
    */
   analyzeJs(app, results) {
+    const promises = [];
+
     Object.keys(results).forEach(string => {
-      Object.keys(results[string]).forEach(index => {
+      promises.push(this.asyncForEach(Object.keys(results[string]), index => {
         const pattern = this.jsPatterns[app.name][string][index];
         const value = results[string][index];
 
-        if ( pattern.regex.test(value) ) {
+        if ( pattern && pattern.regex.test(value) ) {
           this.addDetected(app, pattern, 'js', value);
         }
+      }));
     });
-    });
-  }
-
-  /**
-   * Analyze robots.txt
-   */
-  analyzeRobotsTxt(app, robotsTxt) {
-    var patterns = this.parsePatterns(app.props.robotsTxt);
 
-    if ( patterns.length ) {
+    return promises ? Promise.all(promises) : Promise.resolve();
-      patterns.forEach(pattern => {
-        if ( pattern.regex.test(robotsTxt) ) {
-          this.addDetected(app, pattern, 'robotsTxt', robotsTxt);
-        }
-      });
-    }
   }
 
   /**
@@ -540,6 +552,8 @@ class Wappalyzer {
   addDetected(app, pattern, type, value, key) {
     app.detected = true;
 
+    console.log(app);
+
     // Set confidence level
     app.confidence[type + ' ' + ( key ? key + ' ' : '' ) + pattern.regex] = pattern.confidence || 100;