Sync with AliasIO (#1)

* Refactoring, allow multiple expressions per js field, remove env detection * Add js field to NPM driver * Port env to js
8 years ago · a1079095dd
parent 28d2afaa61
commit a1079095dd
8 changed files with 12285 additions and 11574 deletions
--- a/bin/validate
+++ b/bin/validate
@ -6,7 +6,7 @@ set -eu

 echo "Validating apps.json..."

-jsonlint-cli -s schema.json src/apps.json
+jsonlint-cli -tps schema.json src/apps.json > /tmp/apps.json && mv /tmp/apps.json src/apps.json

 echo "Validating regular expressions..."

--- a/schema.json
+++ b/schema.json
@ -35,9 +35,9 @@
 						},
 						"required": true
 					},
-					"env": {
-						"type": [ "string", "array" ],
-						"items": {
+					"js": {
+						"type": "object",
+						"additionalProperties": {
 							"type": "string"
 						}
 					},
--- a/src/apps.json
+++ b/src/apps.json
--- a/src/drivers/npm/driver.js
+++ b/src/drivers/npm/driver.js
@ -37,6 +37,8 @@ class Driver {
    this.wappalyzer.apps = json.apps;
    this.wappalyzer.categories = json.categories;

+    this.wappalyzer.parseJsPatterns();
+
    this.wappalyzer.driver.log = (message, source, type) => this.log(message, source, type);
    this.wappalyzer.driver.displayApps = detected => this.displayApps(detected);
  }
@ -120,6 +122,28 @@ class Driver {
              .finally(() => {
                this.timer('browser.wait end');

+                const headers = this.getHeaders(browser);
+                const html    = this.getHtml(browser);
+                const scripts = this.getScripts(browser);
+                const js      = this.getJs(browser);
+
+                this.wappalyzer.analyze(pageUrl.hostname, pageUrl.href, {
+                  headers,
+                  html,
+                  js,
+                  scripts
+                });
+
+                const links = browser.body.getElementsByTagName('a');
+
+                resolve(links);
+              });
+          });
+        });
+    });
+  }
+
+  getHeaders(browser) {
    const headers = {};

    browser.resources['0'].response.headers._headers.forEach(header => {
@ -130,6 +154,10 @@ class Driver {
      headers[header[0]].push(header[1]);
    });

+    return headers;
+  }
+
+  getHtml(browser) {
    let html = '';

    try {
@ -138,26 +166,45 @@ class Driver {
      this.wappalyzer.log(error.message, 'browser', 'error');
    }

-                const vars = Object.getOwnPropertyNames(browser.window);
+    return html;
+  }
+
+  getScripts(browser) {
    const scripts = Array.prototype.slice
      .apply(browser.document.scripts)
      .filter(s => s.src)
      .map(s => s.src);

-                this.wappalyzer.analyze(pageUrl.hostname, pageUrl.href, {
-                  headers,
-                  html,
-                  env: vars,
-                  scripts
-                });
+    return scripts;
+  }

-                const links = browser.body.getElementsByTagName('a');
+  getJs(browser) {
+    const patterns = this.wappalyzer.jsPatterns;
+    const js = {};

-                resolve(links);
-              });
+    Object.keys(patterns).forEach(appName => {
+      js[appName] = {};
+
+      Object.keys(patterns[appName]).forEach(chain => {
+        js[appName][chain] = {};
+
+        patterns[appName][chain].forEach((pattern, index) => {
+          const properties = chain.split('.');
+
+          let value = properties.reduce((parent, property) => {
+            return parent && parent.hasOwnProperty(property) ? parent[property] : null;
+          }, browser.window);
+
+          value = typeof value === 'string' ? value : !!value;
+
+          if ( value ) {
+            js[appName][chain][index] = value;
+          }
        });
      });
    });
+
+    return js;
  }

  crawl(pageUrl, index = 1, depth = 1) {
--- a/src/drivers/webextension/js/content.js
+++ b/src/drivers/webextension/js/content.js
@ -1,76 +1,58 @@
 /** global: browser */

 if ( typeof browser !== 'undefined' && typeof document.body !== 'undefined' ) {
+  try {
    var html = document.documentElement.outerHTML;

    if ( html.length > 50000 ) {
      html = html.substring(0, 25000) + html.substring(html.length - 25000, html.length);
    }

-  var scripts = Array.prototype.slice
+    const scripts = Array.prototype.slice
      .apply(document.scripts)
-      .filter(s => s.src)
-      .map(s => s.src);
-
-  try {
-    browser.runtime.sendMessage({
-      id: 'analyze',
-      subject: { html },
-      source: 'content.js'
-    });
+      .filter(script => script.src)
+      .map(script => script.src);

    browser.runtime.sendMessage({
      id: 'analyze',
-      subject: { scripts },
+      subject: { html, scripts },
      source: 'content.js'
    });

-    var container = document.createElement('wappalyzerData');
+    const script = document.createElement('script');

-    container.setAttribute('id',    'wappalyzerData');
-    container.setAttribute('style', 'display: none');
-
-    var script = document.createElement('script');
-
-    script.setAttribute('id', 'wappalyzerEnvDetection');
-    script.setAttribute('src', browser.extension.getURL('js/inject.js'));
-
-    container.addEventListener('wappalyzerEnvEvent', (event => {
-      browser.runtime.sendMessage({
-        id: 'JS_ready',
-        subject: { },
-        source: 'content.js'
-      }, response => {
-        window.postMessage({patterns: response.patterns}, "*");
-      });
-      window.addEventListener('message', (event => {
-        if (event.data.js === undefined)
+    script.onload = () => {
+      addEventListener('message', event => {
+        if ( event.data.id !== 'js' ) {
          return;
-        var js = event.data.js ;
+        }
+
        browser.runtime.sendMessage({
          id: 'analyze',
-          subject: { js },
+          subject: {
+            js: event.data.js
+          },
          source: 'content.js'
        });
-      }), true);
-
-      var env = event.target.childNodes[0].nodeValue;
-
-      document.documentElement.removeChild(container);
-      document.documentElement.removeChild(script);
+      }, true);

-      env = env.split(' ').slice(0, 500);
-
-      browser.runtime.sendMessage({
-        id: 'analyze',
-        subject: { env },
+      ( chrome || browser ).runtime.sendMessage({
+        id: 'init_js',
+        subject: {},
        source: 'content.js'
+      }, response => {
+        postMessage({
+          id: 'patterns',
+          patterns: response.patterns
+        }, '*');
      });
-    }), true);
+    };
+
+    script.setAttribute('id', 'wappalyzer');
+    script.setAttribute('src', browser.extension.getURL('js/inject.js'));

-    document.documentElement.appendChild(container);
-    document.documentElement.appendChild(script);
-  } catch(e) {
+    document.body.appendChild(script);
+  } catch (e) {
    log(e);
  }
 }
--- a/src/drivers/webextension/js/driver.js
+++ b/src/drivers/webextension/js/driver.js
@ -74,7 +74,8 @@ fetch('../apps.json')
  .then(json => {
    wappalyzer.apps = json.apps;
    wappalyzer.categories = json.categories;
-    wappalyzer.parseJS();
+
+    wappalyzer.parseJsPatterns();

    categoryOrder = Object.keys(wappalyzer.categories).sort((a, b) => wappalyzer.categories[a].priority - wappalyzer.categories[b].priority);
  })
@ -193,9 +194,9 @@ browser.webRequest.onCompleted.addListener(request => {
        };

        break;
-      case 'JS_ready':
+      case 'init_js':
        response = {
-          patterns: wappalyzer.parsedJS
+          patterns: wappalyzer.jsPatterns
        };

        break;
--- a/src/drivers/webextension/js/inject.js
+++ b/src/drivers/webextension/js/inject.js
@ -1,41 +1,41 @@
 (function() {
 	try {
-		var i, environmentVars = '', eEnv = document.createEvent('Events'), container = document.getElementById('wappalyzerData');
+    addEventListener('message', (event => {
+      if ( event.data.id !== 'patterns' ) {
+        return;
+      }

-		eEnv.initEvent('wappalyzerEnvEvent', true, false);
+      const patterns = event.data.patterns || {};

-		for ( i in window ) {
-			environmentVars += i + ' ';
-		}
-		container.appendChild(document.createComment(environmentVars));
-		container.dispatchEvent(eEnv);
+      const js = {};

-		window.addEventListener('message', (event => {
-			if (event.data.patterns === undefined)
-				return;
-			var properties = event.data.patterns;
-			var js = {};
-			Object.keys(properties).forEach(appname => {
-				Object.keys(properties[appname]).forEach(property => {
-					var content = false;
-					if( content = JSdetection(property) ){
-						if ( js[appname]  === undefined )
-							js[appname] = {};
-						js[appname][property] = properties[appname][property];
-						js[appname][property]["content"] = content;
+      Object.keys(patterns).forEach(appName => {
+        js[appName] = {};
+
+        Object.keys(patterns[appName]).forEach(chain => {
+          js[appName][chain] = {};
+
+          patterns[appName][chain].forEach((pattern, index) => {
+            const value = detectJs(chain);
+
+            if ( value ) {
+              js[appName][chain][index] = value;
            }
          });
        });
-			window.postMessage({js: js}, "*");
+      });
+
+      postMessage({ id: 'js', js }, '*');
    }), false);
  } catch(e) {
    // Fail quietly
  }
 }());

-function JSdetection(p){
-	const objects = p.split('.');
-	const value = objects.reduce((parent, property) => {
+function detectJs(chain) {
+  const properties = chain.split('.');
+
+  const value = properties.reduce((parent, property) => {
    return parent && parent.hasOwnProperty(property) ? parent[property] : null;
  }, window);

--- a/src/wappalyzer.js
+++ b/src/wappalyzer.js
@ -18,7 +18,7 @@ class Wappalyzer {
    this.apps = {};
    this.categories = {};
    this.driver = {};
-    this.parsedJS = undefined;
+    this.jsPatterns = {};

    this.detected = {};
    this.hostnameCache = {};
@ -85,7 +85,7 @@ class Wappalyzer {

    if ( data.js ) {
      Object.keys(data.js).forEach(appName => {
-        this.analyzeJS(apps[appName], data.js[appName]);
+        this.analyzeJs(apps[appName], data.js[appName]);
      });
    }

@ -252,16 +252,14 @@ class Wappalyzer {
  }

  /**
-   * Parse JS patterns
+   * Parse JavaScript patterns
   */
-  parseJS() {
-    if ( this.parsedJS === undefined){
-      this.parsedJS = {};
+  parseJsPatterns() {
    Object.keys(this.apps).forEach(appName => {
-        if (this.apps[appName].js)
-          this.parsedJS[appName] = this.parsePatterns(this.apps[appName].js);
-      });
+      if ( this.apps[appName].js ) {
+        this.jsPatterns[appName] = this.parsePatterns(this.apps[appName].js);
      }
+    });
  }

  resolveExcludes(apps) {
@ -508,15 +506,19 @@ class Wappalyzer {
  }

  /**
-   * Analyze JS variables
+   * Analyze JavaScript variables
   */
-  analyzeJS(app, js) {
-    Object.keys(js).forEach(property => {
-      var content = js[property]["content"];
-      delete js[property]["content"];
-      js[property].forEach(pattern => {
-          if ( pattern.regex.test(content) ) {
-            this.addDetected(app, pattern, 'js', content, property);
+  analyzeJs(app, results) {
+
+    console.log(app, results);
+
+    Object.keys(results).forEach(string => {
+      Object.keys(results[string]).forEach(index => {
+        const pattern = this.jsPatterns[app.name][string][index];
+        const value = results[string][index];
+
+        if ( pattern.regex.test(value) ) {
+          this.addDetected(app, pattern, 'js', value);
        }
      });
    });