Fix options, replace Promises with async/await, build v5.5.7

src/drivers/npm/npm-shrinkwrap.json

@@ -1,6 +1,6 @@
 {
   "name": "wappalyzer",
-  "version": "5.5.5",
+  "version": "5.5.6",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

src/drivers/npm/package.json

@@ -2,7 +2,7 @@
   "name": "wappalyzer",
   "description": "Uncovers the technologies used on websites",
   "homepage": "https://github.com/AliasIO/Wappalyzer",
-  "version": "5.5.6",
+  "version": "5.5.7",
   "author": "Elbert Alias",
   "license": "GPL-3.0",
   "repository": {

src/drivers/webextension/js/driver.js

@@ -3,20 +3,19 @@
  */
 
 /* eslint-env browser */
-/* global browser, chrome, fetch, Wappalyzer */
+/* global browser, fetch, Wappalyzer */
 
 /** global: browser */
-/** global: chrome */
 /** global: fetch */
 /** global: Wappalyzer */
 
 const wappalyzer = new Wappalyzer();
 
 const tabCache = {};
-let categoryOrder = [];
-const options = {};
 const robotsTxtQueue = {};
 
+let categoryOrder = [];
+
 browser.tabs.onRemoved.addListener((tabId) => {
   tabCache[tabId] = null;
 });
@@ -26,17 +25,19 @@ browser.tabs.onRemoved.addListener((tabId) => {
  */
 function getOption(name, defaultValue = null) {
   return new Promise(async (resolve) => {
-    let option = defaultValue;
+    let value = defaultValue;
 
     try {
-      option = await browser.storage.local.get(name);
+      const option = await browser.storage.local.get(name);
+
+      if (option[name]) {
+        value = option[name];
+      }
     } catch (error) {
       wappalyzer.log(error, 'driver', 'error');
     }
 
-    options[name] = option;
+    resolve(value);
-
-    resolve(option);
   });
 }
 
@@ -44,13 +45,15 @@ function getOption(name, defaultValue = null) {
  * Set a value in localStorage
  */
 function setOption(name, value) {
-  const option = {};
+  return new Promise(async (resolve) => {
-
+    try {
-  option[name] = value;
+      await browser.storage.local.set({ [name]: value });
-
+    } catch (error) {
-  browser.storage.local.set(option);
+      wappalyzer.log(error, 'driver', 'error');
+    }
 
-  options[name] = value;
+    resolve();
+  });
 }
 
 /**
@@ -66,84 +69,33 @@ function openTab(args) {
 /**
  * Make a POST request
  */
-function post(url, body) {
+async function post(url, body) {
-  fetch(url, {
+  try {
+    const response = await fetch(url, {
       method: 'POST',
       body: JSON.stringify(body),
-  })
-    .then(response => wappalyzer.log(`POST ${url}: ${response.status}`, 'driver'))
-    .catch(error => wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error'));
-}
-
-fetch('../apps.json')
-  .then(response => response.json())
-  .then((json) => {
-    wappalyzer.apps = json.apps;
-    wappalyzer.categories = json.categories;
-
-    wappalyzer.parseJsPatterns();
-
-    categoryOrder = Object.keys(wappalyzer.categories)
-      .map(categoryId => parseInt(categoryId, 10))
-      .sort((a, b) => wappalyzer.categories[a].priority - wappalyzer.categories[b].priority);
-  })
-  .catch(error => wappalyzer.log(`GET apps.json: ${error}`, 'driver', 'error'));
-
-// Version check
-const { version } = browser.runtime.getManifest();
-
-getOption('version')
-  .then((previousVersion) => {
-    if (previousVersion === null) {
-      openTab({
-        url: `${wappalyzer.config.websiteURL}installed`,
-      });
-    } else if (version !== previousVersion) {
-      getOption('upgradeMessage', true)
-        .then((upgradeMessage) => {
-          if (upgradeMessage) {
-            openTab({
-              url: `${wappalyzer.config.websiteURL}upgraded?v${version}`,
-              background: true,
     });
+
+    wappalyzer.log(`POST ${url}: ${response.status}`, 'driver');
+  } catch (error) {
+    wappalyzer.log(`POST ${url}: ${error}`, 'driver', 'error');
   }
-        });
 }
 
-    setOption('version', version);
-  });
-
-getOption('dynamicIcon', true);
-getOption('pinnedCategory');
-
-getOption('hostnameCache', {})
-  .then((hostnameCache) => {
-    wappalyzer.hostnameCache = hostnameCache;
-
-    return hostnameCache;
-  });
-
-// Run content script on all tabs
-browser.tabs.query({ url: ['http://*/*', 'https://*/*'] })
-  .then((tabs) => {
-    tabs.forEach((tab) => {
-      browser.tabs.executeScript(tab.id, {
-        file: '../js/content.js',
-      });
-    });
-  })
-  .catch(error => wappalyzer.log(error, 'driver', 'error'));
-
 // Capture response headers
-browser.webRequest.onCompleted.addListener((request) => {
+browser.webRequest.onCompleted.addListener(async (request) => {
   const headers = {};
 
   if (request.responseHeaders) {
     const url = wappalyzer.parseUrl(request.url);
 
-    browser.tabs.query({ url: [url.href] })
+    let tab;
-      .then((tabs) => {
+
-        const tab = tabs[0] || null;
+    try {
+      [tab] = await browser.tabs.query({ url: [url.href] });
+    } catch (error) {
+      wappalyzer.log(error, 'driver', 'error');
+    }
 
     if (tab) {
       request.responseHeaders.forEach((header) => {
@@ -158,19 +110,25 @@ browser.webRequest.onCompleted.addListener((request) => {
         wappalyzer.analyze(url, { headers }, { tab });
       }
     }
-      })
-      .catch(error => wappalyzer.log(error, 'driver', 'error'));
   }
 }, { urls: ['http://*/*', 'https://*/*'], types: ['main_frame'] }, ['responseHeaders']);
 
 // Listen for messages
-(chrome || browser).runtime.onMessage.addListener((message, sender, sendResponse) => {
+browser.runtime.onMessage.addListener(async (message, sender) => {
-  if (typeof message.id !== 'undefined') {
+  if (message.id === undefined) {
+    return Promise.resolve();
+  }
+
   if (message.id !== 'log') {
     wappalyzer.log(`Message${message.source ? ` from ${message.source}` : ''}: ${message.id}`, 'driver');
   }
 
+  const pinnedCategory = await getOption('pinnedCategory');
+
   const url = wappalyzer.parseUrl(sender.tab ? sender.tab.url : '');
+
+  const cookies = await browser.cookies.getAll({ domain: `.${url.hostname}` });
+
   let response;
 
   switch (message.id) {
@@ -179,14 +137,13 @@ browser.webRequest.onCompleted.addListener((request) => {
 
       break;
     case 'init':
-        browser.cookies.getAll({ domain: `.${url.hostname}` })
+      wappalyzer.analyze(url, { cookies }, { tab: sender.tab });
-          .then(cookies => wappalyzer.analyze(url, { cookies }, { tab: sender.tab }));
 
       break;
     case 'analyze':
       wappalyzer.analyze(url, message.subject, { tab: sender.tab });
 
-        setOption('hostnameCache', wappalyzer.hostnameCache);
+      await setOption('hostnameCache', wappalyzer.hostnameCache);
 
       break;
     case 'ad_log':
@@ -198,12 +155,12 @@ browser.webRequest.onCompleted.addListener((request) => {
         tabCache: tabCache[message.tab.id],
         apps: wappalyzer.apps,
         categories: wappalyzer.categories,
-          pinnedCategory: options.pinnedCategory,
+        pinnedCategory,
       };
 
       break;
     case 'set_option':
-        setOption(message.key, message.value);
+      await setOption(message.key, message.value);
 
       break;
     case 'get_js_patterns':
@@ -215,10 +172,7 @@ browser.webRequest.onCompleted.addListener((request) => {
     default:
   }
 
-    sendResponse(response);
+  return Promise.resolve(response);
-  }
-
-  return true;
 });
 
 wappalyzer.driver.document = document;
@@ -227,13 +181,15 @@ wappalyzer.driver.document = document;
  * Log messages to console
  */
 wappalyzer.driver.log = (message, source, type) => {
-  console.log(`[wappalyzer ${type}]`, `[${source}]`, message);
+  const log = ['warn', 'error'].indexOf(type) !== -1 ? type : 'log';
+
+  console[log](`[wappalyzer ${type}]`, `[${source}]`, message);
 };
 
 /**
  * Display apps
  */
-wappalyzer.driver.displayApps = (detected, meta, context) => {
+wappalyzer.driver.displayApps = async (detected, meta, context) => {
   const { tab } = context;
 
   if (tab === undefined) {
@@ -246,20 +202,19 @@ wappalyzer.driver.displayApps = (detected, meta, context) => {
 
   tabCache[tab.id].detected = detected;
 
+  const pinnedCategory = await getOption('pinnedCategory');
+  const dynamicIcon = await getOption('dynamicIcon', true);
+
   let found = false;
 
   // Find the main application to display
-  [options.pinnedCategory].concat(categoryOrder).forEach((match) => {
+  [pinnedCategory].concat(categoryOrder).forEach((match) => {
     Object.keys(detected).forEach((appName) => {
       const app = detected[appName];
 
       app.props.cats.forEach((category) => {
         if (category === match && !found) {
-          let icon = app.props.icon || 'default.svg';
+          let icon = app.props.icon && dynamicIcon ? app.props.icon : 'default.svg';
-
-          if (!options.dynamicIcon) {
-            icon = 'default.svg';
-          }
 
           if (/\.svg$/i.test(icon)) {
             icon = `converted/${icon.replace(/\.svg$/, '.png')}`;
@@ -280,61 +235,53 @@ wappalyzer.driver.displayApps = (detected, meta, context) => {
     });
   });
 
-  if (typeof chrome !== 'undefined') {
-    // Browser polyfill doesn't seem to work here
-    chrome.pageAction.show(tab.id);
-  } else {
   browser.pageAction.show(tab.id);
-  }
 };
 
 /**
  * Fetch and cache robots.txt for host
  */
-wappalyzer.driver.getRobotsTxt = (host, secure = false) => {
+wappalyzer.driver.getRobotsTxt = async (host, secure = false) => {
   if (robotsTxtQueue[host]) {
     return robotsTxtQueue[host];
   }
 
-  robotsTxtQueue[host] = new Promise((resolve) => {
+  const tracking = await getOption('tracking', true);
-    getOption('tracking', true)
+  const robotsTxtCache = await getOption('robotsTxtCache', {});
-      .then((tracking) => {
-        if (!tracking) {
-          resolve([]);
 
-          return;
+  robotsTxtQueue[host] = new Promise(async (resolve) => {
+    if (!tracking) {
+      return resolve([]);
     }
 
-        getOption('robotsTxtCache')
-          .then((robotsTxtCache) => {
-            robotsTxtCache = robotsTxtCache || {};
-
     if (host in robotsTxtCache) {
-              resolve(robotsTxtCache[host]);
+      return resolve(robotsTxtCache[host]);
-
-              return;
     }
 
     const timeout = setTimeout(() => resolve([]), 3000);
 
-            fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { redirect: 'follow' })
+    let response;
-              .then((response) => {
+
+    try {
+      response = await fetch(`http${secure ? 's' : ''}://${host}/robots.txt`, { redirect: 'follow' });
+    } catch (error) {
+      wappalyzer.log(error, 'driver', 'error');
+
+      return resolve([]);
+    }
+
     clearTimeout(timeout);
 
-                return response.ok ? response.text() : '';
+    const robotsTxt = response.ok ? await response.text() : '';
-              })
+
-              .then((robotsTxt) => {
     robotsTxtCache[host] = Wappalyzer.parseRobotsTxt(robotsTxt);
 
-                setOption('robotsTxtCache', robotsTxtCache);
+    await setOption('robotsTxtCache', robotsTxtCache);
 
-                resolve(robotsTxtCache[host]);
+    delete robotsTxtQueue[host];
-              })
+
-              .catch(() => resolve([]));
+    return resolve(robotsTxtCache[host]);
-          });
   });
-  })
-    .finally(() => delete robotsTxtQueue[host]);
 
   return robotsTxtQueue[host];
 };
@@ -342,9 +289,9 @@ wappalyzer.driver.getRobotsTxt = (host, secure = false) => {
 /**
  * Anonymously track detected applications for research purposes
  */
-wappalyzer.driver.ping = (hostnameCache = {}, adCache = []) => {
+wappalyzer.driver.ping = async (hostnameCache = {}, adCache = []) => {
-  getOption('tracking', true)
+  const tracking = await getOption('tracking', true);
-    .then((tracking) => {
+
   if (tracking) {
     if (Object.keys(hostnameCache).length) {
       post('https://api.wappalyzer.com/ping/v1/', hostnameCache);
@@ -354,7 +301,60 @@ wappalyzer.driver.ping = (hostnameCache = {}, adCache = []) => {
       post('https://ad.wappalyzer.com/log/wp/', adCache);
     }
 
-        setOption('robotsTxtCache', {});
+    await setOption('robotsTxtCache', {});
   }
-    });
 };
+
+// Init
+(async () => {
+  // Technologies
+  try {
+    const response = await fetch('../apps.json');
+    const json = await response.json();
+
+    wappalyzer.apps = json.apps;
+    wappalyzer.categories = json.categories;
+  } catch (error) {
+    wappalyzer.log(`GET apps.json: ${error.message}`, 'driver', 'error');
+  }
+
+  wappalyzer.parseJsPatterns();
+
+  categoryOrder = Object.keys(wappalyzer.categories)
+    .map(categoryId => parseInt(categoryId, 10))
+    .sort((a, b) => wappalyzer.categories[a].priority - wappalyzer.categories[b].priority);
+
+  // Version check
+  const { version } = browser.runtime.getManifest();
+  const previousVersion = await getOption('version');
+  const upgradeMessage = await getOption('upgradeMessage', true);
+
+  if (previousVersion === null) {
+    openTab({
+      url: `${wappalyzer.config.websiteURL}installed`,
+    });
+  } else if (version !== previousVersion && upgradeMessage) {
+    //openTab({
+    //  url: `${wappalyzer.config.websiteURL}upgraded?v${version}`,
+    //  background: true,
+    //});
+  }
+
+  await setOption('version', version);
+
+  // Hostname cache
+  wappalyzer.hostnameCache = await getOption('hostnameCache', {});
+
+  // Run content script on all tabs
+  try {
+    const tabs = await browser.tabs.query({ url: ['http://*/*', 'https://*/*'] });
+
+    tabs.forEach((tab) => {
+      browser.tabs.executeScript(tab.id, {
+        file: '../js/content.js',
+      });
+    });
+  } catch (error) {
+    wappalyzer.log(error, 'driver', 'error');
+  }
+})();

src/drivers/webextension/js/popup.js

@@ -1,32 +1,14 @@
+/* eslint-env browser */
+/* global browser, chrome, jsonToDOM */
+
 /** global: chrome */
 /** global: browser */
+/** global: jsonToDOM */
 
 let pinnedCategory = null;
 
-const func = (tabs) => {
+function slugify(string) {
-  (chrome || browser).runtime.sendMessage({
+  return string.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/--+/g, '-').replace(/(?:^-|-$)/, '');
-    id: 'get_apps',
-    tab: tabs[0],
-    source: 'popup.js',
-  }, (response) => {
-    pinnedCategory = response.pinnedCategory;
-
-    replaceDomWhenReady(appsToDomTemplate(response));
-  });
-};
-
-browser.tabs.query({ active: true, currentWindow: true })
-  .then(func)
-  .catch(console.error);
-
-function replaceDomWhenReady(dom) {
-  if (/complete|interactive|loaded/.test(document.readyState)) {
-    replaceDom(dom);
-  } else {
-    document.addEventListener('DOMContentLoaded', () => {
-      replaceDom(dom);
-    });
-  }
 }
 
 function replaceDom(domTemplate) {
@@ -73,6 +55,16 @@ function replaceDom(domTemplate) {
   });
 }
 
+function replaceDomWhenReady(dom) {
+  if (/complete|interactive|loaded/.test(document.readyState)) {
+    replaceDom(dom);
+  } else {
+    document.addEventListener('DOMContentLoaded', () => {
+      replaceDom(dom);
+    });
+  }
+}
+
 function appsToDomTemplate(response) {
   let template = [];
 
@@ -92,8 +84,7 @@ function appsToDomTemplate(response) {
       const apps = [];
 
       for (const appName in categories[cat].apps) {
-        const confidence = response.tabCache.detected[appName].confidenceTotal;
+        const { confidence, version } = response.tabCache.detected[appName];
-        const version = response.tabCache.detected[appName].version;
 
         apps.push(
           [
@@ -190,6 +181,18 @@ function appsToDomTemplate(response) {
   return template;
 }
 
-function slugify(string) {
+const func = (tabs) => {
-  return string.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/--+/g, '-').replace(/(?:^-|-$)/, '');
+  (chrome || browser).runtime.sendMessage({
-}
+    id: 'get_apps',
+    tab: tabs[0],
+    source: 'popup.js',
+  }, (response) => {
+    pinnedCategory = response.pinnedCategory;
+
+    replaceDomWhenReady(appsToDomTemplate(response));
+  });
+};
+
+browser.tabs.query({ active: true, currentWindow: true })
+  .then(func)
+  .catch(console.error);

src/drivers/webextension/manifest.json

@@ -4,7 +4,7 @@
   "author": "Elbert Alias",
   "homepage_url": "https://www.wappalyzer.com",
   "description": "Identify web technologies",
-	"version": "5.5.6",
+  "version": "5.5.7",
   "default_locale": "en",
   "manifest_version": 2,
   "icons": {
@@ -60,7 +60,6 @@
   "web_accessible_resources": [
     "js/inject.js"
   ],
-	"options_page": "html/options.html",
   "options_ui": {
     "page": "html/options.html",
     "open_in_tab": false
@@ -74,5 +73,11 @@
     "http://*/*",
     "https://*/*"
   ],
-	"content_security_policy": "script-src 'self'; object-src 'self'"
+  "content_security_policy": "script-src 'self'; object-src 'self'",
+  "applications": {
+    "gecko": {
+      "id": "wappalyzer@crunchlabz.com",
+      "strict_min_version": "42.0"
+    }
+  }
 }

src/drivers/webextension/npm-shrinkwrap.json

@@ -3,9 +3,9 @@
   "lockfileVersion": 1,
   "dependencies": {
     "webextension-polyfill": {
-      "version": "0.2.1",
+      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/webextension-polyfill/-/webextension-polyfill-0.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/webextension-polyfill/-/webextension-polyfill-0.3.1.tgz",
-      "integrity": "sha1-zfyRJgMwOfFxNVMVfTW+/x1Nb0o="
+      "integrity": "sha512-ISB42vlgMyM7xE1u6pREeCqmmXjLsYu/nqAR8Dl/gIAnylb+KpRpvKbVkUYNFePhhXn0Obkkc3jasOII9ztUtg=="
     }
   }
 }

src/drivers/webextension/package.json

@@ -1,5 +1,5 @@
 {
   "dependencies": {
-    "webextension-polyfill": "^0.2.1"
+    "webextension-polyfill": "^0.3.1"
   }
 }