From fd619684fde75e65d53202c9ac7330118b8171d8 Mon Sep 17 00:00:00 2001
From: Elbert Alias <77259+AliasIO@users.noreply.github.com>
Date: Wed, 19 May 2021 13:57:18 +1000
Subject: [PATCH] Optimise image conversion script, minor fixes
---
bin/convert.js | 123 ++++++----
src/drivers/npm/package.json | 2 +-
src/drivers/webextension/css/styles.css | 2 +-
.../images/icons/Elasticsearch.svg | 9 +
.../webextension/images/icons/pico.svg | 4 +-
src/drivers/webextension/js/content.js | 11 +-
src/drivers/webextension/manifest-safari.json | 4 +-
src/drivers/webextension/manifest.json | 2 +-
src/package.json | 2 +-
src/technologies.json | 222 ++++++++++--------
src/wappalyzer.js | 6 +-
11 files changed, 224 insertions(+), 163 deletions(-)
diff --git a/bin/convert.js b/bin/convert.js
index 735630988..6e2cb0b58 100644
--- a/bin/convert.js
+++ b/bin/convert.js
@@ -1,6 +1,6 @@
const fs = require('fs')
const path = require('path')
-const { convertFile } = require('convert-svg-to-png')
+const { createConverter } = require('convert-svg-to-png')
const appPaths = () => {
const fileDir = path.dirname(require.main.filename).split('/')
@@ -77,52 +77,85 @@ function dateDiff(file) {
return Math.round(Math.abs((then - now) / 86400000))
}
-// Main script
-fs.readdirSync(appPaths().iconPath).forEach((fileName) => {
- const image = {
- id: fileName,
- path: `${appPaths().iconPath}/${fileName}`,
- convertPath: `${appPaths().convertPath}/${fileName}`,
- async convertAndCopy() {
- await convertFile(this.path, {
- height: 32,
- width: 32,
- outputFilePath: this.convertPath,
- }).then((outputFile) => {
- console.log(`SVG Converted: ${outputFile}`)
- })
- },
- processFile() {
- // Setup variables.
- const ext = getFileExtension(this.path)
-
- // If SVG, run checks.
- if (ext === '.svg') {
- // Check if converted file exists.
- const convertFilePath = getConvertFileName(this.path)
- if (checkFileExists(convertFilePath)) {
- // If file has changed in past 7 days.
- if (dateDiff(this.path) > 8) {
- console.log(`File exists, skipping: ${this.id}`)
- return null
- }
- }
- // Convert and copy file.
- this.convertAndCopy()
- } else {
- // If PNG or other, just copy the file as-is.
- // eslint-disable-next-line no-lonely-if
- if (checkIfFile(this.path)) {
- copyFiles(this.path, this.convertPath)
- } else {
- console.info('Not a file, skipping...')
+const converter = createConverter()
+
+;(async () => {
+ // Main script
+ const files = fs.readdirSync(appPaths().iconPath)
+
+ const totalFiles = files.length
+
+ do {
+ await Promise.all(
+ files.splice(0, 50).map(async (fileName) => {
+ const image = {
+ id: fileName,
+ path: `${appPaths().iconPath}/${fileName}`,
+ convertPath: `${appPaths().convertPath}/${fileName}`,
+ async convertAndCopy() {
+ for (let attempt = 1; attempt <= 3; attempt++) {
+ try {
+ await converter
+ .convertFile(this.path, {
+ height: 32,
+ width: 32,
+ outputFilePath: this.convertPath,
+ })
+ .catch((error) => {
+ throw new Error(`${error} (${fileName})`)
+ })
+ } catch (error) {
+ if (attempt >= 3) {
+ throw error
+ } else {
+ await new Promise((resolve) =>
+ setTimeout(resolve, 500 * attempt)
+ )
+ }
+ }
+
+ break
+ }
+ },
+ async processFile() {
+ // Setup variables.
+ const ext = getFileExtension(this.path)
+
+ // If SVG, run checks.
+ if (ext === '.svg') {
+ // Check if converted file exists.
+ const convertFilePath = getConvertFileName(this.path)
+ if (checkFileExists(convertFilePath)) {
+ // If file has changed in past 7 days.
+ if (dateDiff(this.path) > 8) {
+ return null
+ }
+ }
+ // Convert and copy file.
+ await this.convertAndCopy()
+ } else {
+ // If PNG or other, just copy the file as-is.
+ // eslint-disable-next-line no-lonely-if
+ if (checkIfFile(this.path)) {
+ copyFiles(this.path, this.convertPath)
+ }
+ }
+ },
}
- }
- },
- }
- image.processFile()
-})
+ await image.processFile()
+ })
+ )
+
+ console.log(`${100 - Math.round((100 / totalFiles) * files.length)}%`)
+ } while (files.length)
+
+ await converter.destroy()
+
+ console.log(`Converted ${totalFiles.toLocaleString()} files.`)
+
+ process.exit()
+})()
/**
diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json
index 7fe2d6b52..14e890817 100644
--- a/src/drivers/npm/package.json
+++ b/src/drivers/npm/package.json
@@ -13,7 +13,7 @@
"software"
],
"homepage": "https://www.wappalyzer.com/",
- "version": "6.6.0",
+ "version": "6.7.0",
"author": "Wappalyzer",
"license": "MIT",
"repository": {
diff --git a/src/drivers/webextension/css/styles.css b/src/drivers/webextension/css/styles.css
index 0d095f8fd..6d8b06fc4 100644
--- a/src/drivers/webextension/css/styles.css
+++ b/src/drivers/webextension/css/styles.css
@@ -23,7 +23,6 @@ body {
line-height: 1.5rem;
margin: 0;
min-width: 24rem;
- overflow-x: hidden;
}
a, a:focus, a:hover {
@@ -305,6 +304,7 @@ body.dynamic-icon .category__heading:hover .category__pin {
padding: .1rem .3rem;
margin-left: .3rem;
vertical-align: middle;
+ white-space: nowrap;
}
.technology__version {
diff --git a/src/drivers/webextension/images/icons/Elasticsearch.svg b/src/drivers/webextension/images/icons/Elasticsearch.svg
index e69de29bb..38262913e 100644
--- a/src/drivers/webextension/images/icons/Elasticsearch.svg
+++ b/src/drivers/webextension/images/icons/Elasticsearch.svg
@@ -0,0 +1,9 @@
+
diff --git a/src/drivers/webextension/images/icons/pico.svg b/src/drivers/webextension/images/icons/pico.svg
index b9e533d6f..21e8fa12a 100644
--- a/src/drivers/webextension/images/icons/pico.svg
+++ b/src/drivers/webextension/images/icons/pico.svg
@@ -1,3 +1,3 @@
-