Migrations
/
wappalyzer
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use langdetect instead of cld NPM module

Browse Source
main
Elbert Alias 5 years ago
parent 2e2f7246f4
commit f819c4dd09
3 changed files with 14 additions and 135 deletions
Show Stats Download Patch File Download Diff File

18
src/drivers/npm/driver.js
Unescape View File

@ -1,9 +1,13 @@
const url = require('url');
const fs = require('fs');
const path = require('path');
const cld = require('cld');
const LanguageDetect = require('languagedetect');
const Wappalyzer = require('./wappalyzer');
const languageDetect = new LanguageDetect();
languageDetect.setLanguageType('iso2');
const json = JSON.parse(fs.readFileSync(path.resolve(`${__dirname}/apps.json`)));
const extensions = /^([^.]+$|\.(asp|aspx|cgi|htm|html|jsp|php)$)/;
@ -233,17 +237,7 @@ class Driver {
let language = null;
try {
language = await new Promise((resolve, reject) => cld.detect(html, { isHTML: true }, (error, { languages }) => {
if (error) {
reject(error);
}
resolve(
languages
.filter(({ percent }) => percent >= 75)
.map(({ code }) => code)[0],
);
}));
[[language]] = languageDetect.detect(html.replace(/<\/?[^>]+(>|$)/g, ' '), 1);
} catch (error) {
this.wappalyzer.log(`${error.message || error}; url: ${pageUrl.href}`, 'driver', 'error');
}

127
src/drivers/npm/npm-shrinkwrap.json generated
Unescape View File

@ -1,6 +1,6 @@
{
"name": "wappalyzer",
"version": "5.9.32",
"version": "5.9.33",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -93,11 +93,6 @@
"regenerator-runtime": "^0.11.0"
}
},
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"bcrypt-pbkdf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
@ -111,15 +106,6 @@
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
"integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"browser-process-hrtime": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-0.1.3.tgz",
@ -130,31 +116,6 @@
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
},
"cld": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/cld/-/cld-2.6.0.tgz",
"integrity": "sha512-2U8Uiv7Bvl1v4fNWFGB3RYtPvhUWXQJ1MoNKJNVuoALfandEt9oVqK64S+3ZLvQPjDiYjsohtTep/wIs0xOXkw==",
"requires": {
"glob": "^5.0.10",
"node-addon-api": "*",
"rimraf": "^2.4.0",
"underscore": "^1.6.0"
},
"dependencies": {
"glob": {
"version": "5.0.15",
"resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz",
"integrity": "sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=",
"requires": {
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "2 || 3",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
}
}
},
"combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@ -163,11 +124,6 @@
"delayed-stream": "~1.0.0"
}
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"core-js": {
"version": "2.6.11",
"resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz",
@ -331,11 +287,6 @@
"mime-types": "^2.1.12"
}
},
"fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
"getpass": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
@ -344,19 +295,6 @@
"assert-plus": "^1.0.0"
}
},
"glob": {
"version": "7.1.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.0.4",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
},
"har-schema": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
@ -397,20 +335,6 @@
"safer-buffer": ">= 2.1.2 < 3"
}
},
"inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"requires": {
"once": "^1.3.0",
"wrappy": "1"
}
},
"inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
@ -495,6 +419,11 @@
"verror": "1.10.0"
}
},
"languagedetect": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/languagedetect/-/languagedetect-2.0.0.tgz",
"integrity": "sha512-AZb/liiQ+6ZoTj4f1J0aE6OkzhCo8fyH+tuSaPfSo8YHCWLFJrdSixhtO2TYdIkjcDQNaR4RmGaV2A5FJklDMQ=="
},
"left-pad": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz",
@ -537,24 +466,11 @@
"mime-db": "1.43.0"
}
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"requires": {
"brace-expansion": "^1.1.7"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node-addon-api": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.0.tgz",
"integrity": "sha512-ASCL5U13as7HhOExbT6OlWJJUV/lLzL2voOSP1UVehpRD8FbSrSDjfScK/KwAvVTI5AS6r4VwbOMlIqtvRidnA=="
},
"nwsapi": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz",
@ -565,14 +481,6 @@
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ=="
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"requires": {
"wrappy": "1"
}
},
"optionator": {
"version": "0.8.3",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",
@ -599,11 +507,6 @@
"resolved": "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz",
"integrity": "sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA=="
},
"path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
},
"performance-now": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
@ -694,14 +597,6 @@
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
"integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
},
"rimraf": {
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
"requires": {
"glob": "^7.1.3"
}
},
"safe-buffer": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz",
@ -787,11 +682,6 @@
"prelude-ls": "~1.1.2"
}
},
"underscore": {
"version": "1.10.2",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.10.2.tgz",
"integrity": "sha512-N4P+Q/BuyuEKFJ43B9gYuOj4TQUHXX+j2FqguVOpjkssLUUrnJofCcBccJSCoeturDoZU6GorDTHSvUDlSQbTg=="
},
"uri-js": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
@ -865,11 +755,6 @@
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
"integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ=="
},
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"ws": {
"version": "6.2.1",
"resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz",

4
src/drivers/npm/package.json
Unescape View File

@ -2,7 +2,7 @@
"name": "wappalyzer",
"description": "Identify technology on websites",
"homepage": "https://www.wappalyzer.com",
"version": "5.9.32",
"version": "5.9.33",
"author": "Wappalyzer",
"license": "MIT",
"repository": {
@ -27,7 +27,7 @@
"wappalyzer": "./cli.js"
},
"dependencies": {
"cld": "^2.6.0",
"languagedetect": "^2.0.0",
"zombie": "^6.1.4"
},
"peerDependencies": {