From 62f881127c2cac5cb0d4a25fdfc09f0d2947a9e9 Mon Sep 17 00:00:00 2001 From: Elbert Alias <77259+AliasIO@users.noreply.github.com> Date: Thu, 20 Feb 2020 10:16:33 +1100 Subject: [PATCH] Fix sourceMap error, fix apps.json issues --- bin/build | 2 + bin/validate-regex | 182 ++++++++++++------------- src/apps.json | 24 +--- src/drivers/npm/npm-shrinkwrap.json | 2 +- src/drivers/npm/package.json | 2 +- src/drivers/webextension/manifest.json | 2 +- src/icons/pytonik.png | Bin 5674 -> 0 bytes 7 files changed, 100 insertions(+), 114 deletions(-) delete mode 100644 src/icons/pytonik.png diff --git a/bin/build b/bin/build index e39752ccc..d856bb995 100755 --- a/bin/build +++ b/bin/build @@ -41,12 +41,14 @@ set -e sed -i "s/\"version\": \"[^\"]*\"/\"version\": \"$version\"/" src/drivers/npm/package.json # WebExtension + echo "Building WebExtension..." webextension_dir=src/drivers/webextension pushd $webextension_dir > /dev/null +sed -i "/sourceMappingURL/d" node_modules/webextension-polyfill/dist/browser-polyfill.js sed -i "s/\"version\": \"[^\"]*\"/\"version\": \"$version\"/" manifest.json zip -qr ../../../build/wappalyzer_webextension.zip . \ diff --git a/bin/validate-regex b/bin/validate-regex index 380ffdb15..f331a8945 100755 --- a/bin/validate-regex +++ b/bin/validate-regex @@ -1,97 +1,91 @@ #!/usr/bin/env node -var - app, - json = require('../src/apps.json'); - -for ( app in json.apps ) { - ['headers', 'html', 'env', 'meta', 'script'].forEach(function(type) { - var - key, - patterns = json.apps[app][type]; - - if ( patterns !== undefined ) { - patterns = typeof patterns === 'string' ? [patterns] : patterns; - - if ( !( patterns instanceof Array ) ) { - patterns = []; - - for ( key in json.apps[app][type] ) { - patterns.push(json.apps[app][type][key]); - } - } - - patterns.forEach(function(pattern) { - var - attrs = pattern.split('\\;'), - regex = '/' + attrs.shift().replace('/', '\/') + '/', - version = attrs.find( function (attr){ - return attr.indexOf('version:')===0; - } ), - amountOfCaptureGroups, - captureGroupsNeeded, - amountOfCaptureGroupsNeeded, - highestBackReference; - - //Check if the pattern is a valid RegExp - //Note: unlike when used in Wappalyzer, the modifier i isn't added here - new RegExp(regex); - - //Actual amount of capture groups - amountOfCaptureGroups = ''.match(new RegExp('(?:'+pattern+')?')).length - 1; - - //Capture groups of which the result is used - captureGroupsNeeded = version ? (version.match(/\\\d/g) || []).filter( function(value, index, self){ - return self.indexOf(value) === index; - }).map( function(value){//Because Math.max needs the integers only - return parseInt(value.charAt(1), 10);//Will only work if backreferences cannot be any longer than a single digit - }) : []; - - //Amount of capture groups needed - amountOfCaptureGroupsNeeded = captureGroupsNeeded.length; - - //Highest back reference number used - highestBackReference = Math.max.apply(null, captureGroupsNeeded); - - //Report error - if(amountOfCaptureGroups > amountOfCaptureGroupsNeeded) { - throw new Error('The pattern uses more capture groups than needed. Use non-capturing groups where appropriate.\n' + app + ': ' + type + ': ' + pattern); - }else if(amountOfCaptureGroups < amountOfCaptureGroupsNeeded){ - throw new Error('The version string references more capture groups than there are in the pattern! Remove any incorrect back references from the version string and/or add the missing capture groups to the pattern.\n' + app + ': ' + type + ': ' + pattern); - } else if(highestBackReference > amountOfCaptureGroups){ - throw new Error('The version string references one or more capture groups whose index is higher than the amount of capture groups in the pattern. Please use the correct index instead of \\\\'+highestBackReference+'.\n' + app + ': ' + type + ': ' + pattern); - } - - if ( /^\/(?:\^\$|\.\+|\.\*)\/$/.test(regex) ) { - throw new Error('Pattern should be replaced with empty string.\n' + app + ': ' + type + ': ' + pattern); - } - - if ( type === 'html' ) { - if ( /\.(?:\+|\*)/.test(regex) ) { - throw new Error('Avoid ".+" and ".*" in HTML patterns. Consider using "[^>]+" or "[^<]+" instead.\n' + app + ': ' + type + ': ' + pattern); - } - - if ( !/[<>]/.test(regex) ) { - throw new Error('HTML patterns must contain "<" or ">".\n' + app + ': ' + type + ': ' + pattern); - } - } - - //Warn about suspicious periods (".") in patterns which should probably have been escaped - // Periods inside character classes (such as [\d.]) don't count as wildcard, so we'll replace - // the character classes in the pattern with "_". (We could remove them entirely, but then - // we'd have to deal with leftover * and + characters; for example, removing the character - // class entirely from the pattern /test.[a-z]+/ would yield the pattern /test.+/, which - // would mean not detecting the un-escaped ".". Replacing the character class with an - // underscore instead gives /test._+/, which WOULD yield a warning about the unescaped ".".) - if ( /(?:^\/|[^\\])\.(?:[^*+]|\/$)/.test(regex.replace(/([^\\]|^)\[[^\]]+\]/g,'$1_') ) ) { - console.warn('Suspicious period (".") in pattern. Should this have been escaped?\n\tApp: ' + app + '\n\tPattern: ' + type + ': ' + pattern); - } - - }); - } - }); - - if ( !/^https?:\/\//i.test(json.apps[app].website) ) { - throw new Error('Invalid website URL\n' + app + ': ' + json.apps[app].website); - } +let + app, + json = require('../src/apps.json'); + +for (app in json.apps) { + ['headers', 'html', 'env', 'meta', 'script'].forEach((type) => { + let + key, + patterns = json.apps[app][type]; + + if (patterns !== undefined) { + patterns = typeof patterns === 'string' ? [patterns] : patterns; + + if (!(patterns instanceof Array)) { + patterns = []; + + for (key in json.apps[app][type]) { + patterns.push(json.apps[app][type][key]); + } + } + + patterns.forEach((pattern) => { + let + attrs = pattern.split('\\;'), + regex = `/${attrs.shift().replace('/', '\/')}/`, + version = attrs.find(attr => attr.indexOf('version:') === 0), + amountOfCaptureGroups, + captureGroupsNeeded, + amountOfCaptureGroupsNeeded, + highestBackReference; + + // Check if the pattern is a valid RegExp + // Note: unlike when used in Wappalyzer, the modifier i isn't added here + new RegExp(regex); + + // Actual amount of capture groups + amountOfCaptureGroups = ''.match(new RegExp(`(?:${pattern})?`)).length - 1; + + // Capture groups of which the result is used + captureGroupsNeeded = version ? (version.match(/\\\d/g) || []).filter((value, index, self) => self.indexOf(value) === index).map(value => parseInt(value.charAt(1), 10), // Will only work if backreferences cannot be any longer than a single digit + ) : []; + + // Amount of capture groups needed + amountOfCaptureGroupsNeeded = captureGroupsNeeded.length; + + // Highest back reference number used + highestBackReference = Math.max.apply(null, captureGroupsNeeded); + + // Report error + if (amountOfCaptureGroups > amountOfCaptureGroupsNeeded) { + throw new Error(`The pattern uses more capture groups than needed. Use non-capturing groups where appropriate.\n${app}: ${type}: ${pattern}`); + } else if (amountOfCaptureGroups < amountOfCaptureGroupsNeeded) { + throw new Error(`The version string references more capture groups than there are in the pattern! Remove any incorrect back references from the version string and/or add the missing capture groups to the pattern.\n${app}: ${type}: ${pattern}`); + } else if (highestBackReference > amountOfCaptureGroups) { + throw new Error(`The version string references one or more capture groups whose index is higher than the amount of capture groups in the pattern. Please use the correct index instead of \\\\${highestBackReference}.\n${app}: ${type}: ${pattern}`); + } + + if (/^\/(?:\^\$|\.\+|\.\*)\/$/.test(regex)) { + throw new Error(`Pattern should be replaced with empty string.\n${app}: ${type}: ${pattern}`); + } + + if (type === 'html') { + if (/\.(?:\+|\*)/.test(regex)) { + throw new Error(`Avoid ".+" and ".*" in HTML patterns. Consider using "[^>]+" or "[^<]+" instead.\n${app}: ${type}: ${pattern}`); + } + + if (!/[<>]/.test(regex)) { + throw new Error(`HTML patterns must contain "<" or ">".\n${app}: ${type}: ${pattern}`); + } + } + + // Warn about suspicious periods (".") in patterns which should probably have been escaped + // Periods inside character classes (such as [\d.]) don't count as wildcard, so we'll replace + // the character classes in the pattern with "_". (We could remove them entirely, but then + // we'd have to deal with leftover * and + characters; for example, removing the character + // class entirely from the pattern /test.[a-z]+/ would yield the pattern /test.+/, which + // would mean not detecting the un-escaped ".". Replacing the character class with an + // underscore instead gives /test._+/, which WOULD yield a warning about the unescaped ".".) + if (/(?:^\/|[^\\])\.(?:[^*+]|\/$)/.test(regex.replace(/([^\\]|^)\[[^\]]+\]/g, '$1_'))) { + console.warn(`Suspicious period (".") in pattern. Should this have been escaped?\n\tApp: ${app}\n\tPattern: ${type}: ${pattern}`); + } + }); + } + }); + + if (!/^https?:\/\//i.test(json.apps[app].website)) { + throw new Error(`Invalid website URL\n${app}: ${json.apps[app].website}`); + } } diff --git a/src/apps.json b/src/apps.json index 42e45f7f6..9f3d156a1 100644 --- a/src/apps.json +++ b/src/apps.json @@ -1180,7 +1180,7 @@ 62 ], "headers": { - "X-Hacker": "(automattic.com/jobs|wpvip.com/careers)" + "X-Hacker": "(?:automattic.com/jobs|wpvip.com/careers)" }, "icon": "automattic.png", "implies": "WordPress", @@ -4055,7 +4055,7 @@ ], "icon": "GeneXus.png", "js": { - "gx":"", + "gx": "", "gx.gxVersion": "^(.+)-.*$\\;version:\\1" }, "script": [ @@ -4173,7 +4173,10 @@ "
" ], "icon": "GitLab.svg", - "implies": ["Ruby on Rails", "Vue.js"], + "implies": [ + "Ruby on Rails", + "Vue.js" + ], "js": { "GitLab": "", "gl.dashboardOptions": "" @@ -14006,16 +14009,6 @@ "script": "/media/conv/js/jquery\\.js", "website": "https://uknowva.com" }, - "Pytonik": { - "cats": [ - 18, - 22 - ], - "html": "Pytonik ([0-9.]+);version:\\1", - "icon": "pytonik.png", - "implies": "Python", - "website": "https://pytonik.com" - }, "vBulletin": { "cats": [ 2 @@ -14050,7 +14043,7 @@ }, "website": "http://vibecommerce.com.br" }, - "Vikaon": { + "Vikaon": { "cats": [ 6 ], @@ -14318,9 +14311,6 @@ "meta": { "generator": "Varbase" }, - "html": [ - "varbase_" - ], "cpe": "cpe:/a:vardot:varbase", "icon": "varbase.png", "implies": "Drupal", diff --git a/src/drivers/npm/npm-shrinkwrap.json b/src/drivers/npm/npm-shrinkwrap.json index ef47e40cd..f4bde316a 100644 --- a/src/drivers/npm/npm-shrinkwrap.json +++ b/src/drivers/npm/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "wappalyzer", - "version": "5.9.1", + "version": "5.9.18", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/src/drivers/npm/package.json b/src/drivers/npm/package.json index d10044af2..371ee27da 100644 --- a/src/drivers/npm/package.json +++ b/src/drivers/npm/package.json @@ -2,7 +2,7 @@ "name": "wappalyzer", "description": "Uncovers the technologies used on websites", "homepage": "https://github.com/AliasIO/Wappalyzer", - "version": "5.9.18", + "version": "5.9.4", "author": "Elbert Alias", "license": "GPL-3.0", "repository": { diff --git a/src/drivers/webextension/manifest.json b/src/drivers/webextension/manifest.json index ba422fd0f..f1e4c3f7d 100644 --- a/src/drivers/webextension/manifest.json +++ b/src/drivers/webextension/manifest.json @@ -4,7 +4,7 @@ "author": "Elbert Alias", "homepage_url": "https://www.wappalyzer.com", "description": "Identify web technologies", - "version": "5.9.3", + "version": "5.9.4", "default_locale": "en", "manifest_version": 2, "icons": { diff --git a/src/icons/pytonik.png b/src/icons/pytonik.png deleted file mode 100644 index 1bb6c9e02643fc5a75bb2109ee1e956928df2d14..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5674 zcmZ`-cQ{xpL0A=XdXYp1YrC?X}js-tT?C@7wFI9jT>pkBpd+7ytk=6=emT>lyiHfC;YO zFYR3Bt|yRY0$4FKrT zS&iD#*9Syszs~m_xiA9h}@Gy=2(`Vn|-E|G;o|$X^tUy$rj7x)wy<+0_an0uzQI*ky?!5Qwy^ zrM0Avg3>?X*E<<@TMPy*35R=ndcr(~V9u^KZ~+Mk2{=L!E-1)<&ER+QcEXr@@jJP3 z{4Mg|Ito^97Or+^jGeO+x^-} z?wYHeg^Cl#%JtgppJ>5<-2Q*YKU&i8KZXB~M*Q97ztHP8%Mwe&|NUgL#JH2#>l;Gd zs-hsL>xDCDM1s*9PoMM2b=w&n#9O99E|=r;8FU2|(X_!j3^TrIJpgUGQ zaj!I5rrvJalT9Usm_K|aNIS(wOM_$Ny}T-bTLVaC7uk=%z@KSuKtUe|Th_8#)_xy1 za7WT*PQ;%47?iNNdwFqnb!qzhF5!?40o5DyqL+!`8Frdwr6V*m)}j|i{Ivhm%Lm`S zE$P|W#Xi~MJ@@s>xW^u&rysw@%6gx}H)|2y6{D1k>p$mI6~82D!jTwZouHhSXeubd z?Yte8)m&iJ&|u%1KwPsraQX<{5u)7IJMJ&jTG^yOHyi1A5OEqo8g8oYHa^bk($kBx zfy1R+m~`&Zt*eZ1QE=f*c_Kd|mf-f4(1@X32SkOqcfdd#03n z0L}?uBK_6xIxFs^;gyH=v}8a~nfYDLqf^*j8s*3)YjKxJj>8{Z+V zMXHAThRk`x-Ut7rW8jRFSetal52fU?7%lJFKIZSg^qUcJqD{)yLN#a8ZHckXxw;5} zE+|6ik)>3D}q8msVj z7|Po@(*kK`NU`Qd;C_^pnnI4;yo)Bs7%LxNYU_0Cm6n&5Hty;?PN{{FAx!m)Y5;`W zc%Nr6jQ21ioW%|R#h)6>;dx|O+r}+*ISgBxbEi|{tV+JA)Oo!|=Cnn2^vmuTIZht*R!jL`ro6`lf=Bpo@g8U%C{6$mkP|?ZvDY$tlf<>M5 zTXC=!koqa!8R6ZbE0_7 z+lP2=v1~R?c8{QMEScYetFD-c7`xwB?2dAj?UOjQZzX_hVk!C7`hl8+MnpL3DTqOqIkxI)k-%{3 z+mBQ`inj05eTjv!=OlI&nzfmSBxIv5T>5%49yKzKA%#N2imZ9fy7WKkK~6Y&FObWh zAOHR&2wM?C1WQsHYW`G%q9 zNaK=C7?8eGoc@E_@g^T`J!_~LE8CISASUGgU~@hGOfp;<`I`VWswhlBPDMAy2YI?l z*|;jrJ{e`JyRwJhN;z|Kk21|D(X{nw44xZ9aZ40VX)l^f0a|%&wim`*_2t}9u}&-e zB@y*bqu$HITvgbW-EvujU&HsSJERtf<*|86Ezmaj_NE zHG2zvQ_SJv@y*8!`{8u9)8Y!ieO!p8Difs^DVST>scY|>2)D*onbN-1__!mt9ERA} zeG~n<+C<2jCYF|~nk^fI|N4OkL);FN?gn4ZZpfgl8DlqDr}geW)#di2g^Sa7JkLr$Qn-Icu2Us{9+*V^EoGx+CBUpL&Wb1 zh5!oxrvAbs3LQi8EU&GZ!16DQ2wSZ{ZKEc+xJK_Cd97ZbB3Dbuq-WiaF0!q4BglSp zukRFEzsHu)X~{UY&D#tB)dCXjYaaz8#EFyFOW*c^i~6CM(!&9Z8TQ!QeMS0jq{(<3lD}fSna3w02iQ-!xzdlYQ##%_(6GIdNC! zl8YJwl@6~2et(2H+y+5{$G-g1dm)?WW^%`U+?@&}-I}lcPEum1PWR1ftnClbSv>hh z3ebEbmAu)eU7p$7WfaI`&k3=!B3<|-&K7DMRPs4EX2eiDYIXrmm@6HzMKH|vyidEb zj;JSat-2&%?p?Ne)f?cXe?3R|h99GP%`P>(GTQe$cvg-!7p$XhX_wU!tE^OMcxC&J z>Gx)CI=Z)mpcOkg69Qf5@-x$RK!6I$L)y-`v)ATLqI#0^Qa0uBcD_GC_`?Y@^HpEE6ZJ5`g+hFN?^#wG|{y!9}ZDxI>x*xb2TpX*UZ+ z)L1V+67meF%WHgXep}OwF6NRdxbjy^#u$YMG;#t{_%&i3aR>cN5H5my_yJfqsx-Xx zB797D)0!zzLJ_gKLPmT^|YV$CMl277_1#?&`;M3$|pQSSm6F!WwnI zCfEe{ZMMA5rRqME6cX+dH1Yz1p%L2^O3&qH(;3=fG2A@evFBC}<=Lhw~%N&yjG2FwADqU+IKyI%i z60#V!PT2VvL)93nIerF^IQ_htf+RPq(%!jQ`Vo6UzxZPiC4u2M)tHj&&(Yv| zsa!WP)`9Mi<{dX^ObUoYCfsPP@TY0ScJL28@vhj7k9NX4%kwiMR3m|c5$RY$P!%2r zN8l0J0t=D^+3jCx>gg88{D59t6Doqt=)cGLN@=g?oGyY7Z$&s`6xBx?@+O(3NH=)V z=o4hVB&2=5SQHH5Y|?zM!;VG2H>S3n_9aA5p@@r|nyr;iY-$tAw^1mJq?r|d9`h?| z{@7RTv$lwoLj2$}+#+ELM;nKxQ^hnnf?_JKrCOrinMKNl?2}oFK<24%&nQZE+0w`a zY89d)1puRjWsW$WGxA{3Wky|G>FuEh7L;JxL|@9(pc&+X6FhJ@+#uL^Eu3eg)Se8f zNO+<55nKwX^(m$(Py# z4U$Dq;wQ7H`XLTw@){<88?dt%G0zkluQ6orFF^P1?I`e6mcA)5KL>AK{gT?Phzah; z9|l`X>B}nYH@`T$mPG)kcnb3|;y-k^I$ zR@qT<@-VHf-T+Hk{bt#sptvur8^duMUvfXAY{=irL&9(Kv0PBkdL9qUyH(@!QUjr! zl1ZD{(C2jLtqWUme2a0fDvS+1SEAm|asaHPUcFzHrXC|nKJbj3IwiH;4r_efixAum z6B^(W3ZyvUKjQq%%KIRoBOQf}IcZmT4FK+W=bvf@|8e-JT0be2It{G3Z(EhAavcS|_ zc6Yj}vn0-0&dO*vJxv$Z=?!*CTe3736DfQys`_ovcgQ~8lutMLrRF1Zk-^*ik0<%c z9!kqJtNc+CN1ItuZ#6M}n|-Db@*(xP=;-oSm#xdfF6ap9Dtv?%m-u^h!E_n(QK9F8 zfagzu6-H`*r?{7L#Mzc9TiWPtIHL}TmkybE6Wght=fZAcBGx#Nk&-=o1x_PRkGb+y z@^WNgXRweO)LR&C*SH!sBOuovr~ps2VyiXihJP}!>Kki4#F{jt0Fx@Yh!h{kzeV?Z5k{!Irw!D9l zzbcnS>baEp=1vw}Zt#Ir{H65?Hd>rHN9q0feUtYmdf;Nv5$XxX$ZDnLcu~J>VWTAL z0NP4ILY-N*&4m-?)N3=ehog(Sg(h z_sVkKP{wzTU4l7P9iT?&`cLtM}m62r(WTyQAsdn8_!r6T)54 za;Cq-$C#dcu&;n>L@#z|T5(i$Bdf#G?D^NfN zIVKv(SWKUB=)PtX#6$VB zT%2|4biyQh{g6TB;$7Y)y%WokNK?Is)Ewl?o82>^LlG6>MjBJy?}uN|m*z}S zRf62+HWW8A?j(ILLNhNuNR;K2j1s#7(Z04G!>teRT$>M9X*)2za;Tob&ujhp;wc%; zg{C-2Ixv7+;R;QRzDVchd!_=gxHU9f2@sejRE5O7cpwl$ceqg7-mIw(H;Eh(-Q$om e>OX4z9pDzG^)U)>1^nlatcs$BLKVs^=)VA&5;B1R