From 9c1e5ecd753ea184d67939f677b1c5f25b635389 Mon Sep 17 00:00:00 2001 From: jvoisin Date: Sun, 12 Nov 2017 00:05:27 +0100 Subject: [PATCH 1/4] Improve a bit some regexp (#1804) * Improve a bit some regexp - Prepend `^` when possible - Append `$` when possible Those changes have been tested on official websites of changed technologies. * Relax the Apache detection --- src/apps.json | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/apps.json b/src/apps.json index f1f55f0c9..7c448c279 100644 --- a/src/apps.json +++ b/src/apps.json @@ -14,8 +14,8 @@ "1" ], "headers": { - "Set-Cookie": "BITRIX_", - "X-Powered-CMS": "Bitrix Site Manager" + "Set-Cookie": "^BITRIX_", + "X-Powered-CMS": "^Bitrix Site Manager" }, "html": "(?:]+components/bitrix|(?:src|href)=\"/bitrix/(?:js|templates))", "icon": "1C-Bitrix.png", @@ -589,7 +589,7 @@ "22" ], "headers": { - "Server": "Apache-Coyote(/1\\.1)?\\;version:\\1?4.1+:", + "Server": "^Apache-Coyote(/1\\.1)?$\\;version:\\1?4.1+:", "X-Powered-By": "\bTomcat\b(?:-([\\d.]+))?\\;version:\\1" }, "icon": "Apache Tomcat.svg", @@ -1225,7 +1225,7 @@ "1" ], "headers": { - "X-Powered-By": "Dragonfly CMS" + "X-Powered-By": "^Dragonfly CMS" }, "icon": "CPG Dragonfly.png", "implies": "PHP", @@ -1776,7 +1776,7 @@ "1" ], "headers": { - "X-Powered-By": "CppCMS/([\\d.]+)\\;version:\\1" + "X-Powered-By": "^CppCMS/([\\d.]+)$\\;version:\\1" }, "icon": "CppCMS.png", "implies": "C\\+\\+", @@ -1787,8 +1787,8 @@ "1" ], "headers": { - "Set-Cookie": "CraftSessionId=", - "X-Powered-By": "Craft CMS" + "Set-Cookie": "^CraftSessionId=", + "X-Powered-By": "^Craft CMS$" }, "icon": "Craft CMS.svg", "implies": "Yii", @@ -2143,12 +2143,12 @@ "8" ], "headers": { - "Set-Cookie": "DokuWiki=" + "Set-Cookie": "^DokuWiki=" }, "icon": "DokuWiki.png", "implies": "PHP", "meta": { - "generator": "DokuWiki( Release [\\-\\d]+)?\\;version:\\1" + "generator": "^DokuWiki( Release [\\-\\d]+)?\\;version:\\1" }, "website": "http://www.dokuwiki.org" }, @@ -2648,7 +2648,7 @@ "18" ], "headers": { - "X-Powered-By": "Fat-Free Framework" + "X-Powered-By": "^Fat-Free Framework$" }, "icon": "Fat-Free Framework.png", "implies": "PHP", @@ -2722,7 +2722,7 @@ "html": "|
]*x:publishsource=\"?Excel\"?)", + "meta": { + "generator": "Microsoft Excel( [\\d.]+)?\\;version:\\1", + "ProgId": "^Excel\\." + }, + "website": "https://office.microsoft.com/excel" + }, "Microsoft SharePoint": { "cats": [ "1" @@ -5246,6 +5258,18 @@ }, "website": "http://sharepoint.microsoft.com" }, + "Microsoft Word": { + "cats": [ + "20" + ], + "icon": "Microsoft Word.svg", + "html": "(?:]*xmlns:w=\"urn:schemas-microsoft-com:office:word\"||
]*class=\"?WordSection1[\" >]|]*>[^>]*@page WordSection1)", + "meta": { + "generator": "Microsoft Word( [\\d.]+)?\\;version:\\1", + "ProgId": "^Word\\." + }, + "website": "https://office.microsoft.com/word" + }, "Mietshop": { "cats": [ "6" diff --git a/src/icons/Microsoft Excel.svg b/src/icons/Microsoft Excel.svg new file mode 100644 index 000000000..9c912bb4e --- /dev/null +++ b/src/icons/Microsoft Excel.svg @@ -0,0 +1,22 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/icons/Microsoft Word.svg b/src/icons/Microsoft Word.svg new file mode 100644 index 000000000..626051757 --- /dev/null +++ b/src/icons/Microsoft Word.svg @@ -0,0 +1,19 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + \ No newline at end of file From d66a2a7c757090356de9b50c650437611a26ac32 Mon Sep 17 00:00:00 2001 From: Camille Barneaud <1693643+gadcam@users.noreply.github.com> Date: Sun, 12 Nov 2017 00:07:23 +0100 Subject: [PATCH 3/4] Enable multiple headers with the same name & Get scripts via DOM (#1819) * Enable multiple header with same name Enable multiple header with same name for WebExtension driver * Enable multiple header with same name Enable multiple header with same name for Bookmarklet driver * Enable multiple header with same name Enable multiple header with same name for the NPM driver * Enable multiple headers with the same name * Get scripts via DOM (WebExtension) * Get scripts via DOM (Bookmarklet) * Get scripts via DOM * Get scripts via DOM (NPM) --- src/drivers/bookmarklet/driver.js | 19 +++++++++++---- src/drivers/npm/driver.js | 14 ++++++++--- src/drivers/webextension/js/content.js | 11 +++++++++ src/drivers/webextension/js/driver.js | 9 +++++--- src/wappalyzer.js | 32 +++++++++++++++----------- 5 files changed, 61 insertions(+), 24 deletions(-) diff --git a/src/drivers/bookmarklet/driver.js b/src/drivers/bookmarklet/driver.js index 5418ddbbc..1bcb3783c 100644 --- a/src/drivers/bookmarklet/driver.js +++ b/src/drivers/bookmarklet/driver.js @@ -18,8 +18,8 @@ console.log('[wappalyzer ' + type + ']', '[' + source + ']', message); }; - function getEnvironmentVars() { - wappalyzer.log('func: getEnvironmentVars'); + function getPageContent() { + wappalyzer.log('func: getPageContent'); var env = []; @@ -27,9 +27,15 @@ env.push(i); } + var scripts = Array.prototype.slice + .apply(document.scripts) + .filter(s => s.src) + .map(s => s.src); + wappalyzer.analyze(domain, url, { html: document.documentElement.innerHTML, - env: env + env: env, + scripts: scripts }); } @@ -56,7 +62,10 @@ name = line.substring(0, line.indexOf(': ')); value = line.substring(line.indexOf(': ') + 2, line.length - 1); - responseHeaders[name.toLowerCase()] = value; + if ( !responseHeaders[name.toLowerCase()] ){ + responseHeaders[name.toLowerCase()] = [] + } + responseHeaders[name.toLowerCase()].push(value); } }); @@ -139,6 +148,6 @@ return string.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/--+/g, '-').replace(/(?:^-|-$)/, ''); } - getEnvironmentVars(); + getPageContent(); getResponseHeaders(); })(); diff --git a/src/drivers/npm/driver.js b/src/drivers/npm/driver.js index c845a52a0..2c3913f75 100644 --- a/src/drivers/npm/driver.js +++ b/src/drivers/npm/driver.js @@ -72,18 +72,26 @@ const driver = options => { const headers = {}; browser.resources['0'].response.headers._headers.forEach(header => { - headers[header[0]] = header[1]; + if ( !headers[header[0]] ){ + headers[header[0]] = []; + } + headers[header[0]].push(header[1]); }); const vars = Object.getOwnPropertyNames(browser.window); const html = browser.html(); + const scripts = Array.prototype.slice + .apply(browser.document.scripts) + .filter(s => s.src) + .map(s => s.src); - const hostname = wappalyzer.parseUrl(url).hostname; + const hostname = wappalyzer.parseUrl(url).hostname; wappalyzer.analyze(hostname, url, { headers, html, - env: vars + env: vars, + scripts }); }); }); diff --git a/src/drivers/webextension/js/content.js b/src/drivers/webextension/js/content.js index c4e1940b0..7ad284495 100644 --- a/src/drivers/webextension/js/content.js +++ b/src/drivers/webextension/js/content.js @@ -7,6 +7,11 @@ if ( typeof browser !== 'undefined' && typeof document.body !== 'undefined' ) { html = html.substring(0, 25000) + html.substring(html.length - 25000, html.length); } + var scripts = Array.prototype.slice + .apply(document.scripts) + .filter(s => s.src) + .map(s => s.src); + try { browser.runtime.sendMessage({ id: 'analyze', @@ -14,6 +19,12 @@ if ( typeof browser !== 'undefined' && typeof document.body !== 'undefined' ) { source: 'content.js' }); + browser.runtime.sendMessage({ + id: 'analyze', + subject: { scripts }, + source: 'content.js' + }); + var container = document.createElement('wappalyzerData'); container.setAttribute('id', 'wappalyzerData'); diff --git a/src/drivers/webextension/js/driver.js b/src/drivers/webextension/js/driver.js index a0fac8fff..4b9a40dc6 100644 --- a/src/drivers/webextension/js/driver.js +++ b/src/drivers/webextension/js/driver.js @@ -138,20 +138,23 @@ browser.webRequest.onCompleted.addListener(request => { var url = wappalyzer.parseUrl(request.url); request.responseHeaders.forEach(function(header) { - responseHeaders[header.name.toLowerCase()] = header.value || '' + header.binaryValue; + if ( !responseHeaders[header.name.toLowerCase()] ) { + responseHeaders[header.name.toLowerCase()] = [] + } + responseHeaders[header.name.toLowerCase()].push(header.value || '' + header.binaryValue); }); if ( headersCache.length > 50 ) { headersCache = {}; } - if ( /text\/html/.test(responseHeaders['content-type']) ) { + if ( /text\/html/.test(responseHeaders['content-type'][0]) ) { if ( headersCache[url.canonical] === undefined ) { headersCache[url.canonical] = {}; } Object.keys(responseHeaders).forEach(header => { - headersCache[url.canonical][header] = responseHeaders[header]; + headersCache[url.canonical][header] = responseHeaders[header].slice(); }); } } diff --git a/src/wappalyzer.js b/src/wappalyzer.js index 6fed51ec9..39908caee 100644 --- a/src/wappalyzer.js +++ b/src/wappalyzer.js @@ -62,10 +62,13 @@ class Wappalyzer { if ( data.html ) { this.analyzeHtml(app, data.html); - this.analyzeScript(app, data.html); this.analyzeMeta(app, data.html); } + if ( data.scripts ) { + this.analyzeScripts(app, data.scripts); + } + if ( data.headers ) { this.analyzeHeaders(app, data.headers); } @@ -396,19 +399,18 @@ class Wappalyzer { /** * Analyze script tag */ - analyzeScript(app, html) { - var regex = new RegExp(']+src=("|\')([^"\']+)', 'ig'); + analyzeScripts(app, scripts) { var patterns = this.parsePatterns(app.props.script); if ( patterns.length ) { patterns.forEach(pattern => { var match; - while ( ( match = regex.exec(html) ) ) { - if ( pattern.regex.test(match[2]) ) { - this.addDetected(app, pattern, 'script', match[2]); + scripts.forEach(uri => { + if ( pattern.regex.test(uri) ) { + this.addDetected(app, pattern, 'script', uri); } - } + }); }); } } @@ -444,12 +446,16 @@ class Wappalyzer { var patterns = this.parsePatterns(app.props.headers); if ( headers ) { - Object.keys(patterns).forEach(header => { - patterns[header].forEach(pattern => { - header = header.toLowerCase(); - - if ( header in headers && pattern.regex.test(headers[header]) ) { - this.addDetected(app, pattern, 'headers', headers[header], header); + Object.keys(patterns).forEach(headerName => { + patterns[headerName].forEach(pattern => { + headerName = headerName.toLowerCase(); + + if ( headerName in headers ) { + headers[headerName].forEach(headerValue => { + if ( pattern.regex.test(headerValue) ) { + this.addDetected(app, pattern, 'headers', headerValue, headerName); + } + }); } }); }); From 5ae47cc9c6e18ba1091e9014b5e265776eb89728 Mon Sep 17 00:00:00 2001 From: Camille Barneaud <1693643+gadcam@users.noreply.github.com> Date: Sun, 12 Nov 2017 00:08:22 +0100 Subject: [PATCH 4/4] Add JSEcoin, Crypto-Loot & ProjectPoi (#1820) * Enable multiple header with same name Enable multiple header with same name for WebExtension driver * Enable multiple header with same name Enable multiple header with same name for Bookmarklet driver * Enable multiple header with same name Enable multiple header with same name for the NPM driver * Enable multiple headers with the same name * Get scripts via DOM (WebExtension) * Get scripts via DOM (Bookmarklet) * Get scripts via DOM * Get scripts via DOM (NPM) * Add JSEcoin, Crypto-Loot & ProjectPoi Add pattern to detect cryptominers * Add JSEcoin, Crypto-Loot & ProjectPoi icons --- src/apps.json | 27 +++++++++++++++++++++++++++ src/icons/Crypto-Loot.png | Bin 0 -> 458 bytes src/icons/JSEcoin.png | Bin 0 -> 586 bytes src/icons/ProjectPoi.png | Bin 0 -> 3451 bytes 4 files changed, 27 insertions(+) create mode 100644 src/icons/Crypto-Loot.png create mode 100644 src/icons/JSEcoin.png create mode 100644 src/icons/ProjectPoi.png diff --git a/src/apps.json b/src/apps.json index dd8c6674f..5f49a4a67 100644 --- a/src/apps.json +++ b/src/apps.json @@ -1821,6 +1821,15 @@ "script": "tag\\.crsspxl\\.com/s1\\.js", "website": "http://datadesk.crsspxl.com" }, + "Crypto-Loot": { + "cats": [ + "19" + ], + "env": "^CryptoLoot$", + "script": "^(?:https):?//crypto-loot\\.com/lib/", + "icon": "Crypto-Loot.png", + "website": "https://crypto-loot.com/" + }, "CubeCart": { "cats": [ "6" @@ -4236,6 +4245,15 @@ "url": "option=com_", "website": "http://joomla.org" }, + "JSEcoin": { + "cats": [ + "19" + ], + "env": "^jseMine$", + "script": "^(?:https):?//load\\.jsecoin\\.com/server/load/", + "icon": "JSEcoin.png", + "website": "https://jsecoin.com/" + }, "K2": { "cats": [ "19" @@ -6700,6 +6718,15 @@ "script": "^https?://(?:www\\.)?projectwonderful\\.com/(?:pwa\\.js|gen\\.php)", "website": "http://projectwonderful.com" }, + "ProjectPoi": { + "cats": [ + "19" + ], + "env": "^ProjectPoi$", + "script": "^(?:https):?//ppoi\\.org/lib/", + "icon": "ProjectPoi.png", + "website": "https://ppoi.org/" + }, "Prospector": { "cats": [ "36" diff --git a/src/icons/Crypto-Loot.png b/src/icons/Crypto-Loot.png new file mode 100644 index 0000000000000000000000000000000000000000..4bcd3b9b361e234caf96d1683c1b8b0ad2e3c14a GIT binary patch literal 458 zcmV;*0X6=KP)-3rKqVM391;u%1nh}g;DS}$fK=Fi zQqXx%%y&)6bWFT$N2XyvqFz0JNib6|AT%BnCl(JN5)AE%TIz^d=Z09{f>qReQNnRb zv}i<@S2>PSHiu3#g-tSfMlM}8Bts_}IwBUXWkPgAENwt4TQwp%Ar>qe5oDp6r~m)} z>`6pHR5;7klF4?%APhx$2m_wu*^ceZ&CsOd|9@QRBByQ=Sank^xmP-R9I?MG&AF5F zrYkkMgg{2J_Q+wdO&7m!8Fwf=i2T#cwF_k-&gI$@cB2>&$9v|OaR4CgTzJm_`ifik zeDR?aF1f`J&4}pO_K@B~%vvAw8BuXbcJzopeRDIMs$uTkb8UFIHryEP7|eO2g7`kiUQmv0JjMb__K zNrlg?m5Zg}Lru&otU`>n^IUVeb6>pBg8k$A0%XYx@rs=B*8l(j07*qoM6N<$f?YhmIV0) zGn@@#Js##PwJgS3k;m6ecg>>7?WL;XLj0Tu_iUP-lIU)yv3-3{ZC<9_(WwrK+kZ+k zFfh7$x;TbdoW6Q7vS_h^h}*-@+=l&ZIX!PH6^iJt7&7*sZYvhRF(=j zEml^J&YWmvacRvtg}xux=1uhS;;G%R;CWHtkrT)7H}q~h_j&SfJ6W0C2i`Z&-lApu z>o5yXPqf!fyT+$(bFHmU%09c;6U8Mj87o*6X3W12D553QWQiP|gpUXO@ GgeCwoV8=WF literal 0 HcmV?d00001 diff --git a/src/icons/ProjectPoi.png b/src/icons/ProjectPoi.png new file mode 100644 index 0000000000000000000000000000000000000000..65bfdff62b1b6291d26002498630999fb2a677f6 GIT binary patch literal 3451 zcmV->4TSQEP)^`Tzg_{_pDf=-~V9E!*wtN;K1|Ns8~{r&vu z-~adZ|Ml|!{{H>e#sB~R`l5gR%(?&W?EnA%`^2pJ{rvv)@BaDt{nyX^(u&~a2*{U z`uE_wsG5y>X#V!v`|sEP(Zun(u_PlS4G|Xj<=6k_)BM!W_QSf-!?OP5-v8>${Is!2 zML@`9K9v8+IX5@_!ocCmzOj&rnv;b2(#!tM!1%qjzq_c#tf0uBn7NgYt%85#M^^2CQqMU@ z|MAlJ>CFGy%>2p4`@p#Tpr6#Jo~W0Gm5zU&e0OhVU;mC=U|Un+aYfs0L2x@a{^8N+ z(Z2eoq`I!4vZt8*m6)TNi=~i-je&WXeQ@hcU)VxYK|eZ1D=9K5Comx%^Y!)M;pX=9 z;rZRy`0df|-^$*np!~U>{IHR}oqUjWZ*pyJhFV%vQ&Yq1H<9bfJXhyU>MK32pf>1kz9x=+y&-9U(=A&%jQC`}JPSSEq z(Sb>58#(*Dh2^w`pK5BTWogb*VCjcPz*;dqv1qga001p?QchC<0{a8~S z7{|@krh*ze?ElVaQFS(^W5FqDoRDc-_~Az_q19gq*guMt?z9qV&W%cymI_+vj$7l zHtWZgzbF)Ct$nlu$j053mZn*jrrOor1|T~}ZCvncyj2BIJ6hJWZQIJhv6pEb6`sQagD$k23Gffbv%!Ao=n~oUa;^mcQVVX^L^u0)==(0i#bKi zWE8w7W)7tb4Vw>U7|yGUv*H}&`2$H~5puet;b78r48t(Lpe9e|!Nbf$P)B*362QAC z-G2l0*ferb5CnZ;5SR|7x5MCJ%s-(f66?@dyaybc&@qrg(7y?SQP(IG4s|Zyo-ZE9 zm_@`!9cY#tP@~wCg2Dg${%~FxobLE(c=2-GA5&RW9I4nuMdaHbRFOYO8&Md*uNwbF zQ7MQ8he8Jzd5^vqdXRwTc-#`QT!wIo30w?>a4wn1irBn?$G*M?hQCTpxL8MkI>OWP2fq&wvx)_)6pNf*0tJ< zP`$J7UVHe_dv^-+==``$s@gI;zk0GPN7e0C>-Wt~%~T!6+QvUW^p=ua8O0WYNfg?L;W`SEp(}6Tj zTwNtF=5a!U`@~~RBq__PW}<2}8dZaK-i6^8hJtut8m)56$QKYD0RaKc>6E8>F2bN0 zyCr81>xvvLq>9CMyUpr=CHCDK2qSXf960 zgcBxv4I!SEDmhoc=k`v^nlva6be)K=X_FO!$3s|P4oz{egfs>LA$%-3kislp_!)>} zBI#vJ1q`1gH8uQ7xcqvE2Vf1{1loy_Vj>pDsc=(^g~fUy8%$#+lp8SyxnTD~bC{K+ zp)L_haN{W)0l~yS8p|!TX(){2APUYGLGe+27cyxM2iN}_{S z?V{4s)?P-TwAJ>dvtGI?)fQQy+pO(e6?7|8(GA_^L}kLjF%j{C_~JQkeuRXC^Z(`h zPQFVBz$$qUBQq`gZ_*qmxvtC5##n7Rzq}smp-ZDQ0nld7Kr@eSUK++c#>^lBAR+Xa zy*B9Wk^3z);N<=Y2G}H5PCyJz00w$E=fVWZ+KrH5A9Ll%q-%dS!&wmtfo!6e^CDp5 zWJ|vRAo*Iq1Df&869BlLb%0*vm~IkoUOtb{$k0tvZa84ENIf>eYz2aJezx?u-n@PYA_CY-66rVs03ay$`DFmH2ZJKE zV(}^v1dcHn7{joxo%cJ;q9}-tzPY*jY`uPsfmpJI+Cfqo5dgpdmjeoe5sSvM#+QMZ zFNrzOELoYWy5<|wG=+64zU6Nhejb@U!eI>}sZ5Fqfpq|WchcdHX*{pk1JMWp8I2;4 zUj-B`qT7A;6hHFq!q*4&xxK(@5{d?T@Ior4r_>2` zQJt8Gcd=w+usRqFR;sJ2+DPEXF>2@Y%*gv_mbc%dHj&YW8M~;Bg><@bDmz0_&s%uq zK%%Ao-puwIu|9a>z<~o)yVYn6+)6CAp5i~zkS`mLr^4H*U1YMYyKp0&c9+@|QxbV03GG=WXh~jv2TvQo+ z7}=c+PC`8u`nW}3!7>#219XTNi9^c?FCx>mG=WmnLGc#|XsA3II)r=y5jr@b8N4@1rc?^rOu$}dMqe!;Ld$}5ER6W=1!LlC~--~Mt= z;l-UhQr!XXdh+|DOJJFgimWL9>DqFsP zGlTgeW&qaas$D!r%58Ek{*FJN>;PZ8alkah$6YK65T$70QFH6 zmwFT!X{imJ(*yjZRf-P$KD;1}WGqa*cECPRIf;6HP)*-F#=w&*pafpuo4nR=77h?E z3`+y|7Gn#H!1JwVIzE~XbqrmtS>N}x!GD+P;o264uDaHnlndQ;w5WOb3m6`vl&^Qo zLJxv#%?$^D(3;EdHqsS#ivTzqO{2WIiRNYl5R2N32_YC`Hnjk0Y&NmP6NYay{x@#p df8(+E{}