Migrations
/
wappalyzer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added regex validation script, improved patterns

Browse Source
main
Elbert Alias 10 years ago
parent d6debd16bb
commit 512b36aa23
5 changed files with 116 additions and 76 deletions
Show Stats Download Patch File Download Diff File

4
bin/wappalyzer-validate
Unescape View File

@ -29,6 +29,10 @@ echo "Validating apps.json..."
jsonlint --quiet -V $WAPPALYZER_ROOT/schema.json $path/apps.json
echo "Validating regular expressions..."
wappalyzer validate-regex
echo "Validating icons..."
wappalyzer validate-icons

7
bin/wappalyzer-validate-icons
Unescape View File

@ -1,13 +1,14 @@
#!/usr/bin/env node
var
app,
modulesPath = process.env.WAPPALYZER_NODE_PATH !== undefined ? process.env.WAPPALYZER_NODE_PATH + '/node_modules/' : '',
fs = require('fs'),
fileType = require(modulesPath + 'file-type'),
fs = require(modulesPath + 'fs-extra'),
readChunk = require(modulesPath + 'read-chunk')
json = require(process.env.WAPPALYZER_ROOT + '/src/apps.json');
Object.keys(json.apps).forEach(function(app) {
for ( app in json.apps ) {
var path = process.env.WAPPALYZER_ROOT + '/src/icons/' + app + '.png';
fs.exists(path, function(exists) {
@ -23,4 +24,4 @@ Object.keys(json.apps).forEach(function(app) {
throw new Error('Missing file: src/icons/' + app + '.png');
}
});
});
};

50
bin/wappalyzer-validate-regex
Unescape View File

@ -0,0 +1,50 @@
#!/usr/bin/env node
var
app,
modulesPath = process.env.WAPPALYZER_NODE_PATH !== undefined ? process.env.WAPPALYZER_NODE_PATH + '/node_modules/' : '',
json = require(process.env.WAPPALYZER_ROOT + '/src/apps.json');
for ( app in json.apps ) {
['headers', 'html', 'env', 'meta', 'script'].forEach(function(type) {
var
key,
patterns = json.apps[app][type];
if ( patterns !== undefined ) {
patterns = typeof patterns === 'string' ? [patterns] : patterns;
if ( !( patterns instanceof Array ) ) {
patterns = [];
for ( key in json.apps[app][type] ) {
patterns.push(json.apps[app][type][key]);
}
}
patterns.forEach(function(pattern) {
var
attrs = pattern.split('\\;'),
regex = '/' + attrs.shift().replace('/', '\/') + '/';
if ( /^\/(?:\^\$|\.\+|\.\*)\/$/.test(regex) ) {
throw new Error('Pattern should be replaced with empty string.\n' + app + ': ' + type + ': ' + pattern);
}
if ( type === 'html' ) {
if ( /\.(?:\+|\*)/.test(regex) ) {
throw new Error('Avoid ".+" and ".*" in HTML patterns. Consider using "[^>]+" or "[^<]+" instead.\n' + app + ': ' + type + ': ' + pattern);
}
if ( !/[<>]/.test(regex) ) {
throw new Error('HTML patterns must contain "<" or ">".\n' + app + ': ' + type + ': ' + pattern);
}
}
});
}
});
if ( /[a-z]+:\/\//i.test(json.apps[app].website) ) {
throw new Error('Do not include the protocol in the website URL\n' + app + ': ' + json.apps[app].website);
}
}

1
docker/node/package.json
Unescape View File

@ -1,7 +1,6 @@
{
"devDependencies": {
"file-type": "2.2.*",
"fs-extra": "0.16.*",
"read-chunk": "1.0.*"
}
}

130
src/apps.json
Unescape View File

@ -187,11 +187,10 @@
],
"html": [
"<div class=\"[^\"]*parbase",
"_jcr_content",
"/etc/designs/",
"/etc/clientlibs/"
"<div[^>]+data-component-path=\"[^\"+]jcr:"
],
"implies": "Java",
"script": "/etc/designs/",
"website": "adobe.com/products/cq.html"
},
"Adobe ColdFusion": {
@ -332,7 +331,6 @@
"cats": [
1
],
"html": "system_(?:js\\.php\\?script=|css\\.php\\?styles)[^\"]+cv=([\\d.]+)\\;version:\\1",
"implies": "PHP",
"meta": {
"generator": "Amiro"
@ -378,14 +376,14 @@
"cats": [
34
],
"html": "type=\"text/css\" href=\"/static/hbase\\.css\"",
"html": "<style[^>]+static/hbase",
"website": "hbase.apache.org"
},
"Apache Hadoop": {
"cats": [
34
],
"html": "type=\"text/css\" href=\"/static/hadoop\\.css\"",
"html": "<style[^>]+static/hadoop",
"website": "hadoop.apache.org"
},
"Apache JSPWiki": {
@ -436,7 +434,7 @@
"cats": [
2
],
"html": "ping\\.src = node\\.href;",
"html": "ping\\.src = node\\.href;\\s+[^>]+\\s+}\\s+</script>",
"website": "arclanguage.org"
},
"Artifactory": {
@ -667,7 +665,7 @@
],
"implies": "Google Analytics",
"script": "boba(\\.min)?\\.js",
"website": "http://boba.space150.com/"
"website": "boba.space150.com"
},
"Bolt": {
"cats": [
@ -734,7 +732,7 @@
"cats": [
2
],
"html": "<a href=\"[^>]+woltlab\\.com.+Burning Board",
"html": "<a href=\"[^>]+woltlab\\.com[^<]+<strong>Burning Board",
"implies": "PHP",
"website": "www.woltlab.com"
},
@ -824,7 +822,7 @@
"env": "^fn_compare_strings$",
"html": [
"&nbsp;Powered by (?:<a href=[^>]+cs-cart\\.com|CS-Cart)",
"(?:\\$|jQuery)\\.runCart\\('\\w'\\)"
".cm-noscript[^>]+</style>"
],
"implies": "PHP",
"website": "www.cs-cart.com"
@ -952,7 +950,7 @@
"cats": [
10
],
"html": "function loadChartbeat\\(\\) \\{",
"env": "^_sf_(?:endpt|async_config)$",
"script": "chartbeat\\.js",
"website": "chartbeat.com"
},
@ -1014,7 +1012,6 @@
10
],
"env": "^ClickTale",
"html": "if\\(typeof ClickTale\\(Tag\\)*==\\\"function\\\"\\)",
"website": "www.clicktale.com"
},
"Clicky": {
@ -1286,8 +1283,7 @@
1
],
"html": [
"<a href=\"http://www\\.dtg\\.nl/\"[^>]+>Site Powered by DTG",
"var u=\\(\\('https:' == d\\.location\\.protocol\\) \\? 'https://resellerstat\\.mono\\.net/dtg/' : 'http://resellerstat\\.mono\\.net/dtg/'\\);"
"<a[^>]+Site Powered by DTG"
],
"implies": "Mono.net",
"website": "www.dtg.nl"
@ -1561,7 +1557,7 @@
"cats": [
6
],
"html": "(?:id=\"block[_-]commerce[_-]cart[_-]cart|class=\"commerce[_-]product[_-]field)",
"html": "<[^]+(?:id=\"block[_-]commerce[_-]cart[_-]cart|class=\"commerce[_-]product[_-]field)",
"implies": "Drupal",
"website": "drupalcommerce.org"
},
@ -1722,10 +1718,10 @@
"cats": [
25
],
"html": "<link.+?href=\"[^\"]+epoch(?:\\.min)?\\.css",
"html": "<link[^>]+?href=\"[^\"]+epoch(?:\\.min)?\\.css",
"implies": "D3",
"script": "epoch(\\.min)?\\.js",
"website": "https://fastly.github.io/epoch/"
"website": "fastly.github.io/epoch"
},
"Epom": {
"cats": [
@ -2068,7 +2064,7 @@
"meta": {
"description": "GitLab Continuous Integration"
},
"website": "https://about.gitlab.com/gitlab-ci/"
"website": "about.gitlab.com/gitlab-ci"
},
"GlassFish": {
"cats": [
@ -2129,8 +2125,7 @@
"headers": {
"Set-Cookie": "__utma"
},
"html": "_gaq\\.push\\(\\['_setAccount|i\\['GoogleAnalyticsObject'\\]|ga\\.async = true",
"script": "^https?://[^\\/]+\\.google-analytics\\.com\\/(?:ga|urchin|(analytics))\\.js\\;version:\\1?Universal Analytics:",
"script": "^https?://[^\\/]+\\.google-analytics\\.com\\/(?:ga|urchin|(analytics))\\.js\\;version:\\1?UA:",
"website": "google.com/analytics"
},
"Google App Engine": {
@ -2250,7 +2245,7 @@
19
],
"env": "^Gravatar$",
"html": "gravatar\\.com/avatar/",
"html": "<[^]+gravatar\\.com/avatar/",
"website": "gravatar.com"
},
"Gravity Insights": {
@ -2344,7 +2339,7 @@
],
"env": "^Hammer$",
"script": "hammer(\\.min)?\\.js",
"website": "http://hammerjs.github.io"
"website": "hammerjs.github.io"
},
"Handlebars": {
"cats": [
@ -2384,8 +2379,8 @@
"cats": [
5
],
"html": "/hellobar\\.js",
"script": "(?:hellobar\\.com/hellobar\\.js|new HelloBar)",
"env": "^HelloBar$",
"script": "hellobar\\.js",
"website": "hellobar.com"
},
"Hiawatha": {
@ -2459,7 +2454,7 @@
"headers": {
"Set-Cookie": "_hybris"
},
"html": "(?:/sys_master/|/hybr/|/_ui/desktop/)",
"html": "<[^]+(?:/sys_master/|/hybr/|/_ui/desktop/)",
"implies": "Java",
"website": "hybris.com"
},
@ -3070,7 +3065,7 @@
12
],
"script": "lazy(\\.browser)?(\\.min)?\\.js",
"website": "http://danieltao.com/lazy.js/"
"website": "danieltao.com/lazy.js"
},
"Leaflet": {
"cats": [
@ -3168,10 +3163,10 @@
"cats": [
1
],
"env": "^LIVESTREET",
"headers": {
"X-Powered-By": "LiveStreet CMS"
},
"html": "var LIVESTREET_SECURITY_KEY",
"website": "livestreetcms.com"
},
"Livefyre": {
@ -3206,7 +3201,7 @@
"cats": [
1
],
"html": "<link[^>]*/sites/[a-z\\d]{24}/theme/stylesheets/.*>",
"html": "<link[^>]*/sites/[a-z\\d]{24}/theme/stylesheets",
"implies": [
"Ruby on Rails",
"MongoDB"
@ -3330,7 +3325,6 @@
"cats": [
5
],
"html": "\\/assets\\/js\\/manycontacts\\.min\\.js",
"script": "\\/assets\\/js\\/manycontacts\\.min\\.js",
"website": "www.manycontacts.com"
},
@ -3450,7 +3444,7 @@
"cats": [
2
],
"html": "<a href=\"[^>]+minibb.+\\s+<!--End of copyright link",
"html": "<a href=\"[^\"]+minibb[^<]+</a>[^<]+\n<!--End of copyright link",
"website": "www.minibb.com"
},
"MiniServ": {
@ -3610,8 +3604,7 @@
"cats": [
1
],
"env": "_monoTracker",
"html": "var u=\\(\\('https:' == d\\.location\\.protocol\\) \\? 'https://resellerstat\\.mono\\.net/mono/' : 'http://resellerstat\\.mono\\.net/mono/'\\);",
"env": "^_monoTracker$",
"implies": "Piwik",
"script": "monotracker(?:\\.min)?\\.js",
"website": "www.mono.net"
@ -3755,7 +3748,7 @@
"cats": [
10
],
"html": "sitestat\\(\".+nl\\.sitestat\\.com",
"env": "^sitestat$",
"website": "www.nedstat.com"
},
"Nepso": {
@ -3824,7 +3817,7 @@
"html": "<link [^>]*href=\"[^\"]+owl.carousel(?:\\.min)?\\.css",
"implies": "jQuery",
"script": "owl.carousel.*\\.js",
"website": "http://owlgraphic.com/owlcarousel/"
"website": "owlgraphic.com/owlcarousel"
},
"OXID eShop": {
"cats": [
@ -3854,7 +3847,7 @@
"cats": [
10
],
"html": "var p==.+stat\\.onestat\\.com/stat\\.aspx\\?tagver",
"env": "^OneStat",
"website": "www.onestat.com"
},
"Open AdStream": {
@ -3872,7 +3865,7 @@
"author": "open-classifieds\\.com",
"copyright": "Open Classifieds ?([0-9.]+)?\\;version:\\1"
},
"website": "http://open-classifieds.com"
"website": "open-classifieds.com"
},
"Open Journal Systems": {
"cats": [
@ -3905,7 +3898,7 @@
"author": "open-eshop\\.com",
"copyright": "Open eShop ?([0-9.]+)?\\;version:\\1"
},
"website": "http://open-eshop.com/"
"website": "open-eshop.com/"
},
"OpenCart": {
"cats": [
@ -4135,7 +4128,6 @@
32
],
"env": "^pi(?:Tracker|Hostname|Protocol|CId|AId)$",
"html": "piProtocol + \"pi\\.pardot\\.com/pi\\.js",
"website": "pardot.com"
},
"Parse.ly": {
@ -4150,7 +4142,7 @@
25
],
"script": "paths(\\.min)?\\.js",
"website": "https://github.com/andreaferretti/paths-js"
"website": "github.com/andreaferretti/paths-js"
},
"PayPal": {
"cats": [
@ -4248,7 +4240,6 @@
"headers": {
"Set-Cookie": "PIWIK_SESSID="
},
"html": "var piwikTracker = Piwik\\.getTracker\\(",
"meta": {
"apple-itunes-app": "app-id=737216887",
"generator": "Piwik - Open Source Web Analytics",
@ -4349,7 +4340,10 @@
"cats": [
6
],
"html": "(s\\d\\d)\\.php\\?shopid=\\1",
"html": [
"<a[^>]+title=\"POWERGAP",
"<input type=\"hidden\" name=\"shopid\""
],
"website": "powergap.de"
},
"Prefix-Free": {
@ -4588,7 +4582,7 @@
"env": "^Rickshaw$",
"implies": "D3",
"script": "rickshaw(\\.min)?\\.js",
"website": "http://code.shutterstock.com/rickshaw/"
"website": "code.shutterstock.com/rickshaw/"
},
"RiteCMS": {
"cats": [
@ -4666,10 +4660,7 @@
"cats": [
1
],
"html": [
"mfinfo\\.application='Tridion",
"<img[^>]+_tcm\\d{2,3}-\\d{6}\\."
],
"html": "<img[^>]+_tcm\\d{2,3}-\\d{6}\\.",
"website": "www.sdl.com/products/tridion"
},
"SIMsite": {
@ -4840,7 +4831,7 @@
"cats": [
6
],
"html": "http://www\\.getseoshop\\.com",
"html": "<a[^>]+title=\"SEOshop",
"website": "getseoshop.com"
},
"Serendipity": {
@ -4964,7 +4955,6 @@
10
],
"env": "^s_(?:account|objectID|code|INST)$",
"html": "var s_code=s\\.t\\(\\);if\\(s_code\\)document\\.write\\(s_code\\)",
"script": "/s[_-]code.*\\.js",
"website": "www.adobe.com/solutions/digital-marketing.html"
},
@ -5081,7 +5071,6 @@
19
],
"env": "^SobiProUrl$",
"html": " <(?:script|link)[^>].*com_sobipro.*>",
"implies": "Joomla",
"website": "sigsiu.net/sobipro.html"
},
@ -5190,7 +5179,7 @@
"headers": {
"X-Powered-By": "Squiz Matrix"
},
"html": " Running (?:MySource|Squiz) Matrix",
"html": "<!--\\s+Running (?:MySource|Squiz) Matrix",
"implies": "PHP",
"meta": {
"generator": "Squiz Matrix"
@ -5274,9 +5263,9 @@
12
],
"env": "^swal$",
"html": "<link.+?href=\"[^\"]+sweet-alert(?:\\.min)?\\.css",
"html": "<link[^>]+?href=\"[^\"]+sweet-alert(?:\\.min)?\\.css",
"script": "sweet-alert(\\.min)?\\.js",
"website": "http://tristanedwards.me/sweetalert"
"website": "tristanedwards.me/sweetalert"
},
"Swiftlet": {
"cats": [
@ -5356,7 +5345,7 @@
1
],
"excludes": "TYPO3 CMS",
"html": "xmlns:typo3=\\\"http://www\\.typo3\\.org/ns/\\d{4}/Flow/Packages/Neos/Content/",
"html": "<html[^>]+xmlns:typo3=\"[^\"]+Flow/Packages/Neos/",
"implies": [
"PHP",
"TYPO3 Flow"
@ -5375,10 +5364,7 @@
"cats": [
44
],
"html": [
"http://www.jetbrains.com/teamcity/feedback\\?source=footer&version=([\\d\\.]+)\\;version:\\1",
"\n\\s*<span class=\"versionTag\"><span class=\"vWord\">Version</span> ([\\d\\.]+) \\(build \\d+\\)</span>\\;version:\\1"
],
"html": "<span class=\"versionTag\"><span class=\"vWord\">Version</span> ([\\d\\.]+)\\;version:\\1",
"implies": [
"jQuery",
"Prototype"
@ -5515,7 +5501,7 @@
10
],
"env": "^TrackJs$",
"script": "\tracker.js",
"script": "tracker.js",
"website": "trackjs.com"
},
"Tumblr": {
@ -5571,8 +5557,7 @@
],
"env": "^Twipsy$\\;confidence:50",
"html": [
"<style[^>]*>.*Bootstrap v.* Copyright .* Twitter, Inc\\.\\* Licensed under.*<\\/style>",
"<link.+?href=\"[^\"]+bootstrap(?:\\.min)?\\.css",
"<link[^>]+?href=\"[^\"]+bootstrap(?:\\.min)?\\.css",
"<div [^>]*class=\"[^\"]*col-(?:xs|sm|md|lg)-\\d{1,2}"
],
"script": "(?:twitter\\.github\\.com/bootstrap|bootstrap(?:\\.js|\\.min\\.js))",
@ -5584,7 +5569,7 @@
],
"env": "^twemoji$",
"script": "twemoji(\\.min)?\\.js",
"website": "https://twitter.github.io/twemoji/"
"website": "twitter.github.io/twemoji/"
},
"Twitter Flight": {
"cats": [
@ -5592,7 +5577,7 @@
],
"env": "^flight$",
"implies": "jQuery",
"website": "https://flightjs.github.io/"
"website": "flightjs.github.io/"
},
"Twitter typeahead.js": {
"cats": [
@ -5601,7 +5586,7 @@
"env": "^typeahead$",
"implies": "jQuery\\;confidence:50",
"script": "(typeahead|bloodhound)(\\.jquery|\\.bundle)?(\\.min)?\\.js",
"website": "https://twitter.github.io/typeahead.js/"
"website": "twitter.github.io/typeahead.js"
},
"TypePad": {
"cats": [
@ -6086,7 +6071,7 @@
"cats": [
1
],
"html": "(?:<a href=\"[^>]+wolfcms\\.org.+Wolf CMS.+inside|Thank you for using <a[^>]+>Wolf CMS)",
"html": "(?:<a href=\"[^>]+wolfcms\\.org[^>]+>Wolf CMS(?:</a>)? inside|Thank you for using <a[^>]+>Wolf CMS)",
"website": "www.wolfcms.org"
},
"WooCommerce": {
@ -6310,7 +6295,7 @@
"cats": [
10
],
"html": "mc\\.yandex\\.ru\\/metrika\\/watch\\.js|\b(?:yaParams|yaCounter|yandex_metrika_callbacks)\b",
"env": "^yandex_metrika",
"script": "mc\\.yandex\\.ru\\/metrika\\/watch\\.js",
"website": "metrika.yandex.com"
},
@ -6374,7 +6359,8 @@
"cats": [
19
],
"html": "zbxCallPostScripts\\(\\);",
"env": "^zbxCallPostScripts$",
"html": "<body[^>]+zbxCallPostScripts",
"meta": {
"Author": "ZABBIX SIA\\;confidence:70"
},
@ -6475,9 +6461,9 @@
"cats": [
19
],
"html": "id='cgit'",
"html": "<[^]+id='cgit'",
"implies": "Perl",
"website": "git.zx2c4.com/cgit/"
"website": "git.zx2c4.com/cgit"
},
"comScore": {
"cats": [
@ -7040,7 +7026,7 @@
],
"env": "^Shine$",
"script": "shine(\\.min)?\\.js",
"website": "http://bigspaceship.github.io/shine.js/"
"website": "bigspaceship.github.io/shine.js/"
},
"spin.js": {
"cats": [
@ -7124,9 +7110,9 @@
25
],
"env": "^vis$",
"html": "<link.+?href=\"[^\"]+vis(?:\\.min)?\\.css",
"html": "<link[^>]+?href=\"[^\"]+vis(?:\\.min)?\\.css",
"script": "vis(\\.min)?\\.js",
"website": "http://visjs.org"
"website": "visjs.org"
},
"webEdition": {
"cats": [
@ -7152,7 +7138,7 @@
"cats": [
6
],
"html": "<div class=\"copyright\">.+<a[^>]+>xt:Commerce",
"html": "<div class=\"copyright\">[^<]+<a[^>]+>xt:Commerce",
"meta": {
"generator": "xt:Commerce"
},

Write Preview
Loading…
Cancel
Save