Analyze headers (#2129)

* Await analyzeHeaders promise + add tests

* Parse js patterns when analyzing js

* Put back parseJsPatterns where it was

* Separete to devDependencies

* Add apps object to isolate and prevent manual updates

* Simplify wappalyzer.apps

* Only use necessary input data per analyze use case

* Remove package-lock.json

* Checking in yarn.lock

* Remove npm from test script
main
Andres Castillo 7 years ago committed by Elbert Alias
parent 89429b0995
commit 5d6ff3ac5b

@ -3,6 +3,12 @@
"file-type": "7.4.*",
"is-svg": "2.1.*",
"read-chunk": "2.1.*"
},
"devDependencies": {
"chai": "^4.1.2",
"mocha": "^5.0.1"
},
"scripts": {
"test": "mocha -R spec ./test"
}
}

@ -489,17 +489,17 @@ class Wappalyzer {
if ( headers ) {
Object.keys(patterns).forEach(headerName => {
this.asyncForEach(patterns[headerName], pattern => {
promises.push(this.asyncForEach(patterns[headerName], pattern => {
headerName = headerName.toLowerCase();
if ( headerName in headers ) {
promises.push(headers[headerName].forEach(headerValue => {
headers[headerName].forEach(headerValue => {
if ( pattern.regex.test(headerValue) ) {
this.addDetected(app, pattern, 'headers', headerValue, headerName);
}
}));
});
}
});
}));
});
}

@ -0,0 +1,144 @@
/* eslint-env mocha */
'use strict';
const assert = require('chai').assert;
const Wappalyzer = require('../src/wappalyzer');
describe('should analyze website elements properly', function () {
it('should analyze html', async () => {
const html = `
<!DOCTYPE HTML>
<html>
<head>
<title>Page title | Html detection </title>
<meta charset="utf-8" />
</head>
<body>
<h1>Technologies Test Page | Html detection</h1>
<!-- Google Tag Manager -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KAAOEOE"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager -->
</body>
</html>
`;
const wappalyzer = new Wappalyzer();
wappalyzer.apps = {
"Google Tag Manager": {
"html": [
"googletagmanager\\.com/ns\\.html[^>]+></iframe>",
"<!-- (?:End )?Google Tag Manager -->"
]
}
};
var applications = null;
wappalyzer.driver = {
log () {},
displayApps (detectedMap) {
applications = detectedMap;
}
};
await wappalyzer.analyze({ canonical: 'example.com' }, { html });
assert.equal(applications['Google Tag Manager'].name, 'Google Tag Manager');
});
it('should analyze scripts', async () => {
const scripts = [
'http://www.google-analytics.com/analytics.js',
'http://example.com/assets/js/jquery.min.js'
];
const wappalyzer = new Wappalyzer();
wappalyzer.apps = {
"Google Analytics": {
"cats": [
10
],
"script": "google-analytics\\.com\\/(?:ga|urchin|(analytics))\\.js\\;version:\\1?UA:"
},
"jQuery": {
"script": [
"jquery(?:\\-|\\.)([\\d.]*\\d)[^/]*\\.js\\;version:\\1",
"/([\\d.]+)/jquery(?:\\.min)?\\.js\\;version:\\1",
"jquery.*\\.js(?:\\?ver(?:sion)?=([\\d.]+))?\\;version:\\1"
]
}
};
var applications = null;
wappalyzer.driver = {
log () {},
displayApps (detectedMap) {
applications = detectedMap;
}
};
await wappalyzer.analyze({ canonical: 'example.com' }, { scripts });
assert.equal(applications['Google Analytics'].name, 'Google Analytics');
assert.equal(applications['jQuery'].name, 'jQuery');
});
it('should analyze headers', async () => {
const headers = {
'date': [ 'Thu, 01 Feb 2018 11:34:18 GMT' ],
'connection': [ 'keep-alive' ],
'x-powered-by': [ 'Express'],
'etag': [ 'W/125-1jQLmiya7mfec43xR3Eb3pjdu64s' ],
'content-length': [ '293' ],
'content-type': [ 'text/html; charset=utf-8' ]
};
const wappalyzer = new Wappalyzer();
wappalyzer.apps = {
"Express": {
"headers": {
"X-Powered-By": "^Express$"
}
}
};
var applications = null;
wappalyzer.driver = {
log () {},
displayApps (detectedMap) {
applications = detectedMap;
}
};
await wappalyzer.analyze({ canonical: 'example.com' }, { headers });
assert.equal(applications['Express'].name, 'Express');
});
it('should analyze js globals', async () => {
const js = {
'Moment.js': { 'moment': { '0': true } },
'Google Font API': { 'WebFonts': { '0': true } }
};
const wappalyzer = new Wappalyzer();
wappalyzer.apps = {
"Moment.js": {
"js": {
"moment": "",
"moment.version": "(.*)\\;version:\\1"
}
},
"Google Font API": {
"js": {
"WebFonts": ""
}
}
};
var applications = null;
wappalyzer.driver = {
log () {},
displayApps (detectedMap) {
applications = detectedMap;
}
};
wappalyzer.parseJsPatterns();
await wappalyzer.analyze({ canonical: 'example.com' }, { js });
assert.equal(applications['Google Font API'].name, 'Google Font API');
assert.equal(applications['Moment.js'].name, 'Moment.js');
});
});

@ -2,20 +2,173 @@
# yarn lockfile v1
assertion-error@^1.0.1:
version "1.1.0"
resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b"
balanced-match@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
brace-expansion@^1.1.7:
version "1.1.11"
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
dependencies:
balanced-match "^1.0.0"
concat-map "0.0.1"
browser-stdout@1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.0.tgz#f351d32969d32fa5d7a5567154263d928ae3bd1f"
chai@^4.1.2:
version "4.1.2"
resolved "https://registry.yarnpkg.com/chai/-/chai-4.1.2.tgz#0f64584ba642f0f2ace2806279f4f06ca23ad73c"
dependencies:
assertion-error "^1.0.1"
check-error "^1.0.1"
deep-eql "^3.0.0"
get-func-name "^2.0.0"
pathval "^1.0.0"
type-detect "^4.0.0"
check-error@^1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.2.tgz#574d312edd88bb5dd8912e9286dd6c0aed4aac82"
commander@2.11.0:
version "2.11.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563"
concat-map@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
debug@3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
dependencies:
ms "2.0.0"
deep-eql@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-3.0.1.tgz#dfc9404400ad1c8fe023e7da1df1c147c4b444df"
dependencies:
type-detect "^4.0.0"
diff@3.3.1:
version "3.3.1"
resolved "https://registry.yarnpkg.com/diff/-/diff-3.3.1.tgz#aa8567a6eed03c531fc89d3f711cd0e5259dec75"
escape-string-regexp@1.0.5:
version "1.0.5"
resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4"
file-type@7.4.*:
version "7.4.0"
resolved "https://registry.yarnpkg.com/file-type/-/file-type-7.4.0.tgz#2a7c94f62a0030150bb7d9b6c70cfa1d3e759c86"
fs.realpath@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
get-func-name@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.0.tgz#ead774abee72e20409433a066366023dd6887a41"
glob@7.1.2:
version "7.1.2"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
dependencies:
fs.realpath "^1.0.0"
inflight "^1.0.4"
inherits "2"
minimatch "^3.0.4"
once "^1.3.0"
path-is-absolute "^1.0.0"
growl@1.10.3:
version "1.10.3"
resolved "https://registry.yarnpkg.com/growl/-/growl-1.10.3.tgz#1926ba90cf3edfe2adb4927f5880bc22c66c790f"
has-flag@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51"
he@1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd"
html-comment-regex@^1.1.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/html-comment-regex/-/html-comment-regex-1.1.1.tgz#668b93776eaae55ebde8f3ad464b307a4963625e"
inflight@^1.0.4:
version "1.0.6"
resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9"
dependencies:
once "^1.3.0"
wrappy "1"
inherits@2:
version "2.0.3"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
is-svg@2.1.*:
version "2.1.0"
resolved "https://registry.yarnpkg.com/is-svg/-/is-svg-2.1.0.tgz#cf61090da0d9efbcab8722deba6f032208dbb0e9"
dependencies:
html-comment-regex "^1.1.0"
minimatch@^3.0.4:
version "3.0.4"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
dependencies:
brace-expansion "^1.1.7"
minimist@0.0.8:
version "0.0.8"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d"
mkdirp@0.5.1:
version "0.5.1"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903"
dependencies:
minimist "0.0.8"
mocha@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/mocha/-/mocha-5.0.1.tgz#759b62c836b0732382a62b6b1fb245ec1bc943ac"
dependencies:
browser-stdout "1.3.0"
commander "2.11.0"
debug "3.1.0"
diff "3.3.1"
escape-string-regexp "1.0.5"
glob "7.1.2"
growl "1.10.3"
he "1.1.1"
mkdirp "0.5.1"
supports-color "4.4.0"
ms@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
once@^1.3.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
dependencies:
wrappy "1"
path-is-absolute@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
pathval@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.0.tgz#b942e6d4bde653005ef6b71361def8727d0645e0"
pify@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176"
@ -30,3 +183,17 @@ read-chunk@2.1.*:
safe-buffer@^5.1.1:
version "5.1.1"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
supports-color@4.4.0:
version "4.4.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.4.0.tgz#883f7ddabc165142b2a61427f3352ded195d1a3e"
dependencies:
has-flag "^2.0.0"
type-detect@^4.0.0:
version "4.0.8"
resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c"
wrappy@1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"

Loading…
Cancel
Save