Build v6.0.7

main
Elbert Alias 5 years ago
parent 358d93f851
commit 65c2ce86dc

@ -39,6 +39,7 @@ set -e
# NPM # NPM
sed -i "s/\"version\": \"[^\"]*\"/\"version\": \"$version\"/" src/drivers/npm/package.json sed -i "s/\"version\": \"[^\"]*\"/\"version\": \"$version\"/" src/drivers/npm/package.json
sed -i "s/\"version\": \"[^\"]*\"/\"version\": \"$version\"/" src/package.json
# WebExtension # WebExtension

@ -4225,7 +4225,7 @@
}, },
"icon": "GitHub.svg", "icon": "GitHub.svg",
"implies": "Ruby on Rails", "implies": "Ruby on Rails",
"url": "^https?://[^/]+\\.github\\.io/", "url": "^https?://[^/]+\\.github\\.io",
"website": "https://pages.github.com/" "website": "https://pages.github.com/"
}, },
"GitLab": { "GitLab": {

@ -2,22 +2,23 @@
[Wappalyzer](https://www.wappalyzer.com/) indentifies technologies on websites. [Wappalyzer](https://www.wappalyzer.com/) indentifies technologies on websites.
*Note:* The [wappalyzer-core](https://www.npmjs.com/package/wappalyzer-core) package provides a low-level API without dependencies.
## CLI ## Command line
## Installation ### Installation
```shell ```shell
$ npm i -g wappalyzer $ npm i -g wappalyzer
``` ```
## Usage ### Usage
``` ```
wappalyzer <url> [options] wappalyzer <url> [options]
``` ```
### Options #### Options
``` ```
-b, --batch-size=... Process links in batches -b, --batch-size=... Process links in batches
@ -37,13 +38,13 @@ wappalyzer <url> [options]
## Dependency ## Dependency
## Installation ### Installation
```shell ```shell
$ npm i wappalyzer $ npm i wappalyzer
``` ```
## Usage ### Usage
```javascript ```javascript
const Wappalyzer = require('wappalyzer'); const Wappalyzer = require('wappalyzer');

@ -306,7 +306,7 @@ class Site {
} else { } else {
responseReceived = true responseReceived = true
this.onDetect(analyze(url, { headers })) this.onDetect(analyze({ headers }))
} }
} }
} catch (error) { } catch (error) {
@ -349,15 +349,28 @@ class Site {
).jsonValue() ).jsonValue()
// Script tags // Script tags
const scripts = ( const scripts = await (
await (
await page.evaluateHandle(() => await page.evaluateHandle(() =>
Array.from(document.getElementsByTagName('script')).map( Array.from(document.getElementsByTagName('script'))
({ src }) => src .map(({ src }) => src)
.filter((src) => src)
) )
).jsonValue()
// Meta tags
const meta = await (
await page.evaluateHandle(() =>
Array.from(document.querySelectorAll('meta')).reduce((metas, meta) => {
const key = meta.getAttribute('name') || meta.getAttribute('property')
if (key) {
metas[key.toLowerCase()] = [meta.getAttribute('content')]
}
return metas
}, {})
) )
).jsonValue() ).jsonValue()
).filter((script) => script)
// JavaScript // JavaScript
const win = await page.evaluate(getJs) const win = await page.evaluate(getJs)
@ -393,13 +406,12 @@ class Site {
}, []) }, [])
// Cookies // Cookies
const cookies = (await page.cookies()).map( const cookies = (await page.cookies()).reduce(
({ name, value, domain, path }) => ({ (cookies, { name, value }) => ({
name, ...cookies,
value, [name]: [value]
domain, }),
path {}
})
) )
// HTML // HTML
@ -464,10 +476,11 @@ class Site {
this.onDetect( this.onDetect(
url, url,
analyze(url, { analyze({
cookies, cookies,
html, html,
scripts scripts,
meta
}) })
) )

@ -53,12 +53,18 @@ const Content = {
.filter((script) => script.indexOf('data:text/javascript;') !== 0) .filter((script) => script.indexOf('data:text/javascript;') !== 0)
// Meta tags // Meta tags
const meta = Array.from(document.querySelectorAll('meta')) const meta = Array.from(document.querySelectorAll('meta')).reduce(
.map((meta) => ({ (metas, meta) => {
key: meta.getAttribute('name') || meta.getAttribute('property'), const key = meta.getAttribute('name') || meta.getAttribute('property')
value: meta.getAttribute('content')
})) if (key) {
.filter(({ value }) => value) metas[key.toLowerCase()] = [meta.getAttribute('content')]
}
return metas
},
{}
)
Content.port.postMessage({ Content.port.postMessage({
func: 'onContentLoad', func: 'onContentLoad',

@ -68,7 +68,7 @@ const Driver = {
if (previous === null) { if (previous === null) {
Driver.open('https://www.wappalyzer.com/installed') Driver.open('https://www.wappalyzer.com/installed')
} else if (version !== previous && upgradeMessage) { } else if (version !== previous && upgradeMessage) {
// Driver.open(`https://www.wappalyzer.com/upgraded?v${version}`, false) Driver.open(`https://www.wappalyzer.com/upgraded?v${version}`, false)
} }
await setOption('version', version) await setOption('version', version)
@ -111,9 +111,7 @@ const Driver = {
} }
}, },
async analyzeJs(href, js) { async analyzeJs(url, js) {
const url = new URL(href)
await Driver.onDetect( await Driver.onDetect(
url, url,
Array.prototype.concat.apply( Array.prototype.concat.apply(
@ -157,9 +155,9 @@ const Driver = {
const headers = {} const headers = {}
try { try {
const url = new URL(request.url) const [tab] = await promisify(chrome.tabs, 'query', {
url: [request.url]
const [tab] = await promisify(chrome.tabs, 'query', { url: [url.href] }) })
if (tab) { if (tab) {
request.responseHeaders.forEach((header) => { request.responseHeaders.forEach((header) => {
@ -176,7 +174,7 @@ const Driver = {
headers['content-type'] && headers['content-type'] &&
/\/x?html/.test(headers['content-type'][0]) /\/x?html/.test(headers['content-type'][0])
) { ) {
await Driver.onDetect(url, analyze(url.href, { headers }, { tab })) await Driver.onDetect(request.url, analyze({ headers }))
} }
} }
} catch (error) { } catch (error) {
@ -185,15 +183,23 @@ const Driver = {
} }
}, },
async onContentLoad(href, items, language) { async onContentLoad(url, items, language) {
try { try {
const url = new URL(href) const { hostname } = new URL(url)
items.cookies = await promisify(chrome.cookies, 'getAll', { items.cookies = (
domain: `.${url.hostname}` await promisify(chrome.cookies, 'getAll', {
domain: `.${hostname}`
}) })
).reduce(
(cookies, { name, value }) => ({
...cookies,
[name]: [value]
}),
{}
)
await Driver.onDetect(url, analyze(href, items), language, true) await Driver.onDetect(url, analyze({ url, ...items }), language, true)
} catch (error) { } catch (error) {
Driver.error(error) Driver.error(error)
} }
@ -208,7 +214,7 @@ const Driver = {
return return
} }
const { hostname, href } = url const { hostname } = new URL(url)
// Cache detections // Cache detections
const cache = (Driver.cache.hostnames[hostname] = { const cache = (Driver.cache.hostnames[hostname] = {
@ -278,7 +284,7 @@ const Driver = {
await Driver.setIcon(url, resolved) await Driver.setIcon(url, resolved)
const tabs = await promisify(chrome.tabs, 'query', { url: [href] }) const tabs = await promisify(chrome.tabs, 'query', { url })
tabs.forEach(({ id }) => (Driver.cache.tabs[id] = resolved)) tabs.forEach(({ id }) => (Driver.cache.tabs[id] = resolved))
@ -314,7 +320,7 @@ const Driver = {
})[0] || { icon }) })[0] || { icon })
} }
const tabs = await promisify(chrome.tabs, 'query', { url: [url.href] }) const tabs = await promisify(chrome.tabs, 'query', { url })
await Promise.all( await Promise.all(
tabs.map(async ({ id: tabId }) => { tabs.map(async ({ id: tabId }) => {

@ -110,7 +110,7 @@ const Wappalyzer = {
const index = resolved.findIndex(({ name }) => name === excluded.name) const index = resolved.findIndex(({ name }) => name === excluded.name)
if (index === -1) { if (index !== -1) {
resolved.splice(index, 1) resolved.splice(index, 1)
} }
}) })
@ -145,7 +145,7 @@ const Wappalyzer = {
} }
}, },
analyze(url, { html, meta, headers, cookies, scripts }) { analyze({ url, html, meta, headers, cookies, scripts }) {
const oo = Wappalyzer.analyzeOneToOne const oo = Wappalyzer.analyzeOneToOne
const om = Wappalyzer.analyzeOneToMany const om = Wappalyzer.analyzeOneToMany
const mm = Wappalyzer.analyzeManyToMany const mm = Wappalyzer.analyzeManyToMany
@ -158,10 +158,10 @@ const Wappalyzer = {
flatten([ flatten([
oo(technology, 'url', url), oo(technology, 'url', url),
oo(technology, 'html', html), oo(technology, 'html', html),
om(technology, 'meta', meta), om(technology, 'scripts', scripts),
mm(technology, 'headers', headers), mm(technology, 'cookies', cookies),
om(technology, 'cookies', cookies), mm(technology, 'meta', meta),
om(technology, 'scripts', scripts) mm(technology, 'headers', headers)
]) ])
) )
).filter((technology) => technology) ).filter((technology) => technology)
@ -196,15 +196,7 @@ const Wappalyzer = {
categories: cats || [], categories: cats || [],
slug: Wappalyzer.slugify(name), slug: Wappalyzer.slugify(name),
url: transform(url), url: transform(url),
headers: transform( headers: transform(headers),
Object.keys(headers || {}).reduce(
(lcHeaders, header) => ({
...lcHeaders,
[header.toLowerCase()]: headers[header]
}),
{}
)
),
cookies: transform(cookies), cookies: transform(cookies),
html: transform(html), html: transform(html),
meta: transform(meta), meta: transform(meta),
@ -248,7 +240,7 @@ const Wappalyzer = {
} }
const parsed = Object.keys(patterns).reduce((parsed, key) => { const parsed = Object.keys(patterns).reduce((parsed, key) => {
parsed[key] = toArray(patterns[key]).map((pattern) => { parsed[key.toLowerCase()] = toArray(patterns[key]).map((pattern) => {
const { regex, confidence, version } = pattern const { regex, confidence, version } = pattern
.split('\\;') .split('\\;')
.reduce((attrs, attr, i) => { .reduce((attrs, attr, i) => {
@ -295,8 +287,8 @@ const Wappalyzer = {
}, },
analyzeOneToMany(technology, type, items = []) { analyzeOneToMany(technology, type, items = []) {
return items.reduce((technologies, { key, value }) => { return items.reduce((technologies, value) => {
const patterns = technology[type][key] || [] const patterns = technology[type] || []
patterns.forEach((pattern) => { patterns.forEach((pattern) => {
if (pattern.regex.test(value)) { if (pattern.regex.test(value)) {

@ -1,273 +0,0 @@
/* eslint-env mocha */
const { assert, expect } = require('chai')
const Wappalyzer = require('../src/wappalyzer')
const appsJson = {
appUrl: {
url: 'test'
},
appCookies: {
cookies: {
test: 'test'
}
},
appUppercaseCookies: {
cookies: {
Test: 'Test'
}
},
appHeaders: {
headers: {
'X-Powered-By': 'test'
}
},
appHtml: {
html: 'test v(\\d)\\;confidence:50\\;version:\\1',
implies: 'appImplies',
excludes: 'appExcludes'
},
appMeta: {
meta: {
generator: 'test'
}
},
appScript: {
script: 'test'
},
appJs: {
js: {
key: 'value'
}
},
appImplies: {},
appExcludes: {
html: 'test'
}
}
const driverData = {
cookies: [
{
name: 'test',
value: 'test',
domain: '',
path: ''
}
],
headers: {
'x-powered-by': ['test']
},
html: '<meta name="generator" content="test"> html test v1',
scripts: ['test'],
js: {
appJs: {
key: ['value']
}
}
}
describe('Wappalyzer', () => {
describe('#analyze()', () => {
let apps
before(async () => {
const wappalyzer = new Wappalyzer()
wappalyzer.apps = appsJson
wappalyzer.parseJsPatterns()
wappalyzer.driver.displayApps = (detected) => {
apps = detected
}
await wappalyzer.analyze({ canonical: 'test' }, driverData)
})
it('should identify technologies using URLs', () => {
expect(apps).to.have.any.keys('appUrl')
})
it('should identify technologies using HTML', () => {
expect(apps).to.have.any.keys('appHtml')
})
it('should identify technologies using meta tags', () => {
expect(apps).to.have.any.keys('appMeta')
})
it('should identify technologies using script URLs', () => {
expect(apps).to.have.any.keys('appScript')
})
it('should identify technologies using headers', () => {
expect(apps).to.have.any.keys('appHeaders')
})
it('should identify technologies using cookies', () => {
expect(apps).to.have.any.keys('appCookies')
})
it('should identify technologies using uppercase named cookies', () => {
expect(apps).to.have.any.keys('appUppercaseCookies')
})
it('should identify technologies using JavaScript', () => {
expect(apps).to.have.any.keys('appJs')
})
it('should return the implied technology', () => {
expect(apps).to.have.any.keys('appImplies')
})
it('should not return the excluded technology', () => {
expect(apps).to.not.have.any.keys('appExcludes')
})
it('should return the confidence value', () => {
assert.equal(apps.appHtml.confidenceTotal, 50)
})
it('should return the version number', () => {
assert.equal(apps.appHtml.version, '1')
})
it('should analyze html', async () => {
const html = `
<!DOCTYPE HTML>
<html>
<head>
<title>Page title | Html detection </title>
<meta charset="utf-8" />
</head>
<body>
<h1>Technologies Test Page | Html detection</h1>
<!-- Google Tag Manager -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KAAOEOE"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager -->
</body>
</html>
`
const wappalyzer = new Wappalyzer()
wappalyzer.apps = {
'Google Tag Manager': {
html: [
'googletagmanager\\.com/ns\\.html[^>]+></iframe>',
'<!-- (?:End )?Google Tag Manager -->'
]
}
}
let applications = null
wappalyzer.driver = {
log() {},
displayApps(detectedMap) {
applications = detectedMap
}
}
await wappalyzer.analyze({ canonical: 'example.com' }, { html })
assert.equal(
applications['Google Tag Manager'].name,
'Google Tag Manager'
)
})
it('should analyze scripts', async () => {
const scripts = [
'http://www.google-analytics.com/analytics.js',
'http://example.com/assets/js/jquery.min.js'
]
const wappalyzer = new Wappalyzer()
wappalyzer.apps = {
'Google Analytics': {
cats: [10],
script:
'google-analytics\\.com\\/(?:ga|urchin|(analytics))\\.js\\;version:\\1?UA:'
},
jQuery: {
script: [
'jquery(?:\\-|\\.)([\\d.]*\\d)[^/]*\\.js\\;version:\\1',
'/([\\d.]+)/jquery(?:\\.min)?\\.js\\;version:\\1',
'jquery.*\\.js(?:\\?ver(?:sion)?=([\\d.]+))?\\;version:\\1'
]
}
}
let applications = null
wappalyzer.driver = {
log() {},
displayApps(detectedMap) {
applications = detectedMap
}
}
await wappalyzer.analyze({ canonical: 'example.com' }, { scripts })
assert.equal(applications['Google Analytics'].name, 'Google Analytics')
assert.equal(applications.jQuery.name, 'jQuery')
})
it('should analyze headers', async () => {
const headers = {
date: ['Thu, 01 Feb 2018 11:34:18 GMT'],
connection: ['keep-alive'],
'x-powered-by': ['Express'],
etag: ['W/125-1jQLmiya7mfec43xR3Eb3pjdu64s'],
'content-length': ['293'],
'content-type': ['text/html; charset=utf-8']
}
const wappalyzer = new Wappalyzer()
wappalyzer.apps = {
Express: {
headers: {
'X-Powered-By': '^Express$'
}
}
}
let applications = null
wappalyzer.driver = {
log() {},
displayApps(detectedMap) {
applications = detectedMap
}
}
await wappalyzer.analyze({ canonical: 'example.com' }, { headers })
assert.equal(applications.Express.name, 'Express')
})
it('should analyze js globals', async () => {
const js = {
'Moment.js': { moment: { '0': true } },
'Google Font API': { WebFonts: { '0': true } }
}
const wappalyzer = new Wappalyzer()
wappalyzer.apps = {
'Moment.js': {
js: {
moment: '',
'moment.version': '(.*)\\;version:\\1'
}
},
'Google Font API': {
js: {
WebFonts: ''
}
}
}
let applications = null
wappalyzer.driver = {
log() {},
displayApps(detectedMap) {
applications = detectedMap
}
}
wappalyzer.parseJsPatterns()
await wappalyzer.analyze({ canonical: 'example.com' }, { js })
assert.equal(applications['Google Font API'].name, 'Google Font API')
assert.equal(applications['Moment.js'].name, 'Moment.js')
})
})
})