diff --git a/src/apps.json b/src/apps.json index 07c2025ae..3024e0e9c 100644 --- a/src/apps.json +++ b/src/apps.json @@ -1016,6 +1016,31 @@ ], "website": "http://www.ozerov.de/bigdump.php" }, + "Ant Design": { + "cats": [ + "12" + ], + "implies": [ + "React" + ], + "icon": "Ant Design.svg", + "env": "^antd$", + "html": [ + "<(?:div|button) class=\"ant-(?:btn|col|row|layout|breadcrumb|menu|pagination|steps|select|cascader|checkbox|calendar|form|input-number|input|mention|rate|radio|slider|switch|tree-select|time-picker|transfer|upload|avatar|badge|card|carousel|collapse|list|popover|tooltip|table|tabs|tag|timeline|tree|alert|modal|message|notification|progress|popconfirm|spin|anchor|back-top|divider)", + "]* href=[^>]+tilda(?:cdn|\\.ws|-blocks)", "script": "tilda(?:cdn|\\.ws|-blocks)", "website": "https://tilda.cc" diff --git a/src/drivers/bookmarklet/driver.js b/src/drivers/bookmarklet/driver.js index 1bcb3783c..7725b9a5f 100644 --- a/src/drivers/bookmarklet/driver.js +++ b/src/drivers/bookmarklet/driver.js @@ -6,9 +6,10 @@ /** global: XMLHttpRequest */ (function() { + wappalyzer.driver.document = document; + const container = document.getElementById('wappalyzer-container'); - const domain = top.location.host; - const url = top.location.href.replace(/#.*$/, ''); + const url = wappalyzer.parseUrl(top.location.href); const hasOwn = Object.prototype.hasOwnProperty; /** @@ -19,7 +20,7 @@ }; function getPageContent() { - wappalyzer.log('func: getPageContent'); + wappalyzer.log('func: getPageContent', 'driver'); var env = []; @@ -32,7 +33,7 @@ .filter(s => s.src) .map(s => s.src); - wappalyzer.analyze(domain, url, { + wappalyzer.analyze(url, { html: document.documentElement.innerHTML, env: env, scripts: scripts @@ -40,7 +41,7 @@ } function getResponseHeaders() { - wappalyzer.log('func: getResponseHeaders'); + wappalyzer.log('func: getResponseHeaders', 'driver'); var xhr = new XMLHttpRequest(); @@ -51,7 +52,7 @@ var headers = xhr.getAllResponseHeaders().split("\n"); if ( headers.length > 0 && headers[0] != '' ) { - wappalyzer.log('responseHeaders: ' + xhr.getAllResponseHeaders()); + wappalyzer.log('responseHeaders: ' + xhr.getAllResponseHeaders(), 'driver'); var responseHeaders = {}; @@ -69,7 +70,7 @@ } }); - wappalyzer.analyze(domain, url, { + wappalyzer.analyze(url, { headers: responseHeaders }); } @@ -83,7 +84,7 @@ * Display apps */ wappalyzer.driver.displayApps = detected => { - wappalyzer.log('func: diplayApps'); + wappalyzer.log('func: diplayApps', 'driver'); var first = true; var app; @@ -104,14 +105,14 @@ var version = detected[app].version, confidence = detected[app].confidence; - + html += '
' + '' + '' + ' ' + app + '' + - ( version ? ' ' + version : '' ) + ( confidence < 100 ? ' (' + confidence + '% sure)' : '' ) + + ( version ? ' ' + version : '' ) + ( confidence < 100 ? ' (' + confidence + '% sure)' : '' ) + ''; for ( let i in wappalyzer.apps[app].cats ) { diff --git a/src/drivers/npm/browsers/zombie.js b/src/drivers/npm/browsers/zombie.js new file mode 100644 index 000000000..399999089 --- /dev/null +++ b/src/drivers/npm/browsers/zombie.js @@ -0,0 +1,313 @@ +'use strict'; + +const Wappalyzer = require('./wappalyzer'); +const request = require('request'); +const url = require('url'); +const fs = require('fs'); +const Browser = require('zombie'); + +const json = JSON.parse(fs.readFileSync(__dirname + '/apps.json')); + +const extensions = /^([^.]+$|\.(asp|aspx|cgi|htm|html|jsp|php)$)/; + +class Driver { + constructor(pageUrl, options) { + this.options = Object.assign({}, { + chunkSize: 5, + debug: false, + delay: 500, + maxDepth: 3, + maxUrls: 10, + maxWait: 5000, + recursive: false, + userAgent: 'Mozilla/5.0 (compatible; Wappalyzer)', + }, options || {}); + + this.options.debug = Boolean(this.options.debug); + this.options.delay = this.options.recursive ? parseInt(this.options.delay, 10) : 0; + this.options.maxDepth = parseInt(this.options.maxDepth, 10); + this.options.maxUrls = parseInt(this.options.maxUrls, 10); + this.options.maxWait = parseInt(this.options.maxWait, 10); + this.options.recursive = Boolean(this.options.recursive); + + this.origPageUrl = url.parse(pageUrl); + this.analyzedPageUrls = []; + this.apps = []; + this.meta = {}; + + this.wappalyzer = new Wappalyzer(); + + this.wappalyzer.apps = json.apps; + this.wappalyzer.categories = json.categories; + + this.wappalyzer.parseJsPatterns(); + + this.wappalyzer.driver.log = (message, source, type) => this.log(message, source, type); + this.wappalyzer.driver.displayApps = (detected, meta, context) => this.displayApps(detected, meta, context); + } + + analyze() { + this.time = { + start: new Date().getTime(), + last: new Date().getTime(), + } + + return this.crawl(this.origPageUrl); + } + + log(message, source, type) { + this.options.debug && console.log('[wappalyzer ' + type + ']', '[' + source + ']', message); + } + + displayApps(detected, meta) { + this.meta = meta; + + Object.keys(detected).forEach(appName => { + const app = detected[appName]; + + var categories = []; + + app.props.cats.forEach(id => { + var category = {}; + + category[id] = json.categories[id].name; + + categories.push(category) + }); + + if ( !this.apps.some(detectedApp => detectedApp.name === app.name) ) { + this.apps.push({ + name: app.name, + confidence: app.confidenceTotal.toString(), + version: app.version, + icon: app.props.icon || 'default.svg', + website: app.props.website, + categories + }); + } + }); + } + + fetch(pageUrl, index, depth) { + // Return when the URL is a duplicate or maxUrls has been reached + if ( this.analyzedPageUrls.indexOf(pageUrl.href) !== -1 || this.analyzedPageUrls.length >= this.options.maxUrls ) { + return Promise.resolve(); + } + + this.analyzedPageUrls.push(pageUrl.href); + + const timerScope = { + last: new Date().getTime() + }; + + this.timer('fetch; url: ' + pageUrl.href + '; depth: ' + depth + '; delay: ' + ( this.options.delay * index ) + 'ms', timerScope); + + return new Promise(resolve => this.sleep(this.options.delay * index).then(() => this.visit(pageUrl, timerScope, resolve))); + } + + visit(pageUrl, timerScope, resolve) { + const browser = new Browser({ + silent: true, + userAgent: this.options.userAgent, + waitDuration: this.options.maxWait, + }); + + this.timer('browser.visit start; url: ' + pageUrl.href, timerScope); + + browser.visit(pageUrl.href, () => { + this.timer('browser.visit end; url: ' + pageUrl.href, timerScope); + + if ( !this.responseOk(browser, pageUrl) ) { + return resolve(); + } + + const headers = this.getHeaders(browser); + const html = this.getHtml(browser); + const scripts = this.getScripts(browser); + const js = this.getJs(browser); + + this.wappalyzer.analyze(pageUrl, { + headers, + html, + scripts, + js + }); + + const links = Array.from(browser.document.getElementsByTagName('a')) + .filter(link => link.protocol === 'http:' || link.protocol === 'https:') + .filter(link => link.hostname === this.origPageUrl.hostname) + .filter(link => extensions.test(link.pathname)) + .map(link => { link.hash = ''; return url.parse(link.href) }); + + return resolve(links); + }); + } + + responseOk(browser, pageUrl) { + // Validate response + const resource = browser.resources.length ? browser.resources.filter(resource => resource.response).shift() : null; + + if ( !resource ) { + this.wappalyzer.log('No response from server; url: ' + pageUrl.href, 'driver', 'error'); + + return false; + } + + if ( resource.response.status !== 200 ) { + this.wappalyzer.log('Response was not OK; status: ' + resource.response.status + ' ' + resource.response.statusText + '; url: ' + pageUrl.href, 'driver', 'error'); + + return false; + } + + const headers = this.getHeaders(browser); + + // Validate content type + const contentType = headers.hasOwnProperty('content-type') ? headers['content-type'].shift() : null; + + if ( !contentType || !/\btext\/html\b/.test(contentType) ) { + this.wappalyzer.log('Skipping; url: ' + pageUrl.href + '; content type: ' + contentType, 'driver'); + + this.analyzedPageUrls.splice(this.analyzedPageUrls.indexOf(pageUrl.href), 1); + + return false; + } + + // Validate document + if ( !browser.document || !browser.document.documentElement ) { + this.wappalyzer.log('No HTML document; url: ' + pageUrl.href, 'driver', 'error'); + + return false; + } + + return true; + } + + getHeaders(browser) { + const headers = {}; + + const resource = browser.resources.length ? browser.resources.filter(resource => resource.response).shift() : null; + + if ( resource ) { + resource.response.headers._headers.forEach(header => { + if ( !headers[header[0]] ){ + headers[header[0]] = []; + } + + headers[header[0]].push(header[1]); + }); + } + + return headers; + } + + getHtml(browser) { + let html = ''; + + try { + html = browser.html(); + + if ( html.length > 50000 ) { + html = html.substring(0, 25000) + html.substring(html.length - 25000, html.length); + } + } catch ( error ) { + this.wappalyzer.log(error.message, 'browser', 'error'); + } + + return html; + } + + getScripts(browser) { + if ( !browser.document || !browser.document.scripts ) { + return []; + } + + const scripts = Array.prototype.slice + .apply(browser.document.scripts) + .filter(script => script.src) + .map(script => script.src); + + return scripts; + } + + getJs(browser) { + const patterns = this.wappalyzer.jsPatterns; + const js = {}; + + Object.keys(patterns).forEach(appName => { + js[appName] = {}; + + Object.keys(patterns[appName]).forEach(chain => { + js[appName][chain] = {}; + + patterns[appName][chain].forEach((pattern, index) => { + const properties = chain.split('.'); + + let value = properties.reduce((parent, property) => { + return parent && parent.hasOwnProperty(property) ? parent[property] : null; + }, browser.window); + + value = typeof value === 'string' ? value : !!value; + + if ( value ) { + js[appName][chain][index] = value; + } + }); + }); + }); + + return js; + } + + crawl(pageUrl, index, depth = 1) { + pageUrl.canonical = pageUrl.protocol + '//' + pageUrl.host + pageUrl.pathname; + + return new Promise(resolve => { + this.fetch(pageUrl, index, depth) + .catch(() => {}) + .then(links => { + if ( links && Boolean(this.options.recursive) && depth < this.options.maxDepth ) { + return this.chunk(links.slice(0, this.options.maxUrls), depth + 1); + } else { + return Promise.resolve(); + } + }) + .then(() => { + resolve({ + urls: this.analyzedPageUrls, + applications: this.apps, + meta: this.meta + }); + }); + }); + } + + chunk(links, depth, chunk = 0) { + if ( links.length === 0 ) { + return Promise.resolve(); + } + + const chunked = links.splice(0, this.options.chunkSize); + + return new Promise(resolve => { + Promise.all(chunked.map((link, index) => this.crawl(link, index, depth))) + .then(() => this.chunk(links, depth, chunk + 1)) + .then(() => resolve()); + }); + } + + sleep(ms) { + return ms ? new Promise(resolve => setTimeout(resolve, ms)) : Promise.resolve(); + } + + timer(message, scope) { + const time = new Date().getTime(); + const sinceStart = ( Math.round(( time - this.time.start ) / 10) / 100) + 's'; + const sinceLast = ( Math.round(( time - scope.last ) / 10) / 100) + 's'; + + this.wappalyzer.log('[timer] ' + message + '; lapsed: ' + sinceLast + ' / ' + sinceStart, 'driver'); + + scope.last = time; + } +}; + +module.exports = Driver; diff --git a/src/icons/Ant Design.svg b/src/icons/Ant Design.svg new file mode 100644 index 000000000..e9f8c2a9d --- /dev/null +++ b/src/icons/Ant Design.svg @@ -0,0 +1,43 @@ + + + + Group 28 Copy 5 + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/icons/Sqreen.png b/src/icons/Sqreen.png new file mode 100644 index 000000000..c4e1000e6 Binary files /dev/null and b/src/icons/Sqreen.png differ diff --git a/src/icons/Tilda.svg b/src/icons/Tilda.svg new file mode 100644 index 000000000..d3c193bcb --- /dev/null +++ b/src/icons/Tilda.svg @@ -0,0 +1,18 @@ + + + + Page 1 + Created with Sketch. + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/icons/tilda.png b/src/icons/tilda.png deleted file mode 100644 index 8d942ef57..000000000 Binary files a/src/icons/tilda.png and /dev/null differ