Migrations
/
wappalyzer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #917 from passmarked/npm-driver-detectFromHTML

Browse Source 
NPM driver detect from html
main
Elbert Alias 9 years ago
parent 13bb3cd212 938ccca753
commit ee9b4e5b0d
5 changed files with 278 additions and 19 deletions
Show Stats Download Patch File Download Diff File

86
src/drivers/npm/index.js
Unescape View File

@ -4,26 +4,65 @@ var request = require('request');
var fs = require('fs'); var fs = require('fs');
var path = require('path'); var path = require('path');
//TODO /**
exports.detectFromHTML = function(options) {}; * Does the actual detection with information passed
**/
exports.detect = function(options, data, cb) {
// run the wrapper function that will
// trigger the actual library to run
runWrappalyer(options, data, cb);
};
/**
* Wraps the detect method, just kept to reuse old method names
* and not break anything. Although this was just stubbed out.
**/
exports.detectFromHTML = function(options, data, cb) {
// run the detect method
exports.detect(options, data, cb);
};
/**
* Do a actual request for the body & headers, then
* run through detection
**/
exports.detectFromUrl = function(options, cb) { exports.detectFromUrl = function(options, cb) {
var url = options.url; // ensure options and url were
if(!options || !options.url) {
if (options.debug) { // send back a error ...
console.log('Fetching the page'); cb(new Error("\"url\" is a required option to run"
} + " wappalyzer and get the page content"))
getHTMLFromUrl(url, function(err, data) { } else {
if (err || data === null) {
cb(err, null); // local variables to
} else { var url = options.url;
runWrappalyer(options, data, function(err, detected, appInfo) {
cb(null, detected, appInfo); // get the body content from the url
}); getHTMLFromUrl(url, function(err, data) {
}
}); // check for error or if we got no data back
if (err || data === null) {
// handle the error and don't do anything else ..
cb(err, null);
} else {
// run actual detection
exports.detect(options, data, cb);
}
});
}
}; };
function getHTMLFromUrl(url, cb) { function getHTMLFromUrl(url, cb) {
@ -42,7 +81,7 @@ function getHTMLFromUrl(url, cb) {
} }
function getAppsJson(cb) { function getAppsJson(cb) {
fs.readFile(path.resolve(__dirname, 'apps.json'), 'utf8', function(err, data) { fs.readFile(path.resolve(__dirname, '../../apps.json'), 'utf8', function(err, data) {
if (err) throw err; if (err) throw err;
return cb(null, JSON.parse(data)); return cb(null, JSON.parse(data));
}); });
@ -51,7 +90,20 @@ function getAppsJson(cb) {
function runWrappalyer(options, data, cb) { function runWrappalyer(options, data, cb) {
var debug = options.debug || false; var debug = options.debug || false;
var wappalyzer = require('./wappalyzer').wappalyzer; // according to environment check it
var wappalyzer = null;
// change depending on the environment
if( process.env.NODE_ENV ) {
wappalyzer = require('../../wappalyzer').wappalyzer;
} else {
wappalyzer = require('./wappalyzer').wappalyzer;
}
getAppsJson(function(err, apps) { getAppsJson(function(err, apps) {
var w = wappalyzer; var w = wappalyzer;
w.driver = { w.driver = {

2
src/drivers/npm/package.json
Unescape View File

@ -4,7 +4,7 @@
"description": "NPM Module that uncovers the technologies used on websites", "description": "NPM Module that uncovers the technologies used on websites",
"main": "index.js", "main": "index.js",
"scripts": { "scripts": {
"test": "mocha -t 15000" "test": "NODE_ENV=testing mocha -t 15000"
}, },
"repository": { "repository": {
"type": "git", "type": "git",

20
src/drivers/npm/test/sample.headers.json
Unescape View File

@ -0,0 +1,20 @@
{
"Date":"Fri, 17 Jul 2015 11:05:08 GMT",
"Content-Encoding":"gzip",
"Age":"0",
"X-Cache":"MISS",
"Connection":"keep-alive",
"Content-Length":"1940",
"Via":"1.1 varnish",
"X-Served-By":"cache-lcy1122-LCY",
"Last-Modified":"Thu, 16 Apr 2015 14:24:18 GMT",
"Server":"GitHub.com",
"X-Timer":"S1437131108.300792,VS0,VE84",
"Vary":"Accept-Encoding",
"Content-Type":"text/html; charset=utf-8",
"Access-Control-Allow-Origin":"*",
"Expires":"Fri, 17 Jul 2015 11:10:54 GMT",
"Cache-Control":"max-age=600",
"Accept-Ranges":"bytes",
"X-Cache-Hits":"0"
}

149
src/drivers/npm/test/sample.html
Unescape View File

@ -0,0 +1,149 @@
<!DOCTYPE html>
<html lang="en" ng-app="presen" >
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Preso Engines | CodeLanka</title>
<!-- Bootstrap CSS -->
<link href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
<link href="//cdnjs.cloudflare.com/ajax/libs/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet">
<script src="//code.jquery.com/jquery.js"></script>
<script src="//ajax.googleapis.com/ajax/libs/angularjs/1.3.5/angular.min.js"></script>
<link href='//fonts.googleapis.com/css?family=Ubuntu' rel='stylesheet' type='text/css'>
<link href="css/style.css" rel="stylesheet">
<script type="text/javascript">
$(function () {
"use strict";
var $bgobj = $(".ha-bg-parallax"); // assigning the object
$(window).on("scroll", function () {
var yPos = -($(window).scrollTop() / $bgobj.data('speed'));
// Put together our final background position
var coords = '100% ' + yPos + 'px';
// Move the background
$bgobj.css({ backgroundPosition: coords });
});
$('div.product-chooser').not('.disabled').find('div.product-chooser-item').on('click', function(){
$(this).parent().parent().find('div.product-chooser-item').removeClass('selected');
$(this).addClass('selected');
$(this).find('input[type="radio"]').prop("checked", true);
});
});
</script>
</head>
<body>
<div class="container-fluid">
<div>
<div class="ha-bg-parallax text-center block-marginb-none" data-type="background" data-speed="20">
<div class="ha-parallax-body">
<div class="ha-content ha-content-whitecolor">
<h1 id="headernav">Preso Engines</h1>
"Presentation Engines"
</div>
<div class="ha-parallax-divider-wrapper">
<span class="ha-diamond-divider-md img-responsive"></span>
</div>
<div class="ha-heading-parallax">By | CodeLanka</div>
</div>
</div>
<div class="container" ng-controller="presentationController">
<h2></h2>
<div class="row form-group product-chooser">
<div class="col-xs-12 col-sm-12 col-md-4 col-lg-4"
ng-repeat="presentation in presentations">
<div class="product-chooser-item">
<div class="col-xs-8 col-sm-8 col-md-12 col-lg-12">
<hr/>
<span class="title">
{{presentation.name}}
</span>
<span class="website">
<a href="{{presentation.website}}">{{presentation.website}}</a>
</span>
<span class="description">
{{presentation.description}}</span>
<input type="radio" name="product" value="mobile_desktop"
checked="checked">
<hr/>
<div class="col-xs-4 col-sm-4 col-md-4 col-lg-4 text-center">
<i class="fa fa-star"></i>
<h2>{{presentation.stargazers_count}}</h2>
</div>
<div class="col-xs-4 col-sm-4 col-md-4 col-lg-4 text-center">
<i class="fa fa-users"></i>
<h2>{{presentation.watchers_count}}</h2>
</div>
<div class="col-xs-4 col-sm-4 col-md-4 col-lg-4 text-center">
<i class="fa fa-bug"></i>
<h2>{{presentation.open_issues_count}}</h2>
</div>
<div class="clear"></div>
<hr>
<p>Github -
<a href="http://github.com/{{presentation.github}}">{{presentation.github}}</a>
</p>
<p>Demo -
<a href="{{presentation.demo}}">{{presentation.demo}}</a>
</p>
<p>License - {{presentation.license}} </p>
<p>Language - {{presentation.language}} </p>
<hr/>
</div>
<div class="clear"></div>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- jQuery -->
<!-- Bootstrap JavaScript -->
<script src="//netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js"></script>
<script>
var phonecatApp = angular.module('presen', []);
phonecatApp.controller('presentationController', function ($scope,$http) {
var github = "https://api.github.com/repos/";
$http.get('data.json').
success(function(data, status, headers, config) {
$scope.presentations=data.engines;
for (var i = data.engines.length - 1; i >= 0; i--) {
$scope.presentations[i].stargazers_count=0;
$scope.presentations[i].watchers_count=0;
$scope.presentations[i].open_issues_count=0;
(function(no){
var githuburl = github+data.engines[no].github;
$http.get(githuburl)
.success(function(data, status, headers, config) {
$scope.presentations[no].stargazers_count=data.stargazers_count;
$scope.presentations[no].watchers_count=data.watchers_count;
$scope.presentations[no].open_issues_count=data.open_issues_count;
})
.error(function(data, status, headers, config) {});
}(i));
};
}).
error(function(data, status, headers, config) {});
});
</script>
</body>
</html>

40
src/drivers/npm/test/test.js
Unescape View File

@ -1,4 +1,5 @@
var assert = require("assert") var assert = require("assert")
var fs = require('fs')
describe('wappalyzer', function(){ describe('wappalyzer', function(){
describe('detectFromUrl', function(){ describe('detectFromUrl', function(){
@ -24,4 +25,41 @@ describe('wappalyzer', function(){
}) })
}) })
describe('detectFromHTML', function(){
it('should have the expected apps detected when passed raw info', function(done){
var wappalyzer = require("../index");
var expect = ['AngularJS','Font Awesome','Google Font API','Twitter Bootstrap','jQuery'];
var options={
url : "http://codelanka.github.io/Presentation-Engines",
hostname:"codelanka.github.io",
debug:false
}
var data = {
url: options.url,
headers: require('./sample.headers.json'),
html: fs.readFileSync('./test/sample.html').toString(),
headers: {
headers: {}
}
};
wappalyzer.detectFromHTML(options, data, function (err,apps) {
assert.equal(expect[0], apps[0]);
assert.equal(expect[1], apps[1]);
assert.equal(expect[2], apps[2]);
assert.equal(expect[3], apps[3]);
done();
})
})
})
}) })

Write Preview
Loading…
Cancel
Save