|
|
@ -116,11 +116,7 @@ class Wappalyzer {
|
|
|
|
|
|
|
|
|
|
|
|
this.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
|
|
|
|
this.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
|
|
|
|
.then(robotsTxt => {
|
|
|
|
.then(robotsTxt => {
|
|
|
|
robotsTxt.forEach(disallow => {
|
|
|
|
robotsTxt.forEach(disallow => parsed.pathname.indexOf(disallow) === 0 && reject());
|
|
|
|
if ( parsed.pathname.indexOf(disallow) === 0 ) {
|
|
|
|
|
|
|
|
reject();
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
resolve();
|
|
|
|
resolve();
|
|
|
|
});
|
|
|
|
});
|
|
|
@ -352,7 +348,7 @@ class Wappalyzer {
|
|
|
|
this.hostnameCache[hostname].applications[appName].version = app.version;
|
|
|
|
this.hostnameCache[hostname].applications[appName].version = app.version;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
})
|
|
|
|
.catch(() => console.log('Disallowed in robots.txt: ' + url))
|
|
|
|
.catch(() => this.log('Disallowed in robots.txt: ' + url), 'core')
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
|
|
|