Update hostname ignore list

main
Elbert Alias 4 years ago
parent 80b5e8b0b3
commit 42ea89947b

@ -13,6 +13,8 @@ const { agent, promisify, getOption, setOption, open } = Utils
const expiry = 1000 * 60 * 60 * 24
const hostnameIgnoreList = /((local|dev(elop(ment)?)?|stag(e|ing)?|preprod|test(ing)?|demo(shop)?|admin|cache)[.-]|localhost|wappalyzer|google|facebook|twitter|reddit|yahoo|wikipedia|amazon|youtube|\/admin|\.local|\.test|\.dev|127\.|0\.)/
const Driver = {
lastPing: Date.now(),
@ -534,7 +536,10 @@ const Driver = {
* @param {Boolean} secure
*/
async getRobots(hostname, secure = false) {
if (!(await getOption('tracking', true))) {
if (
!(await getOption('tracking', true)) ||
hostnameIgnoreList.test(hostname)
) {
return
}
@ -649,12 +654,7 @@ const Driver = {
hostname
]
if (
!/((local|dev(elop(ment)?)?|stag(e|ing)?|preprod|test(ing)?|demo(shop)?|admin|cache)[.-]|localhost|google|\/admin|\.local|\.test|\.dev|127\.|0\.)/.test(
hostname
) &&
hits >= 3
) {
if (!hostnameIgnoreList.test(hostname) && hits >= 3) {
hostnames[url] = hostnames[url] || {
applications: resolve(detections).reduce(
(technologies, { name, confidence, version }) => {