|
|
@ -175,6 +175,7 @@ class Driver {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
})
|
|
|
|
} catch (error) {
|
|
|
|
} catch (error) {
|
|
|
|
|
|
|
|
console.log('a', error)
|
|
|
|
throw new Error(error.toString())
|
|
|
|
throw new Error(error.toString())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -190,6 +191,7 @@ class Driver {
|
|
|
|
|
|
|
|
|
|
|
|
this.log('Browser closed')
|
|
|
|
this.log('Browser closed')
|
|
|
|
} catch (error) {
|
|
|
|
} catch (error) {
|
|
|
|
|
|
|
|
console.log('b', error)
|
|
|
|
throw new Error(error.toString())
|
|
|
|
throw new Error(error.toString())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -225,6 +227,7 @@ class Site {
|
|
|
|
try {
|
|
|
|
try {
|
|
|
|
this.originalUrl = new URL(url)
|
|
|
|
this.originalUrl = new URL(url)
|
|
|
|
} catch (error) {
|
|
|
|
} catch (error) {
|
|
|
|
|
|
|
|
console.log('c', error)
|
|
|
|
throw new Error(error.toString())
|
|
|
|
throw new Error(error.toString())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -345,6 +348,7 @@ class Site {
|
|
|
|
request.continue({ headers })
|
|
|
|
request.continue({ headers })
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (error) {
|
|
|
|
} catch (error) {
|
|
|
|
|
|
|
|
console.log('d', error)
|
|
|
|
this.error(error)
|
|
|
|
this.error(error)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
})
|
|
|
@ -387,6 +391,7 @@ class Site {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (error) {
|
|
|
|
} catch (error) {
|
|
|
|
|
|
|
|
console.log('e', error)
|
|
|
|
this.error(error)
|
|
|
|
this.error(error)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
})
|
|
|
@ -690,7 +695,11 @@ class Site {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Validate response
|
|
|
|
// Validate response
|
|
|
|
if (url.protocol !== 'file:' && !this.analyzedUrls[url.href].status) {
|
|
|
|
if (
|
|
|
|
|
|
|
|
url.protocol !== 'file:' &&
|
|
|
|
|
|
|
|
this.analyzedUrls[url.href] &&
|
|
|
|
|
|
|
|
!!this.analyzedUrls[url.href].status
|
|
|
|
|
|
|
|
) {
|
|
|
|
await page.close()
|
|
|
|
await page.close()
|
|
|
|
|
|
|
|
|
|
|
|
this.log('Page closed')
|
|
|
|
this.log('Page closed')
|
|
|
@ -751,6 +760,7 @@ class Site {
|
|
|
|
|
|
|
|
|
|
|
|
return reducedLinks
|
|
|
|
return reducedLinks
|
|
|
|
} catch (error) {
|
|
|
|
} catch (error) {
|
|
|
|
|
|
|
|
console.log('f', error)
|
|
|
|
if (error.constructor.name === 'TimeoutError') {
|
|
|
|
if (error.constructor.name === 'TimeoutError') {
|
|
|
|
throw new Error('The website took too long to respond')
|
|
|
|
throw new Error('The website took too long to respond')
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -779,6 +789,7 @@ class Site {
|
|
|
|
await this.batch(links.slice(0, this.options.maxUrls), depth + 1)
|
|
|
|
await this.batch(links.slice(0, this.options.maxUrls), depth + 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (error) {
|
|
|
|
} catch (error) {
|
|
|
|
|
|
|
|
console.log('g', error)
|
|
|
|
this.analyzedUrls[url.href] = {
|
|
|
|
this.analyzedUrls[url.href] = {
|
|
|
|
status: 0,
|
|
|
|
status: 0,
|
|
|
|
error: error.message || error.toString(),
|
|
|
|
error: error.message || error.toString(),
|
|
|
|