|
|
@ -196,10 +196,11 @@ class Site {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
timeout() {
|
|
|
|
timeout() {
|
|
|
|
return new Promise(() =>
|
|
|
|
return new Promise((resolve, reject) =>
|
|
|
|
setTimeout(() => {
|
|
|
|
setTimeout(
|
|
|
|
throw new Error('The website took too long to respond')
|
|
|
|
() => reject(new Error('The website took too long to respond')),
|
|
|
|
}, this.options.maxWait)
|
|
|
|
this.options.maxWait
|
|
|
|
|
|
|
|
)
|
|
|
|
)
|
|
|
|
)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -300,7 +301,7 @@ class Site {
|
|
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
try {
|
|
|
|
await Promise.race([
|
|
|
|
await Promise.race([
|
|
|
|
this.timeout(),
|
|
|
|
await this.timeout(),
|
|
|
|
page.goto(url.href, { waitUntil: 'domcontentloaded' })
|
|
|
|
page.goto(url.href, { waitUntil: 'domcontentloaded' })
|
|
|
|
])
|
|
|
|
])
|
|
|
|
|
|
|
|
|
|
|
@ -308,8 +309,8 @@ class Site {
|
|
|
|
|
|
|
|
|
|
|
|
// Links
|
|
|
|
// Links
|
|
|
|
const links = await Promise.race([
|
|
|
|
const links = await Promise.race([
|
|
|
|
this.timeout(),
|
|
|
|
await this.timeout(),
|
|
|
|
await (
|
|
|
|
(
|
|
|
|
await page.evaluateHandle(() =>
|
|
|
|
await page.evaluateHandle(() =>
|
|
|
|
Array.from(document.getElementsByTagName('a')).map(
|
|
|
|
Array.from(document.getElementsByTagName('a')).map(
|
|
|
|
({ hash, hostname, href, pathname, protocol, rel }) => ({
|
|
|
|
({ hash, hostname, href, pathname, protocol, rel }) => ({
|
|
|
@ -327,8 +328,8 @@ class Site {
|
|
|
|
|
|
|
|
|
|
|
|
// Script tags
|
|
|
|
// Script tags
|
|
|
|
const scripts = await Promise.race([
|
|
|
|
const scripts = await Promise.race([
|
|
|
|
this.timeout(),
|
|
|
|
await this.timeout(),
|
|
|
|
await (
|
|
|
|
(
|
|
|
|
await page.evaluateHandle(() =>
|
|
|
|
await page.evaluateHandle(() =>
|
|
|
|
Array.from(document.getElementsByTagName('script'))
|
|
|
|
Array.from(document.getElementsByTagName('script'))
|
|
|
|
.map(({ src }) => src)
|
|
|
|
.map(({ src }) => src)
|
|
|
@ -339,8 +340,8 @@ class Site {
|
|
|
|
|
|
|
|
|
|
|
|
// Meta tags
|
|
|
|
// Meta tags
|
|
|
|
const meta = await Promise.race([
|
|
|
|
const meta = await Promise.race([
|
|
|
|
this.timeout(),
|
|
|
|
await this.timeout(),
|
|
|
|
await (
|
|
|
|
(
|
|
|
|
await page.evaluateHandle(() =>
|
|
|
|
await page.evaluateHandle(() =>
|
|
|
|
Array.from(document.querySelectorAll('meta')).reduce(
|
|
|
|
Array.from(document.querySelectorAll('meta')).reduce(
|
|
|
|
(metas, meta) => {
|
|
|
|
(metas, meta) => {
|
|
|
@ -361,8 +362,8 @@ class Site {
|
|
|
|
|
|
|
|
|
|
|
|
// JavaScript
|
|
|
|
// JavaScript
|
|
|
|
const js = await Promise.race([
|
|
|
|
const js = await Promise.race([
|
|
|
|
this.timeout(),
|
|
|
|
await this.timeout(),
|
|
|
|
await page.evaluate(
|
|
|
|
page.evaluate(
|
|
|
|
(technologies) => {
|
|
|
|
(technologies) => {
|
|
|
|
return technologies.reduce((technologies, { name, chains }) => {
|
|
|
|
return technologies.reduce((technologies, { name, chains }) => {
|
|
|
|
chains.forEach((chain) => {
|
|
|
|
chains.forEach((chain) => {
|
|
|
@ -441,8 +442,8 @@ class Site {
|
|
|
|
|
|
|
|
|
|
|
|
if (!this.language) {
|
|
|
|
if (!this.language) {
|
|
|
|
this.language = await Promise.race([
|
|
|
|
this.language = await Promise.race([
|
|
|
|
this.timeout(),
|
|
|
|
await this.timeout(),
|
|
|
|
await (
|
|
|
|
(
|
|
|
|
await page.evaluateHandle(
|
|
|
|
await page.evaluateHandle(
|
|
|
|
() =>
|
|
|
|
() =>
|
|
|
|
document.documentElement.getAttribute('lang') ||
|
|
|
|
document.documentElement.getAttribute('lang') ||
|
|
|
|