|
|
@ -300,16 +300,21 @@ class Site {
|
|
|
|
|
|
|
|
|
|
|
|
promiseTimeout(
|
|
|
|
promiseTimeout(
|
|
|
|
promise,
|
|
|
|
promise,
|
|
|
|
|
|
|
|
fallback,
|
|
|
|
errorMessage = 'The website took too long to respond'
|
|
|
|
errorMessage = 'The website took too long to respond'
|
|
|
|
) {
|
|
|
|
) {
|
|
|
|
let timeout = null
|
|
|
|
let timeout = null
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (!(promise instanceof Promise)) {
|
|
|
|
|
|
|
|
return Promise.resolve(promise)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
return Promise.race([
|
|
|
|
return Promise.race([
|
|
|
|
new Promise((resolve, reject) => {
|
|
|
|
new Promise((resolve, reject) => {
|
|
|
|
timeout = setTimeout(() => {
|
|
|
|
timeout = setTimeout(() => {
|
|
|
|
clearTimeout(timeout)
|
|
|
|
clearTimeout(timeout)
|
|
|
|
|
|
|
|
|
|
|
|
reject(new Error(errorMessage))
|
|
|
|
fallback ? resolve(fallback) : reject(new Error(errorMessage))
|
|
|
|
}, this.options.maxWait)
|
|
|
|
}, this.options.maxWait)
|
|
|
|
}),
|
|
|
|
}),
|
|
|
|
promise.then((value) => {
|
|
|
|
promise.then((value) => {
|
|
|
@ -469,10 +474,12 @@ class Site {
|
|
|
|
rel,
|
|
|
|
rel,
|
|
|
|
})
|
|
|
|
})
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
),
|
|
|
|
|
|
|
|
{ jsonValue: () => [] }
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
).jsonValue(),
|
|
|
|
|
|
|
|
[]
|
|
|
|
)
|
|
|
|
)
|
|
|
|
).catch(() => ({ jsonValue: () => [] }))
|
|
|
|
|
|
|
|
).jsonValue()
|
|
|
|
|
|
|
|
).catch(() => [])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// CSS
|
|
|
|
// CSS
|
|
|
|
const css = await this.promiseTimeout(
|
|
|
|
const css = await this.promiseTimeout(
|
|
|
@ -500,10 +507,12 @@ class Site {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
return css.join('\n')
|
|
|
|
return css.join('\n')
|
|
|
|
}, this.options.htmlMaxRows)
|
|
|
|
}, this.options.htmlMaxRows),
|
|
|
|
).catch(() => ({ jsonValue: () => '' }))
|
|
|
|
{ jsonValue: () => '' }
|
|
|
|
).jsonValue()
|
|
|
|
)
|
|
|
|
).catch(() => '')
|
|
|
|
).jsonValue(),
|
|
|
|
|
|
|
|
''
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
// Script tags
|
|
|
|
// Script tags
|
|
|
|
const scripts = await this.promiseTimeout(
|
|
|
|
const scripts = await this.promiseTimeout(
|
|
|
@ -513,10 +522,12 @@ class Site {
|
|
|
|
Array.from(document.getElementsByTagName('script'))
|
|
|
|
Array.from(document.getElementsByTagName('script'))
|
|
|
|
.map(({ src }) => src)
|
|
|
|
.map(({ src }) => src)
|
|
|
|
.filter((src) => src)
|
|
|
|
.filter((src) => src)
|
|
|
|
|
|
|
|
),
|
|
|
|
|
|
|
|
{ jsonValue: () => [] }
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
).jsonValue(),
|
|
|
|
|
|
|
|
[]
|
|
|
|
)
|
|
|
|
)
|
|
|
|
).catch(() => ({ jsonValue: () => [] }))
|
|
|
|
|
|
|
|
).jsonValue()
|
|
|
|
|
|
|
|
).catch(() => [])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Meta tags
|
|
|
|
// Meta tags
|
|
|
|
const meta = await this.promiseTimeout(
|
|
|
|
const meta = await this.promiseTimeout(
|
|
|
@ -536,10 +547,12 @@ class Site {
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{}
|
|
|
|
{}
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
),
|
|
|
|
|
|
|
|
{ jsonValue: () => [] }
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
).jsonValue(),
|
|
|
|
|
|
|
|
[]
|
|
|
|
)
|
|
|
|
)
|
|
|
|
).catch(() => ({ jsonValue: () => [] }))
|
|
|
|
|
|
|
|
).jsonValue()
|
|
|
|
|
|
|
|
).catch(() => [])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// JavaScript
|
|
|
|
// JavaScript
|
|
|
|
const js = await this.promiseTimeout(
|
|
|
|
const js = await this.promiseTimeout(
|
|
|
@ -574,8 +587,9 @@ class Site {
|
|
|
|
Wappalyzer.technologies
|
|
|
|
Wappalyzer.technologies
|
|
|
|
.filter(({ js }) => Object.keys(js).length)
|
|
|
|
.filter(({ js }) => Object.keys(js).length)
|
|
|
|
.map(({ name, js }) => ({ name, chains: Object.keys(js) }))
|
|
|
|
.map(({ name, js }) => ({ name, chains: Object.keys(js) }))
|
|
|
|
|
|
|
|
),
|
|
|
|
|
|
|
|
[]
|
|
|
|
)
|
|
|
|
)
|
|
|
|
).catch(() => [])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// DOM
|
|
|
|
// DOM
|
|
|
|
const dom = await this.promiseTimeout(
|
|
|
|
const dom = await this.promiseTimeout(
|
|
|
@ -667,8 +681,9 @@ class Site {
|
|
|
|
Wappalyzer.technologies
|
|
|
|
Wappalyzer.technologies
|
|
|
|
.filter(({ dom }) => dom && dom.constructor === Object)
|
|
|
|
.filter(({ dom }) => dom && dom.constructor === Object)
|
|
|
|
.map(({ name, dom }) => ({ name, dom }))
|
|
|
|
.map(({ name, dom }) => ({ name, dom }))
|
|
|
|
|
|
|
|
),
|
|
|
|
|
|
|
|
[]
|
|
|
|
)
|
|
|
|
)
|
|
|
|
).catch(() => [])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Cookies
|
|
|
|
// Cookies
|
|
|
|
const cookies = (await page.cookies()).reduce(
|
|
|
|
const cookies = (await page.cookies()).reduce(
|
|
|
@ -716,8 +731,9 @@ class Site {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
return []
|
|
|
|
return []
|
|
|
|
})
|
|
|
|
}),
|
|
|
|
).catch(() => [])
|
|
|
|
[]
|
|
|
|
|
|
|
|
)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
const domain = url.hostname.replace(/^www\./, '')
|
|
|
|
const domain = url.hostname.replace(/^www\./, '')
|
|
|
|