Migrations
/
wappalyzer
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix promise timeouts in NPM driver

Browse Source
main
Elbert Alias 4 years ago
parent 675150ad76
commit f5270fb2cf
1 changed files with 94 additions and 80 deletions
Show Stats Download Patch File Download Diff File

174
src/drivers/npm/driver.js
Unescape View File

@ -225,27 +225,26 @@ class Site {
}
}
/**
* @param {Promise} promise
* @param {string} [errorMessage]
*
* @return {Promise}
*/
async promiseTimeout(promise, errorMessage = 'The website took too long to respond') {
let timerId = null;
promiseTimeout(
promise,
errorMessage = 'The website took too long to respond'
) {
let timeout = null
return Promise.race([
new Promise((resolve, reject) => {
timerId = setTimeout(() => {
clearTimeout(timerId);
reject(new Error(errorMessage));
}, this.options.maxWait);
timeout = setTimeout(() => {
clearTimeout(timeout)
reject(new Error(errorMessage))
}, this.options.maxWait)
}),
promise.then((value) => {
clearTimeout(timerId);
return value;
clearTimeout(timeout)
return value
}),
]);
])
}
async goto(url) {
@ -352,87 +351,98 @@ class Site {
)
try {
await this.promiseTimeout(page.goto(url.href, { waitUntil: 'domcontentloaded' }))
await this.promiseTimeout(
page.goto(url.href, { waitUntil: 'domcontentloaded' })
)
await sleep(1000)
// Links
const links = await (
await this.promiseTimeout(
page.evaluateHandle(() =>
Array.from(document.getElementsByTagName('a')).map(
({ hash, hostname, href, pathname, protocol, rel }) => ({
hash,
hostname,
href,
pathname,
protocol,
rel,
})
const links = await this.promiseTimeout(
(
await this.promiseTimeout(
page.evaluateHandle(() =>
Array.from(document.getElementsByTagName('a')).map(
({ hash, hostname, href, pathname, protocol, rel }) => ({
hash,
hostname,
href,
pathname,
protocol,
rel,
})
)
)
),
)
).jsonValue()
)
).jsonValue()
)
// CSS
const css = await (
await this.promiseTimeout(
page.evaluateHandle((maxRows) => {
const css = []
try {
if (!document.styleSheets.length) {
return ''
}
const css = await this.promiseTimeout(
(
await this.promiseTimeout(
page.evaluateHandle((maxRows) => {
const css = []
try {
if (!document.styleSheets.length) {
return ''
}
for (const sheet of Array.from(document.styleSheets)) {
for (const rules of Array.from(sheet.cssRules)) {
css.push(rules.cssText)
for (const sheet of Array.from(document.styleSheets)) {
for (const rules of Array.from(sheet.cssRules)) {
css.push(rules.cssText)
if (css.length >= maxRows) {
break
if (css.length >= maxRows) {
break
}
}
}
} catch (error) {
return ''
}
} catch (error) {
return ''
}
return css.join('\n')
}, this.options.htmlMaxRows),
)
).jsonValue()
return css.join('\n')
}, this.options.htmlMaxRows)
)
).jsonValue()
)
// Script tags
const scripts = await (
await this.promiseTimeout(
page.evaluateHandle(() =>
Array.from(document.getElementsByTagName('script'))
.map(({ src }) => src)
.filter((src) => src)
),
)
).jsonValue()
const scripts = await this.promiseTimeout(
(
await this.promiseTimeout(
page.evaluateHandle(() =>
Array.from(document.getElementsByTagName('script'))
.map(({ src }) => src)
.filter((src) => src)
)
)
).jsonValue()
)
// Meta tags
const meta = await (
await this.promiseTimeout(
page.evaluateHandle(() =>
Array.from(document.querySelectorAll('meta')).reduce(
(metas, meta) => {
const key =
meta.getAttribute('name') || meta.getAttribute('property')
if (key) {
metas[key.toLowerCase()] = [meta.getAttribute('content')]
}
const meta = await this.promiseTimeout(
(
await this.promiseTimeout(
page.evaluateHandle(() =>
Array.from(document.querySelectorAll('meta')).reduce(
(metas, meta) => {
const key =
meta.getAttribute('name') || meta.getAttribute('property')
if (key) {
metas[key.toLowerCase()] = [meta.getAttribute('content')]
}
return metas
},
{}
return metas
},
{}
)
)
),
)
).jsonValue()
)
).jsonValue()
)
// JavaScript
const js = await this.promiseTimeout(
@ -467,7 +477,7 @@ class Site {
Wappalyzer.technologies
.filter(({ js }) => Object.keys(js).length)
.map(({ name, js }) => ({ name, chains: Object.keys(js) }))
),
)
)
// Cookies
@ -565,7 +575,11 @@ class Site {
return reducedLinks
} catch (error) {
this.error(error)
if (error.constructor.name === 'TimeoutError') {
throw new Error('The website took too long to respond')
}
throw new Error(error.message)
}
}