|
|
@ -40,12 +40,6 @@ languageDetect.setLanguageType('iso2')
|
|
|
|
|
|
|
|
|
|
|
|
const extensions = /^([^.]+$|\.(asp|aspx|cgi|htm|html|jsp|php)$)/
|
|
|
|
const extensions = /^([^.]+$|\.(asp|aspx|cgi|htm|html|jsp|php)$)/
|
|
|
|
|
|
|
|
|
|
|
|
const errorTypes = {
|
|
|
|
|
|
|
|
RESPONSE_NOT_OK: 'Response was not ok',
|
|
|
|
|
|
|
|
NO_RESPONSE: 'No response from server',
|
|
|
|
|
|
|
|
NO_HTML_DOCUMENT: 'No HTML document'
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const { apps: technologies, categories } = JSON.parse(
|
|
|
|
const { apps: technologies, categories } = JSON.parse(
|
|
|
|
fs.readFileSync(path.resolve(`${__dirname}/apps.json`))
|
|
|
|
fs.readFileSync(path.resolve(`${__dirname}/apps.json`))
|
|
|
|
)
|
|
|
|
)
|
|
|
@ -157,7 +151,7 @@ class Site {
|
|
|
|
try {
|
|
|
|
try {
|
|
|
|
this.originalUrl = new URL(url)
|
|
|
|
this.originalUrl = new URL(url)
|
|
|
|
} catch (error) {
|
|
|
|
} catch (error) {
|
|
|
|
throw new Error(error.message || error.toString())
|
|
|
|
throw new Error(error.toString())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
this.analyzedUrls = {}
|
|
|
|
this.analyzedUrls = {}
|
|
|
@ -411,7 +405,7 @@ class Site {
|
|
|
|
|
|
|
|
|
|
|
|
this.log('Page closed')
|
|
|
|
this.log('Page closed')
|
|
|
|
|
|
|
|
|
|
|
|
throw new Error('NO_RESPONSE')
|
|
|
|
throw new Error('No response from server')
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if (!this.language) {
|
|
|
|
if (!this.language) {
|
|
|
@ -493,21 +487,9 @@ class Site {
|
|
|
|
await this.batch(links.slice(0, this.options.maxUrls), depth + 1)
|
|
|
|
await this.batch(links.slice(0, this.options.maxUrls), depth + 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} catch (error) {
|
|
|
|
} catch (error) {
|
|
|
|
const type =
|
|
|
|
|
|
|
|
error.message && errorTypes[error.message]
|
|
|
|
|
|
|
|
? error.message
|
|
|
|
|
|
|
|
: 'UNKNOWN_ERROR'
|
|
|
|
|
|
|
|
const message =
|
|
|
|
|
|
|
|
error.message && errorTypes[error.message]
|
|
|
|
|
|
|
|
? errorTypes[error.message]
|
|
|
|
|
|
|
|
: 'Unknown error'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
this.analyzedUrls[url.href] = {
|
|
|
|
this.analyzedUrls[url.href] = {
|
|
|
|
status: 0,
|
|
|
|
status: 0,
|
|
|
|
error: {
|
|
|
|
error: error.message || error.toString()
|
|
|
|
type,
|
|
|
|
|
|
|
|
message
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
this.error(error)
|
|
|
|
this.error(error)
|
|
|
|