Migrations
/
wappalyzer
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix NPM/Puppeteer error handling

Browse Source
main
Elbert Alias 5 years ago
parent f522a2b549
commit 3c6dcf6afb
3 changed files with 103 additions and 99 deletions
Show Stats Download Patch File Download Diff File

198
src/drivers/npm/browsers/puppeteer.js
Unescape View File

@ -57,117 +57,121 @@ class PuppeteerBrowser extends Browser {
super(options); super(options);
} }
visit(url) { async visit(url) {
return new Promise(async (resolve, reject) => { let done = false;
let done = false; let browser;
let browser;
try {
browser = await puppeteer.launch(chromium ? {
args: [...chromium.args, '--ignore-certificate-errors'],
defaultViewport: chromium.defaultViewport,
executablePath: await chromium.executablePath,
headless: chromium.headless,
} : {
args: ['--no-sandbox', '--headless', '--disable-gpu', '--ignore-certificate-errors', '--disable-dev-shm-usage'],
executablePath: CHROME_BIN,
});
browser.on('disconnected', () => { try {
if (!done) { await new Promise(async (resolve, _reject) => {
reject(new Error('browser: disconnected')); try {
} browser = await puppeteer.launch(chromium ? {
}); args: [...chromium.args, '--ignore-certificate-errors'],
defaultViewport: chromium.defaultViewport,
executablePath: await chromium.executablePath,
headless: chromium.headless,
} : {
args: ['--no-sandbox', '--headless', '--disable-gpu', '--ignore-certificate-errors', '--disable-dev-shm-usage'],
executablePath: CHROME_BIN,
});
browser.on('disconnected', () => {
if (!done) {
_reject(new Error('browser: disconnected'));
}
});
const page = await browser.newPage(); const page = await browser.newPage();
page.setDefaultTimeout(this.options.maxWait * 2); page.setDefaultTimeout(this.options.maxWait * 2);
page.on('error', error => reject(new Error(`page error: ${error.message || error}`))); page.on('error', error => _reject(new Error(`page error: ${error.message || error}`)));
page.on('response', (response) => { page.on('response', (response) => {
try { try {
if (response.status() === 301 || response.status() === 302) { if (response.status() === 301 || response.status() === 302) {
return; return;
} }
if (!this.statusCode) { if (!this.statusCode) {
this.statusCode = response.status(); this.statusCode = response.status();
this.headers = {}; this.headers = {};
const headers = response.headers(); const headers = response.headers();
Object.keys(headers).forEach((key) => { Object.keys(headers).forEach((key) => {
this.headers[key] = Array.isArray(headers[key]) ? headers[key] : [headers[key]]; this.headers[key] = Array.isArray(headers[key]) ? headers[key] : [headers[key]];
}); });
this.contentType = headers['content-type'] || null; this.contentType = headers['content-type'] || null;
}
} catch (error) {
_reject(new Error(`page error: ${error.message || error}`));
} }
} catch (error) { });
reject(new Error(`page error: ${error.message || error}`));
} page.on('console', ({ _type, _text, _location }) => this.log(`${_text} (${_location.url}: ${_location.lineNumber})`, _type));
});
await page.setUserAgent(this.options.userAgent);
await Promise.race([
page.goto(url, { waitUntil: 'domcontentloaded' }),
new Promise(_resolve => setTimeout(() => {
this.log('Timeout', 'error');
_resolve();
}, this.options.maxWait)),
]);
// eslint-disable-next-line no-undef
const links = await page.evaluateHandle(() => Array.from(document.getElementsByTagName('a')).map(({
hash, hostname, href, pathname, protocol, rel,
}) => ({
hash,
hostname,
href,
pathname,
protocol,
rel,
})));
page.on('console', ({ _type, _text, _location }) => this.log(`${_text} (${_location.url}: ${_location.lineNumber})`, _type)); this.links = await links.jsonValue();
await page.setUserAgent(this.options.userAgent); // eslint-disable-next-line no-undef
const scripts = await page.evaluateHandle(() => Array.from(document.getElementsByTagName('script')).map(({
await Promise.race([ src,
page.goto(url, { waitUntil: 'domcontentloaded' }), }) => src));
new Promise(_resolve => setTimeout(() => {
this.log('Timeout', 'error'); this.scripts = (await scripts.jsonValue()).filter(script => script);
_resolve(); this.js = await page.evaluate(getJs);
}, this.options.maxWait)),
]); this.cookies = (await page.cookies()).map(({
name, value, domain, path,
// eslint-disable-next-line no-undef }) => ({
const links = await page.evaluateHandle(() => Array.from(document.getElementsByTagName('a')).map(({ name, value, domain, path,
hash, hostname, href, pathname, protocol, rel, }));
}) => ({
hash, this.html = await page.content();
hostname,
href, resolve();
pathname, } catch (error) {
protocol, _reject(new Error(`visit error: ${error.message || error}`));
rel, }
}))); });
} catch (error) {
this.links = await links.jsonValue(); done = true;
// eslint-disable-next-line no-undef if (browser) {
const scripts = await page.evaluateHandle(() => Array.from(document.getElementsByTagName('script')).map(({ try {
src, await browser.close();
}) => src)); } catch (_error) {
this.log(_error.message || _error.toString(), 'error');
this.scripts = (await scripts.jsonValue()).filter(script => script);
this.js = await page.evaluate(getJs);
this.cookies = (await page.cookies()).map(({
name, value, domain, path,
}) => ({
name, value, domain, path,
}));
this.html = await page.content();
resolve();
} catch (error) {
reject(new Error(`visit error: ${error.message || error}`));
} finally {
done = true;
if (browser) {
try {
await browser.close();
} catch (error) {
this.log(error.message || error.toString(), 'error');
}
} }
} }
});
throw new Error(error.message || error.toString());
}
} }
} }

2
src/drivers/npm/package.json
Unescape View File

@ -2,7 +2,7 @@
"name": "wappalyzer", "name": "wappalyzer",
"description": "Uncovers the technologies used on websites", "description": "Uncovers the technologies used on websites",
"homepage": "https://github.com/AliasIO/Wappalyzer", "homepage": "https://github.com/AliasIO/Wappalyzer",
"version": "5.9.21", "version": "5.9.22",
"author": "Elbert Alias", "author": "Elbert Alias",
"license": "GPL-3.0", "license": "GPL-3.0",
"repository": { "repository": {

2
src/drivers/webextension/manifest.json
Unescape View File

@ -4,7 +4,7 @@
"author": "Elbert Alias", "author": "Elbert Alias",
"homepage_url": "https://www.wappalyzer.com", "homepage_url": "https://www.wappalyzer.com",
"description": "Identify web technologies", "description": "Identify web technologies",
"version": "5.9.20", "version": "5.9.22",
"default_locale": "en", "default_locale": "en",
"manifest_version": 2, "manifest_version": 2,
"icons": { "icons": {