|
|
|
@ -68,100 +68,106 @@ class PuppeteerBrowser extends Browser {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async visit(url) {
|
|
|
|
|
return new Promise(async (resolve, reject) => {
|
|
|
|
|
let done = false;
|
|
|
|
|
let browser;
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
browser = await this.browser();
|
|
|
|
|
} catch (error) {
|
|
|
|
|
reject(new Error(error.message || error.toString()));
|
|
|
|
|
}
|
|
|
|
|
let done = false;
|
|
|
|
|
let browser;
|
|
|
|
|
|
|
|
|
|
browser.on('disconnected', () => {
|
|
|
|
|
if (!done) {
|
|
|
|
|
reject(new Error('Disconnected'));
|
|
|
|
|
try {
|
|
|
|
|
await new Promise(async (resolve, reject) => {
|
|
|
|
|
try {
|
|
|
|
|
browser = await this.browser();
|
|
|
|
|
} catch (error) {
|
|
|
|
|
return reject(error);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
const page = await browser.newPage();
|
|
|
|
|
browser.on('disconnected', () => {
|
|
|
|
|
if (!done) {
|
|
|
|
|
reject(new Error('Disconnected'));
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
page.setDefaultTimeout(this.options.maxWait);
|
|
|
|
|
try {
|
|
|
|
|
const page = await browser.newPage();
|
|
|
|
|
|
|
|
|
|
await page.setRequestInterception(true);
|
|
|
|
|
page.setDefaultTimeout(this.options.maxWait);
|
|
|
|
|
|
|
|
|
|
page.on('error', reject);
|
|
|
|
|
await page.setRequestInterception(true);
|
|
|
|
|
|
|
|
|
|
page.on('request', request => request.continue());
|
|
|
|
|
page.on('error', reject);
|
|
|
|
|
|
|
|
|
|
page.on('response', (response) => {
|
|
|
|
|
if (response.status() === 301 || response.status() === 302) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
page.on('request', request => request.continue());
|
|
|
|
|
|
|
|
|
|
if (!this.statusCode) {
|
|
|
|
|
this.statusCode = response.status();
|
|
|
|
|
page.on('response', (response) => {
|
|
|
|
|
if (response.status() === 301 || response.status() === 302) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
this.headers = {};
|
|
|
|
|
if (!this.statusCode) {
|
|
|
|
|
this.statusCode = response.status();
|
|
|
|
|
|
|
|
|
|
const headers = response.headers();
|
|
|
|
|
this.headers = {};
|
|
|
|
|
|
|
|
|
|
Object.keys(headers).forEach((key) => {
|
|
|
|
|
this.headers[key] = Array.isArray(headers[key]) ? headers[key] : [headers[key]];
|
|
|
|
|
});
|
|
|
|
|
const headers = response.headers();
|
|
|
|
|
|
|
|
|
|
this.contentType = headers['content-type'] || null;
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
Object.keys(headers).forEach((key) => {
|
|
|
|
|
this.headers[key] = Array.isArray(headers[key]) ? headers[key] : [headers[key]];
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
page.on('console', ({ _type, _text, _location }) => this.log(`${_text} (${_location.url}: ${_location.lineNumber})`, _type));
|
|
|
|
|
this.contentType = headers['content-type'] || null;
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
await page.setUserAgent(this.options.userAgent);
|
|
|
|
|
page.on('console', ({ _type, _text, _location }) => this.log(`${_text} (${_location.url}: ${_location.lineNumber})`, _type));
|
|
|
|
|
|
|
|
|
|
await Promise.race([
|
|
|
|
|
page.goto(url, { waitUntil: 'networkidle2' }),
|
|
|
|
|
new Promise(_resolve => setTimeout(_resolve, this.options.maxWait)),
|
|
|
|
|
]);
|
|
|
|
|
await page.setUserAgent(this.options.userAgent);
|
|
|
|
|
|
|
|
|
|
// eslint-disable-next-line no-undef
|
|
|
|
|
const links = await page.evaluateHandle(() => Array.from(document.getElementsByTagName('a')).map(({
|
|
|
|
|
hash, hostname, href, pathname, protocol, rel,
|
|
|
|
|
}) => ({
|
|
|
|
|
hash,
|
|
|
|
|
hostname,
|
|
|
|
|
href,
|
|
|
|
|
pathname,
|
|
|
|
|
protocol,
|
|
|
|
|
rel,
|
|
|
|
|
})));
|
|
|
|
|
await Promise.race([
|
|
|
|
|
page.goto(url, { waitUntil: 'networkidle2' }),
|
|
|
|
|
new Promise(_resolve => setTimeout(_resolve, this.options.maxWait)),
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
this.links = await links.jsonValue();
|
|
|
|
|
// eslint-disable-next-line no-undef
|
|
|
|
|
const links = await page.evaluateHandle(() => Array.from(document.getElementsByTagName('a')).map(({
|
|
|
|
|
hash, hostname, href, pathname, protocol, rel,
|
|
|
|
|
}) => ({
|
|
|
|
|
hash,
|
|
|
|
|
hostname,
|
|
|
|
|
href,
|
|
|
|
|
pathname,
|
|
|
|
|
protocol,
|
|
|
|
|
rel,
|
|
|
|
|
})));
|
|
|
|
|
|
|
|
|
|
// eslint-disable-next-line no-undef
|
|
|
|
|
const scripts = await page.evaluateHandle(() => Array.from(document.getElementsByTagName('script')).map(({
|
|
|
|
|
src,
|
|
|
|
|
}) => src));
|
|
|
|
|
this.links = await links.jsonValue();
|
|
|
|
|
|
|
|
|
|
this.scripts = (await scripts.jsonValue()).filter(script => script);
|
|
|
|
|
// eslint-disable-next-line no-undef
|
|
|
|
|
const scripts = await page.evaluateHandle(() => Array.from(document.getElementsByTagName('script')).map(({
|
|
|
|
|
src,
|
|
|
|
|
}) => src));
|
|
|
|
|
|
|
|
|
|
this.js = await page.evaluate(getJs);
|
|
|
|
|
this.scripts = (await scripts.jsonValue()).filter(script => script);
|
|
|
|
|
|
|
|
|
|
this.cookies = (await page.cookies()).map(({
|
|
|
|
|
name, value, domain, path,
|
|
|
|
|
}) => ({
|
|
|
|
|
name, value, domain, path,
|
|
|
|
|
}));
|
|
|
|
|
this.js = await page.evaluate(getJs);
|
|
|
|
|
|
|
|
|
|
this.html = await page.content();
|
|
|
|
|
} catch (error) {
|
|
|
|
|
reject(error.message || error.toString());
|
|
|
|
|
} finally {
|
|
|
|
|
done = true;
|
|
|
|
|
this.cookies = (await page.cookies()).map(({
|
|
|
|
|
name, value, domain, path,
|
|
|
|
|
}) => ({
|
|
|
|
|
name, value, domain, path,
|
|
|
|
|
}));
|
|
|
|
|
|
|
|
|
|
await browser.close();
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
this.html = await page.content();
|
|
|
|
|
} catch (error) {
|
|
|
|
|
return reject(error);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return resolve();
|
|
|
|
|
});
|
|
|
|
|
} catch (error) {
|
|
|
|
|
throw new Error(error.message || error.toString());
|
|
|
|
|
} finally {
|
|
|
|
|
done = true;
|
|
|
|
|
|
|
|
|
|
await browser.close();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|