extern/se-scraper
1
0
Fork 1
mirror of https://github.com/NikolaiT/se-scraper.git synced 2024-11-21 23:23:07 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #62 from aularon/patch-1

Browse Source 
Take screenshot before modifying HTML
This commit is contained in:
Nikolai Tschacher 2020-02-13 20:04:20 +01:00 committed by GitHub
commit 33fa371716
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/modules/se_scraper.js
View File

@ -193,6 +193,13 @@ module.exports = class Scraper {
let parsed = this.parse(html); let parsed = this.parse(html);
this.results[keyword][this.page_num] = parsed ? parsed : await this.parse_async(html); this.results[keyword][this.page_num] = parsed ? parsed : await this.parse_async(html);
if (this.config.screen_output) {
this.results[keyword][this.page_num].screenshot = await this.page.screenshot({
encoding: 'base64',
fullPage: false,
});
}
if (this.config.html_output) { if (this.config.html_output) {
if (this.config.clean_html_output) { if (this.config.clean_html_output) {
@ -239,13 +246,6 @@ module.exports = class Scraper {
this.results[keyword][this.page_num].html = html_contents; this.results[keyword][this.page_num].html = html_contents;
} }
if (this.config.screen_output) {
this.results[keyword][this.page_num].screenshot = await this.page.screenshot({
encoding: 'base64',
fullPage: false,
});
}
this.page_num += 1; this.page_num += 1;
// only load the next page when we will pass the next iteration // only load the next page when we will pass the next iteration