holm/se-scraper
1
0
Fork 0
forked from extern/se-scraper
Code Issues Pull Requests Packages Projects Releases Wiki Activity

some minor changes

Browse Source
This commit is contained in:
Nikolai Tschacher 2019-09-04 12:27:53 +02:00
parent e661241f6f
commit 855a874f9e
1 changed files with 15 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/modules/se_scraper.js
View File

@ -151,6 +151,8 @@ module.exports = class Scraper {
this.results[keyword] = {}; this.results[keyword] = {};
this.result_rank = 1; this.result_rank = 1;
try {
if (this.pluggable && this.pluggable.before_keyword_scraped) { if (this.pluggable && this.pluggable.before_keyword_scraped) {
await this.pluggable.before_keyword_scraped({ await this.pluggable.before_keyword_scraped({
results: this.results, results: this.results,
@ -162,8 +164,6 @@ module.exports = class Scraper {
this.page_num = 1; this.page_num = 1;
try {
// load scraped page from file if `scrape_from_file` is given // load scraped page from file if `scrape_from_file` is given
if (this.config.scrape_from_file.length <= 0) { if (this.config.scrape_from_file.length <= 0) {
await this.search_keyword(keyword); await this.search_keyword(keyword);
@ -286,24 +286,18 @@ module.exports = class Scraper {
if (this.config.throw_on_detection === true) { if (this.config.throw_on_detection === true) {
throw( e ); throw( e );
} else { } else {
break; return;
} }
} }
} else { } else {
// some other error, quit scraping process if stuff is broken // some other error, quit scraping process if stuff is broken
if (this.config.is_local === true) {
console.error('You have 30 seconds to fix this.');
await this.sleep(30000);
} else {
if (this.config.throw_on_detection === true) { if (this.config.throw_on_detection === true) {
throw( e ); throw( e );
} else { } else {
break; return;
} }
} }
} }
}
} }
} }