extern/se-scraper
1
0
Fork 1
mirror of https://github.com/NikolaiT/se-scraper.git synced 2024-10-05 01:22:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

some minor changes

Browse Source
This commit is contained in:
Nikolai Tschacher 2019-09-04 12:27:53 +02:00
parent e661241f6f
commit 855a874f9e
1 changed files with 15 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/modules/se_scraper.js
View File

@ -151,19 +151,19 @@ module.exports = class Scraper {
this.results[keyword] = {};
this.result_rank = 1;
if (this.pluggable && this.pluggable.before_keyword_scraped) {
await this.pluggable.before_keyword_scraped({
results: this.results,
num_keywords: this.num_keywords,
num_requests: this.num_requests,
keyword: keyword,
});
}
this.page_num = 1;
try {
if (this.pluggable && this.pluggable.before_keyword_scraped) {
await this.pluggable.before_keyword_scraped({
results: this.results,
num_keywords: this.num_keywords,
num_requests: this.num_requests,
keyword: keyword,
});
}
this.page_num = 1;
// load scraped page from file if `scrape_from_file` is given
if (this.config.scrape_from_file.length <= 0) {
await this.search_keyword(keyword);
@ -286,23 +286,17 @@ module.exports = class Scraper {
if (this.config.throw_on_detection === true) {
throw( e );
} else {
break;
return;
}
}
} else {
// some other error, quit scraping process if stuff is broken
if (this.config.is_local === true) {
console.error('You have 30 seconds to fix this.');
await this.sleep(30000);
if (this.config.throw_on_detection === true) {
throw( e );
} else {
if (this.config.throw_on_detection === true) {
throw( e );
} else {
break;
}
return;
}
}
}
}
}