extern/se-scraper
1
0
Fork 1
mirror of https://github.com/NikolaiT/se-scraper.git synced 2025-06-20 17:47:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

Remove uneeded try catch block

Browse Source 
Add proper error for ip matching test
This commit is contained in:
HugoPoi 2019-08-06 11:46:53 +02:00
parent 50bda275a6
commit 0db6e068da
1 changed files with 12 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/modules/se_scraper.js
View File

@ -1,3 +1,4 @@
'use strict';
const meta = require('./metadata.js'); const meta = require('./metadata.js');
const common = require('./common.js'); const common = require('./common.js');
var log = common.log; var log = common.log;
@ -41,12 +42,8 @@ module.exports = class Scraper {
let settings = this.config[`${this.config.search_engine}_settings`]; let settings = this.config[`${this.config.search_engine}_settings`];
if (settings) { if (settings) {
if (typeof settings === 'string') { if (typeof settings === 'string') {
try { settings = JSON.parse(settings);
settings = JSON.parse(settings); this.config[`${this.config.search_engine}_settings`] = settings;
this.config[`${this.config.search_engine}_settings`] = settings;
} catch (e) {
console.error(e);
}
} }
} }
} }
@ -59,13 +56,8 @@ module.exports = class Scraper {
await this.page.setViewport({ width: 1920, height: 1040 }); await this.page.setViewport({ width: 1920, height: 1040 });
let do_continue = await this.load_search_engine(); await this.load_search_engine();
await this.scraping_loop();
if (!do_continue) {
console.error('Failed to load the search engine: load_search_engine()');
} else {
await this.scraping_loop();
}
return this.results; return this.results;
} }
@ -120,24 +112,16 @@ module.exports = class Scraper {
if (this.proxy && this.config.log_ip_address === true) { if (this.proxy && this.config.log_ip_address === true) {
log(this.config, 3, `${this.metadata.ipinfo.ip} vs ${this.proxy}`); log(this.config, 3, `${this.metadata.ipinfo.ip} vs ${this.proxy}`);
try { // if the ip returned by ipinfo is not a substring of our proxystring, get the heck outta here
// if the ip returned by ipinfo is not a substring of our proxystring, get the heck outta here if (!this.proxy.includes(this.metadata.ipinfo.ip)) {
if (!this.proxy.includes(this.metadata.ipinfo.ip)) { throw new Error(`Proxy output ip ${this.proxy} does not match with provided one`);
console.error(`Proxy ${this.proxy} does not work.`); } else {
return false; log(this.config, 1, `Using valid Proxy: ${this.proxy}`);
} else {
log(this.config, 1, `Using valid Proxy: ${this.proxy}`);
}
} catch (exception) {
} }
} }
try { return await this.load_start_page();
return await this.load_start_page();
} catch (e) {
console.error(e);
return false;
}
} }
/** /**
@ -382,7 +366,6 @@ module.exports = class Scraper {
// This is where we'll put the code to get around the tests. // This is where we'll put the code to get around the tests.
async function evadeChromeHeadlessDetection(page) { async function evadeChromeHeadlessDetection(page) {
try {
// Pass the Webdriver Test. // Pass the Webdriver Test.
await page.evaluateOnNewDocument(() => { await page.evaluateOnNewDocument(() => {
const newProto = navigator.__proto__; const newProto = navigator.__proto__;
@ -509,8 +492,4 @@ async function evadeChromeHeadlessDetection(page) {
return null; return null;
}; };
}); });
} catch (e) {
console.error(e);
}
} }