From c58d4fa74d29dc82f5cd90194f513fa5eec1166f Mon Sep 17 00:00:00 2001 From: HugoPoi Date: Fri, 17 Jan 2020 15:55:17 +0100 Subject: [PATCH] fix(proxy): throw on use_proxies_only if no proxies given --- src/modules/se_scraper.js | 2 +- src/node_scraper.js | 4 ++++ test/proxy.js | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/modules/se_scraper.js b/src/modules/se_scraper.js index 95e0c16..7de0a3d 100644 --- a/src/modules/se_scraper.js +++ b/src/modules/se_scraper.js @@ -52,7 +52,7 @@ module.exports = class Scraper { async run({page, data, worker}) { - debug('worker.id=%s', worker.id, this.config.keywords); + debug('worker=%o', worker, this.config.keywords); if (page) { this.page = page; diff --git a/src/node_scraper.js b/src/node_scraper.js index ed9564e..2dec432 100644 --- a/src/node_scraper.js +++ b/src/node_scraper.js @@ -186,6 +186,10 @@ class ScrapeManager { this.logger.info(`${this.config.proxies.length} proxies read from file.`); } + if (!this.config.proxies && this.config.use_proxies_only) { + throw new Error('Must provide at least one proxy in proxies if you enable use_proxies_only'); + } + debug('this.config=%O', this.config); } diff --git a/test/proxy.js b/test/proxy.js index d95e1d2..c1092ea 100644 --- a/test/proxy.js +++ b/test/proxy.js @@ -136,6 +136,26 @@ describe('Config', function(){ await scraper.quit(); }); + it('zero proxy given, use_proxies_only=true', async function () { + + const scrape_job = { + search_engine: MockScraperTestProxy, + keywords: ['news', 'some stuff', 'i work too much', 'what to do?', 'javascript is hard'], + }; + + await assert.rejects(async () => { + var scraper = new se_scraper.ScrapeManager({ + throw_on_detection: true, + use_proxies_only: true, + logger: testLogger, + }); + await scraper.start(); + const { results } = await scraper.scrape(scrape_job); + await scraper.quit(); + }, /Must provide at least one proxy in proxies if you enable use_proxies_only/); + + }); + }); }); \ No newline at end of file