fix(proxy): throw on use_proxies_only if no proxies given

This commit is contained in:
HugoPoi 2020-01-17 15:55:17 +01:00
parent 4f467abf1e
commit c58d4fa74d
3 changed files with 25 additions and 1 deletions

View File

@ -52,7 +52,7 @@ module.exports = class Scraper {
async run({page, data, worker}) { async run({page, data, worker}) {
debug('worker.id=%s', worker.id, this.config.keywords); debug('worker=%o', worker, this.config.keywords);
if (page) { if (page) {
this.page = page; this.page = page;

View File

@ -186,6 +186,10 @@ class ScrapeManager {
this.logger.info(`${this.config.proxies.length} proxies read from file.`); this.logger.info(`${this.config.proxies.length} proxies read from file.`);
} }
if (!this.config.proxies && this.config.use_proxies_only) {
throw new Error('Must provide at least one proxy in proxies if you enable use_proxies_only');
}
debug('this.config=%O', this.config); debug('this.config=%O', this.config);
} }

View File

@ -136,6 +136,26 @@ describe('Config', function(){
await scraper.quit(); await scraper.quit();
}); });
it('zero proxy given, use_proxies_only=true', async function () {
const scrape_job = {
search_engine: MockScraperTestProxy,
keywords: ['news', 'some stuff', 'i work too much', 'what to do?', 'javascript is hard'],
};
await assert.rejects(async () => {
var scraper = new se_scraper.ScrapeManager({
throw_on_detection: true,
use_proxies_only: true,
logger: testLogger,
});
await scraper.start();
const { results } = await scraper.scrape(scrape_job);
await scraper.quit();
}, /Must provide at least one proxy in proxies if you enable use_proxies_only/);
});
}); });
}); });