mirror of
https://github.com/NikolaiT/se-scraper.git
synced 2024-11-21 23:23:07 +01:00
fix(proxy): throw on use_proxies_only if no proxies given
This commit is contained in:
parent
4f467abf1e
commit
c58d4fa74d
@ -52,7 +52,7 @@ module.exports = class Scraper {
|
|||||||
|
|
||||||
async run({page, data, worker}) {
|
async run({page, data, worker}) {
|
||||||
|
|
||||||
debug('worker.id=%s', worker.id, this.config.keywords);
|
debug('worker=%o', worker, this.config.keywords);
|
||||||
|
|
||||||
if (page) {
|
if (page) {
|
||||||
this.page = page;
|
this.page = page;
|
||||||
|
@ -186,6 +186,10 @@ class ScrapeManager {
|
|||||||
this.logger.info(`${this.config.proxies.length} proxies read from file.`);
|
this.logger.info(`${this.config.proxies.length} proxies read from file.`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!this.config.proxies && this.config.use_proxies_only) {
|
||||||
|
throw new Error('Must provide at least one proxy in proxies if you enable use_proxies_only');
|
||||||
|
}
|
||||||
|
|
||||||
debug('this.config=%O', this.config);
|
debug('this.config=%O', this.config);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -136,6 +136,26 @@ describe('Config', function(){
|
|||||||
await scraper.quit();
|
await scraper.quit();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('zero proxy given, use_proxies_only=true', async function () {
|
||||||
|
|
||||||
|
const scrape_job = {
|
||||||
|
search_engine: MockScraperTestProxy,
|
||||||
|
keywords: ['news', 'some stuff', 'i work too much', 'what to do?', 'javascript is hard'],
|
||||||
|
};
|
||||||
|
|
||||||
|
await assert.rejects(async () => {
|
||||||
|
var scraper = new se_scraper.ScrapeManager({
|
||||||
|
throw_on_detection: true,
|
||||||
|
use_proxies_only: true,
|
||||||
|
logger: testLogger,
|
||||||
|
});
|
||||||
|
await scraper.start();
|
||||||
|
const { results } = await scraper.scrape(scrape_job);
|
||||||
|
await scraper.quit();
|
||||||
|
}, /Must provide at least one proxy in proxies if you enable use_proxies_only/);
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
||||||
});
|
});
|
Loading…
Reference in New Issue
Block a user