holm/se-scraper
1
0
Fork 0
forked from extern/se-scraper
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(proxy): throw on use_proxies_only if no proxies given

Browse Source
This commit is contained in:
HugoPoi 2020-01-17 15:55:17 +01:00
parent 4f467abf1e
commit c58d4fa74d
3 changed files with 25 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/modules/se_scraper.js
View File

@ -52,7 +52,7 @@ module.exports = class Scraper {
async run({page, data, worker}) {
debug('worker.id=%s', worker.id, this.config.keywords);
debug('worker=%o', worker, this.config.keywords);
if (page) {
this.page = page;

4
src/node_scraper.js
View File

@ -186,6 +186,10 @@ class ScrapeManager {
this.logger.info(`${this.config.proxies.length} proxies read from file.`);
}
if (!this.config.proxies && this.config.use_proxies_only) {
throw new Error('Must provide at least one proxy in proxies if you enable use_proxies_only');
}
debug('this.config=%O', this.config);
}

20
test/proxy.js
View File

@ -136,6 +136,26 @@ describe('Config', function(){
await scraper.quit();
});
it('zero proxy given, use_proxies_only=true', async function () {
const scrape_job = {
search_engine: MockScraperTestProxy,
keywords: ['news', 'some stuff', 'i work too much', 'what to do?', 'javascript is hard'],
};
await assert.rejects(async () => {
var scraper = new se_scraper.ScrapeManager({
throw_on_detection: true,
use_proxies_only: true,
logger: testLogger,
});
await scraper.start();
const { results } = await scraper.scrape(scrape_job);
await scraper.quit();
}, /Must provide at least one proxy in proxies if you enable use_proxies_only/);
});
});
});