From 301695cd2b1218ab0023b5b0e835870f034c8c20 Mon Sep 17 00:00:00 2001 From: HugoPoi Date: Fri, 20 Dec 2019 19:35:23 +0100 Subject: [PATCH] fix(scrape-manager): proxy_file options can be used with proxies default value --- src/node_scraper.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/node_scraper.js b/src/node_scraper.js index 8ae673a..2f51169 100644 --- a/src/node_scraper.js +++ b/src/node_scraper.js @@ -139,7 +139,7 @@ class ScrapeManager { custom_func: null, throw_on_detection: false, // List of proxies to use ['socks5://78.94.172.42:1080', 'http://localhost:1080'] - proxies: [], + proxies: null, // a file with one proxy per line. Example: // socks5://78.94.172.42:1080 // http://118.174.233.10:48400 @@ -180,7 +180,7 @@ class ScrapeManager { throw new Error('Either use a proxy_file or specify a proxy for all connections. Do not use both options.'); } - if (fs.existsSync(this.config.proxy_file)) { + if (this.config.proxy_file) { this.config.proxies = read_keywords_from_file(this.config.proxy_file); this.logger.info(`${this.config.proxies.length} proxies read from file.`); } @@ -253,7 +253,7 @@ class ScrapeManager { // if we have at least one proxy, always use CONCURRENCY_BROWSER // and set maxConcurrency to this.config.proxies.length + 1 // else use whatever this.configuration was passed - if (this.config.proxies.length > 0) { + if (this.config.proxies && this.config.proxies.length > 0) { this.config.puppeteer_cluster_config.concurrency = Cluster.CONCURRENCY_BROWSER; // because we use real browsers, we ran out of memory on normal laptops