diff --git a/package.json b/package.json index 428043e..2c06778 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "se-scraper", - "version": "1.4.6", + "version": "1.4.7", "description": "A module using puppeteer to scrape several search engines such as Google, Duckduckgo, Bing or Baidu", "homepage": "https://scrapeulous.com/", "main": "index.js", diff --git a/src/node_scraper.js b/src/node_scraper.js index 20ed8c4..bfa535c 100644 --- a/src/node_scraper.js +++ b/src/node_scraper.js @@ -364,6 +364,7 @@ class ScrapeManager { } if (this.pluggable) { + this.scraper = getScraper(this.config.search_engine, { config: this.config, context: this.context, @@ -428,24 +429,10 @@ class ScrapeManager { log(this.config, 1, `Scraper took ${timeDelta}ms to perform ${num_requests} requests.`); log(this.config, 1, `On average ms/request: ${ms_per_request}ms/request`); - if (this.config.compress) { - log(this.config, 1, 'Compressing results'); - results = JSON.stringify(results); - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding - results = zlib.deflateSync(results).toString('base64'); - } - if (this.pluggable && this.pluggable.handle_results) { await this.pluggable.handle_results(results); } - if (this.config.chunk_lines) { - metadata.chunk_lines = this.config.chunk_lines; - if (this.config.job_name) { - metadata.id = `${this.config.job_name} ${this.config.chunk_lines}`; - } - } - metadata.elapsed_time = timeDelta.toString(); metadata.ms_per_keyword = ms_per_request.toString(); metadata.num_requests = num_requests; @@ -468,7 +455,7 @@ class ScrapeManager { } /* - * Quits the puppeteer cluster/browser. + * Quit the puppeteer cluster/browser. */ async quit() { if (this.pluggable && this.pluggable.close_browser) {