extern/se-scraper
1
0
Fork 1
mirror of https://github.com/NikolaiT/se-scraper.git synced 2024-12-01 03:53:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

removed some stuff

Browse Source
This commit is contained in:
Nikolai Tschacher 2019-09-12 10:43:57 +02:00
parent b513bb0f5b
commit 77d6c4f04a
2 changed files with 3 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "se-scraper", "name": "se-scraper",
"version": "1.4.6", "version": "1.4.7",
"description": "A module using puppeteer to scrape several search engines such as Google, Duckduckgo, Bing or Baidu", "description": "A module using puppeteer to scrape several search engines such as Google, Duckduckgo, Bing or Baidu",
"homepage": "https://scrapeulous.com/", "homepage": "https://scrapeulous.com/",
"main": "index.js", "main": "index.js",

17
src/node_scraper.js
View File

@ -364,6 +364,7 @@ class ScrapeManager {
} }
if (this.pluggable) { if (this.pluggable) {
this.scraper = getScraper(this.config.search_engine, { this.scraper = getScraper(this.config.search_engine, {
config: this.config, config: this.config,
context: this.context, context: this.context,
@ -428,24 +429,10 @@ class ScrapeManager {
log(this.config, 1, `Scraper took ${timeDelta}ms to perform ${num_requests} requests.`); log(this.config, 1, `Scraper took ${timeDelta}ms to perform ${num_requests} requests.`);
log(this.config, 1, `On average ms/request: ${ms_per_request}ms/request`); log(this.config, 1, `On average ms/request: ${ms_per_request}ms/request`);
if (this.config.compress) {
log(this.config, 1, 'Compressing results');
results = JSON.stringify(results);
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
results = zlib.deflateSync(results).toString('base64');
}
if (this.pluggable && this.pluggable.handle_results) { if (this.pluggable && this.pluggable.handle_results) {
await this.pluggable.handle_results(results); await this.pluggable.handle_results(results);
} }
if (this.config.chunk_lines) {
metadata.chunk_lines = this.config.chunk_lines;
if (this.config.job_name) {
metadata.id = `${this.config.job_name} ${this.config.chunk_lines}`;
}
}
metadata.elapsed_time = timeDelta.toString(); metadata.elapsed_time = timeDelta.toString();
metadata.ms_per_keyword = ms_per_request.toString(); metadata.ms_per_keyword = ms_per_request.toString();
metadata.num_requests = num_requests; metadata.num_requests = num_requests;
@ -468,7 +455,7 @@ class ScrapeManager {
} }
/* /*
* Quits the puppeteer cluster/browser. * Quit the puppeteer cluster/browser.
*/ */
async quit() { async quit() {
if (this.pluggable && this.pluggable.close_browser) { if (this.pluggable && this.pluggable.close_browser) {