From da6991327219ef5346838bac41dccd70ca35c0f8 Mon Sep 17 00:00:00 2001 From: Nikolai Tschacher Date: Sun, 6 Oct 2019 15:34:18 +0200 Subject: [PATCH] added detected status to metadata --- package.json | 3 +-- src/modules/se_scraper.js | 8 ++++++-- src/puppeteer-cluster | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 8c738fb..9504f6a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "se-scraper", - "version": "1.5.4", + "version": "1.5.6", "description": "A module using puppeteer to scrape several search engines such as Google, Bing and Duckduckgo", "homepage": "https://scrapeulous.com/", "main": "index.js", @@ -27,7 +27,6 @@ "got": "^9.6.0", "lodash": "^4.17.14", "puppeteer": "^1.20.0", - "puppeteer-cluster": "^0.13.0", "puppeteer-extra": "^2.1.3", "puppeteer-extra-plugin-stealth": "^2.2.2", "user-agents": "^1.0.378" diff --git a/src/modules/se_scraper.js b/src/modules/se_scraper.js index 5674754..68e7732 100644 --- a/src/modules/se_scraper.js +++ b/src/modules/se_scraper.js @@ -21,7 +21,9 @@ module.exports = class Scraper { this.page = page; this.last_response = null; // the last response object - this.metadata = {}; + this.metadata = { + scraping_detected: false, + }; this.pluggable = pluggable; this.config = config; this.context = context; @@ -275,7 +277,9 @@ module.exports = class Scraper { } } - if (await this.detected() === true) { + this.metadata.scraping_detected = await this.detected(); + + if (this.metadata.scraping_detected === true) { console.error(`${this.config.search_engine_name} detected the scraping!`); if (this.config.is_local === true) { diff --git a/src/puppeteer-cluster b/src/puppeteer-cluster index 67bfa1b..221e682 160000 --- a/src/puppeteer-cluster +++ b/src/puppeteer-cluster @@ -1 +1 @@ -Subproject commit 67bfa1bd613fbe2b63a073ac1fef59131c1d97a3 +Subproject commit 221e6821d1d5d8c57bdf7b2cfef71d64dbf006a2