extern/se-scraper
1
0
Fork 1
mirror of https://github.com/NikolaiT/se-scraper.git synced 2024-11-28 18:43:35 +01:00
Code Issues Packages Projects Releases Wiki Activity

added detected status to metadata

Browse Source
This commit is contained in:
Nikolai Tschacher 2019-10-06 15:34:18 +02:00
parent 4a3a0e6fd4
commit da69913272
3 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "se-scraper", "name": "se-scraper",
"version": "1.5.4", "version": "1.5.6",
"description": "A module using puppeteer to scrape several search engines such as Google, Bing and Duckduckgo", "description": "A module using puppeteer to scrape several search engines such as Google, Bing and Duckduckgo",
"homepage": "https://scrapeulous.com/", "homepage": "https://scrapeulous.com/",
"main": "index.js", "main": "index.js",
@ -27,7 +27,6 @@
"got": "^9.6.0", "got": "^9.6.0",
"lodash": "^4.17.14", "lodash": "^4.17.14",
"puppeteer": "^1.20.0", "puppeteer": "^1.20.0",
"puppeteer-cluster": "^0.13.0",
"puppeteer-extra": "^2.1.3", "puppeteer-extra": "^2.1.3",
"puppeteer-extra-plugin-stealth": "^2.2.2", "puppeteer-extra-plugin-stealth": "^2.2.2",
"user-agents": "^1.0.378" "user-agents": "^1.0.378"

8
src/modules/se_scraper.js
View File

@ -21,7 +21,9 @@ module.exports = class Scraper {
this.page = page; this.page = page;
this.last_response = null; // the last response object this.last_response = null; // the last response object
this.metadata = {}; this.metadata = {
scraping_detected: false,
};
this.pluggable = pluggable; this.pluggable = pluggable;
this.config = config; this.config = config;
this.context = context; this.context = context;
@ -275,7 +277,9 @@ module.exports = class Scraper {
} }
} }
if (await this.detected() === true) { this.metadata.scraping_detected = await this.detected();
if (this.metadata.scraping_detected === true) {
console.error(`${this.config.search_engine_name} detected the scraping!`); console.error(`${this.config.search_engine_name} detected the scraping!`);
if (this.config.is_local === true) { if (this.config.is_local === true) {

2
src/puppeteer-cluster

@ -1 +1 @@
Subproject commit 67bfa1bd613fbe2b63a073ac1fef59131c1d97a3 Subproject commit 221e6821d1d5d8c57bdf7b2cfef71d64dbf006a2