using random user agents now from https://github.com/intoli/user-agents

This commit is contained in:
Nikolai Tschacher
2019-07-18 19:34:09 +02:00
parent 59154694f2
commit fcbe66b56b
6 changed files with 59 additions and 130 deletions

View File

@ -4,17 +4,20 @@ const se_scraper = require('./../src/node_scraper.js');
let browser_config = {
debug_level: 1,
test_evasion: false,
headless: false,
block_assets: false,
random_user_agent: false,
log_http_headers: false,
html_output: false,
log_http_headers: true,
random_user_agent: true,
};
let scrape_job = {
search_engine: 'google',
keywords: ['mercedes reifen'],
keywords: ['too tired all the time'],
num_pages: 1,
google_settings: {
"gl": "us",
"hl": "en",
"start": 0,
"num": 10
}
};
var scraper = new se_scraper.ScrapeManager(browser_config);