extern/se-scraper
1
0
Fork 1
mirror of https://github.com/NikolaiT/se-scraper.git synced 2025-08-09 05:54:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

parsing ads works for duckduckgo, google, bing. tested.

Browse Source
This commit is contained in:
Nikolai Tschacher
2019-07-07 19:38:28 +02:00
parent bbebe3ce60
commit a413cb54ef
12 changed files with 211 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
examples/quickstart.js
View File

@ -2,25 +2,19 @@ const se_scraper = require('./../src/node_scraper.js');
(async () => {
let browser_config = {
debug_level: 2,
output_file: 'examples/results/data.json',
debug_level: 1,
test_evasion: false,
headless: false,
headless: true,
block_assets: false,
random_user_agent: true,
random_user_agent: false,
log_http_headers: false,
html_output: false,
};
let scrape_job = {
search_engine: 'google',
keywords: ['cloud service'],
search_engine: 'bing',
keywords: ['auto verkaufen'],
num_pages: 1,
// add some cool google search settings
google_settings: {
gl: 'us', // The gl parameter determines the Google country to use for the query.
hl: 'en', // The hl parameter determines the Google UI language to return results.
start: 0, // Determines the results offset to use, defaults to 0.
num: 10, // Determines the number of results to show, defaults to 10. Maximum is 100.
},
};
var scraper = new se_scraper.ScrapeManager(browser_config);