mirror of
https://github.com/NikolaiT/se-scraper.git
synced 2025-08-18 09:29:44 +02:00
.
This commit is contained in:
14
run.js
14
run.js
@@ -6,22 +6,24 @@ let config = {
|
||||
// if random_user_agent is set to True, a random user agent is chosen
|
||||
random_user_agent: false,
|
||||
// get meta data of scraping in return object
|
||||
write_meta_data: 'true',
|
||||
write_meta_data: false,
|
||||
// how long to sleep between requests. a random sleep interval within the range [a,b]
|
||||
// is drawn before every request. empty string for no sleeping.
|
||||
sleep_range: '[1,1]',
|
||||
// which search engine to scrape
|
||||
search_engine: 'yahoo_news',
|
||||
search_engine: 'google',
|
||||
// whether debug information should be printed
|
||||
debug: 'true',
|
||||
debug: true,
|
||||
// whether verbose program output should be printed
|
||||
verbose: 'false',
|
||||
verbose: false,
|
||||
// an array of keywords to scrape
|
||||
keywords: ['GOOGL', ],
|
||||
keywords: ['scrapeulous.com', ],
|
||||
// alternatively you can specify a keyword_file. this overwrites the keywords array
|
||||
keyword_file: './keywords.txt',
|
||||
keyword_file: '',
|
||||
// whether to start the browser in headless mode
|
||||
headless: false,
|
||||
// path to output file, data will be stored in JSON
|
||||
output_file: 'data.json',
|
||||
};
|
||||
|
||||
se_scraper.scrape(config, (err, response) => {
|
||||
|
Reference in New Issue
Block a user