faster scraping, added ticker search engines

This commit is contained in:
Nikolai Tschacher
2019-01-27 01:27:52 +01:00
parent b0e588f916
commit e78d7145b5
13 changed files with 244 additions and 63 deletions

7
run.js
View File

@ -11,19 +11,22 @@ let config = {
// is drawn before every request. empty string for no sleeping.
sleep_range: '[1,1]',
// which search engine to scrape
search_engine: 'google',
search_engine: 'bing',
// whether debug information should be printed
debug: true,
// whether verbose program output should be printed
verbose: false,
// an array of keywords to scrape
keywords: ['scrapeulous.com', ],
keywords: ['MSFT', ],
// alternatively you can specify a keyword_file. this overwrites the keywords array
keyword_file: '',
// whether to start the browser in headless mode
headless: false,
// path to output file, data will be stored in JSON
output_file: 'data.json',
// whether to prevent images, css, fonts from being loaded
// will speed up scraping a great deal
block_assets: true
};
se_scraper.scrape(config, (err, response) => {