extern/se-scraper
1
0
Fork 1
mirror of https://github.com/NikolaiT/se-scraper.git synced 2025-08-14 07:48:21 +02:00
Code Issues Packages Projects Releases Wiki Activity

cleaned up google scrapers. All scrapers are classes now. from 600 LOC to 400 LOC. HIGH IQ MOVE

Browse Source
This commit is contained in:
Nikolai Tschacher
2019-01-30 20:24:03 +01:00
parent 4306848657
commit 581568ff18
5 changed files with 252 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
run.js
View File

@ -12,7 +12,7 @@ let config = {
// is drawn before every request. empty string for no sleeping.
sleep_range: '[1,2]',
// which search engine to scrape
search_engine: 'google',
search_engine: 'google_news',
// whether debug information should be printed
// debug info is useful for developers when debugging
debug: true,
@ -20,7 +20,7 @@ let config = {
// this output is informational
verbose: true,
// an array of keywords to scrape
keywords: ['trump', ],
keywords: ['hacking', 'trump'],
// alternatively you can specify a keyword_file. this overwrites the keywords array
keyword_file: '',
// the number of pages to scrape for each keyword
@ -54,7 +54,7 @@ function callback(err, response) {
response.statusCode - status code of the scraping process
*/
console.dir(response.results, {depth: null, colors: true});
// console.dir(response.results, {depth: null, colors: true});
}
se_scraper.scrape(config, callback);