extern/se-scraper
1
0
Fork 1
mirror of https://github.com/NikolaiT/se-scraper.git synced 2025-08-13 23:38:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

before_keyword_scraped() hook supported

Browse Source
This commit is contained in:
Nikolai Tschacher
2019-01-29 13:29:24 +01:00
parent c5e3e84e1d
commit 89441070cd
15 changed files with 309 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
run.js
View File

@ -10,7 +10,7 @@ let config = {
write_meta_data: false,
// how long to sleep between requests. a random sleep interval within the range [a,b]
// is drawn before every request. empty string for no sleeping.
sleep_range: '',
sleep_range: '[1,1]',
// which search engine to scrape
search_engine: 'google',
// whether debug information should be printed
@ -20,7 +20,7 @@ let config = {
// this output is informational
verbose: false,
// an array of keywords to scrape
keywords: ['incolumitas.com news', ],
keywords: ['trump', 'chief'],
// alternatively you can specify a keyword_file. this overwrites the keywords array
keyword_file: '',
// whether to start the browser in headless mode