holm/se-scraper
1
0
Fork 0
forked from extern/se-scraper
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cleaned up google scrapers. All scrapers are classes now. from 600 LOC to 400 LOC. HIGH IQ MOVE

Browse Source
This commit is contained in:
Nikolai Tschacher
2019-01-30 20:24:03 +01:00
parent 4306848657
commit 581568ff18
5 changed files with 252 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/node_scraper.js
View File

@ -131,11 +131,12 @@ module.exports.handler = async function handler (event, context, callback) {
Scraper = {
google: google.GoogleScraper,
google_news_old: google.scrape_google_news_old_pup,
google_news: google.scrape_google_news_pup,
google_image: google.scrape_google_image_pup,
google_news_old: google.GoogleNewsOldScraper,
google_news: google.GoogleNewsScraper,
google_image: google.GoogleImageScraper,
bing: bing.BingScraper,
bing_news: bing.BingNewsScraper,
infospace: infospace.scrape_infospace_pup,
webcrawler: infospace.scrape_webcrawler_news_pup,
baidu: baidu.scrape_baidu_pup,
@ -158,7 +159,6 @@ module.exports.handler = async function handler (event, context, callback) {
let results = await scraper.run();
if (pluggable.close_browser) {
await pluggable.close_browser();
} else {