holm/se-scraper
1
0
Fork 0
forked from extern/se-scraper
Code Issues Pull Requests Packages Projects Releases Wiki Activity

before_keyword_scraped() hook supported

Browse Source
This commit is contained in:
Nikolai Tschacher
2019-01-29 13:29:24 +01:00
parent c5e3e84e1d
commit 89441070cd
15 changed files with 309 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/modules/bing.js
View File

@ -6,7 +6,7 @@ module.exports = {
scrape_bing_news_pup: scrape_bing_news_pup,
};
async function scrape_bing_pup(page, event, context) {
async function scrape_bing_pup(page, event, context, pluggable) {
await page.goto('https://www.bing.com/');
try {
@ -22,6 +22,15 @@ async function scrape_bing_pup(page, event, context) {
keyword = keywords[i];
if (pluggable.before_keyword_scraped) {
await pluggable.before_keyword_scraped({
keyword: keyword,
page: page,
event: event,
context: context,
});
}
try {
const input = await page.$('input[name="q"]');
// overwrites last text in input
@ -90,7 +99,7 @@ function parse(html) {
}
}
async function scrape_bing_news_pup(page, event, context) {
async function scrape_bing_news_pup(page, event, context, pluggable) {
await page.goto('https://www.bing.com/news/search?');
if (event.set_manual_settings === true) {
@ -109,12 +118,17 @@ async function scrape_bing_news_pup(page, event, context) {
for (var i = 0; i < keywords.length; i++) {
if (sfunctions.should_turn_down(context)) {
break;
}
keyword = keywords[i];
if (pluggable.before_keyword_scraped) {
await pluggable.before_keyword_scraped({
keyword: keyword,
page: page,
event: event,
context: context,
});
}
try {
const input = await page.$('input[name="q"]');
// overwrites last text in input