extern/se-scraper
1
0
Fork 1
mirror of https://github.com/NikolaiT/se-scraper.git synced 2025-08-09 05:54:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

changed api big time

Browse Source
This commit is contained in:
Nikolai Tschacher
2019-06-11 18:16:59 +02:00
parent 3d69f4e249
commit 6825c97790
31 changed files with 2491 additions and 2536 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
examples/multiple_search_engines.js
View File

@ -1,35 +1,30 @@
const se_scraper = require('../index.js');
const se_scraper = require('./../src/node_scraper.js');
async function multiple_search_engines() {
(async () => {
let browser_config = {
random_user_agent: true,
write_meta_data: true,
sleep_range: '[1,1]',
debug_level: 1,
headless: true,
output_file: `multiple_search_engines.json`
};
var searchEnginesList = ['google', 'bing'];
let scrape_job = {
search_engine: 'google',
keywords: ['news', 'se-scraper'],
num_pages: 1,
};
for (let index = 0; index < searchEnginesList.length; index++) {
const searchEngine = searchEnginesList[index];
let config = {
random_user_agent: true,
write_meta_data: true,
sleep_range: '[1,1]',
search_engine: searchEngine,
debug: false,
verbose: false,
// the list of keywords to scrape
keywords: ['scrapeulous.com',],
// whether to start the browser in headless mode
headless: true,
output_file: `${searchEngine}.json`
};
var scraper = new se_scraper.ScrapeManager(browser_config);
await scraper.start();
await se_scraper.scrape(config, (err, response) => {
if (err) {
console.error(err)
}
console.dir(response.results, {
depth: null,
colors: true
});
});
for (var se of ['google', 'bing']) {
scrape_job.search_engine = se;
var results = await scraper.scrape(scrape_job);
console.dir(results, {depth: null, colors: true});
}
}
multiple_search_engines();
await scraper.quit();
})();