holm/se-scraper
1
0
Fork 0
forked from extern/se-scraper
Code Issues Pull Requests Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
Nikolai Tschacher
2019-01-26 20:15:19 +01:00
parent bab902e80a
commit b354e6918d
10 changed files with 83 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/node_scraper.js
View File

@ -1,5 +1,6 @@
const puppeteer = require('puppeteer');
const zlib = require('zlib');
var fs = require('fs');
// local module imports
const google = require('./modules/google.js');
@ -12,6 +13,14 @@ const meta = require('./modules/metadata.js');
const duckduckgo = require('./modules/duckduckgo.js');
const tickersearch = require('./modules/ticker_search.js');
function write_results(fname, data) {
fs.writeFileSync(fname, data, (err) => {
if (err) throw err;
console.log(`Results written to file ${fname}`);
});
}
module.exports.handler = async function handler (event, context, callback) {
try {
@ -127,6 +136,10 @@ module.exports.handler = async function handler (event, context, callback) {
}
}
if (event.output_file) {
write_results(event.output_file, JSON.stringify(results));
}
let response = {
headers: {
'Content-Type': 'text/json',