extern/se-scraper
1
0
Fork 1
mirror of https://github.com/NikolaiT/se-scraper.git synced 2025-08-18 17:38:17 +02:00
Code Issues Packages Projects Releases Wiki Activity

supporting yahoo ticker search for news

Browse Source
This commit is contained in:
Nikolai Tschacher
2019-01-24 15:50:03 +01:00
parent 9cfa502851
commit bab902e80a
11 changed files with 106 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
index.js
View File

@@ -1,5 +1,6 @@
const handler = require('./src/node_scraper.js');
var fs = require('fs');
var os = require("os");
exports.scrape = function(config, callback) {
// options for scraping
@@ -21,14 +22,14 @@ exports.scrape = function(config, callback) {
compress: 'false', // compress
debug: 'false',
verbose: 'false',
keywords: [],
keywords: ['test'],
};
for (var key in config) {
event[key] = config[key];
}
if (fs.existsSync( event.keyword_file )) {
if (fs.existsSync(event.keyword_file)) {
event.keywords = read_keywords_from_file(event.keyword_file);
}
@@ -47,7 +48,7 @@ exports.scrape = function(config, callback) {
};
function read_keywords_from_file(fname) {
let kws = fs.readFileSync(fname).toString().split("\n");
let kws = fs.readFileSync(fname).toString().split(os.EOL);
// clean keywords
kws = kws.filter((kw) => {
return kw.trim().length > 0;