holm/se-scraper
1
0
Fork 0
forked from extern/se-scraper
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added handle_results

Browse Source
This commit is contained in:
Nikolai Tschacher
2019-01-27 20:08:09 +01:00
parent 86a66a09fd
commit a20b6c79a9
7 changed files with 30 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/node_scraper.js
View File

@ -68,7 +68,7 @@ module.exports.handler = async function handler (config, context, callback) {
let launch_args = {
args: ADDITIONAL_CHROME_FLAGS,
headless: config.headless !== false,
headless: config.headless,
};
if (config.debug === true) {
@ -149,6 +149,13 @@ module.exports.handler = async function handler (config, context, callback) {
results = zlib.deflateSync(results).toString('base64');
}
if (pluggable && pluggable.handle_results) {
await pluggable.handle_results({
config: config,
results: results,
});
}
if (config.write_meta_data === true) {
metadata.id = `${config.job_name} ${config.chunk_lines}`;
metadata.chunk_lines = config.chunk_lines;