holm/se-scraper
1
0
Fork 0
forked from extern/se-scraper
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #59 from TDenoncin/refactor/logging

Browse Source 
Refactor logging
This commit is contained in:
Nikolai Tschacher 2019-12-20 14:42:09 +01:00 committed by GitHub
commit d362e4ae2c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 264 additions and 564 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
examples/bing_multiple_browser_multiple_pages.js
View File

@ -32,12 +32,6 @@ let browser_config = {
verbose: true, verbose: true,
// whether to start the browser in headless mode // whether to start the browser in headless mode
headless: true, headless: true,
// whether debug information should be printed
// level 0: print nothing
// level 1: print most important info
// ...
// level 4: print all shit nobody wants to know
debug_level: 1,
is_local: false, is_local: false,
throw_on_detection: false, throw_on_detection: false,
puppeteer_cluster_config: { puppeteer_cluster_config: {

6
examples/for_the_lulz.js
View File

@ -30,12 +30,6 @@ let browser_config = {
// if random_user_agent is set to True, a random user agent is chosen // if random_user_agent is set to True, a random user agent is chosen
random_user_agent: true, random_user_agent: true,
headless: true, headless: true,
// whether debug information should be printed
// level 0: print nothing
// level 1: print most important info
// ...
// level 4: print all shit nobody wants to know
debug_level: 1,
is_local: false, is_local: false,
throw_on_detection: false, throw_on_detection: false,
puppeteer_cluster_config: { puppeteer_cluster_config: {

1
examples/gimage.js
View File

@ -2,7 +2,6 @@ const se_scraper = require('./../src/node_scraper.js');
(async () => { (async () => {
let browser_config = { let browser_config = {
debug_level: 1,
output_file: '', output_file: '',
}; };

1
examples/gnold.js
View File

@ -2,7 +2,6 @@ const se_scraper = require('./../src/node_scraper.js');
(async () => { (async () => {
let browser_config = { let browser_config = {
debug_level: 2,
output_file: 'examples/results/gnold.json', output_file: 'examples/results/gnold.json',
google_news_old_settings: { google_news_old_settings: {
gl: 'us', // The gl parameter determines the Google country to use for the query. gl: 'us', // The gl parameter determines the Google country to use for the query.

1
examples/google_maps.js
View File

@ -2,7 +2,6 @@ const se_scraper = require('./../src/node_scraper.js');
(async () => { (async () => {
let browser_config = { let browser_config = {
debug_level: 1,
output_file: 'examples/results/maps.json', output_file: 'examples/results/maps.json',
test_evasion: false, test_evasion: false,
block_assets: false, block_assets: false,

1
examples/multiple_browsers.js
View File

@ -3,7 +3,6 @@ const se_scraper = require('./../src/node_scraper.js');
(async () => { (async () => {
let browser_config = { let browser_config = {
search_engine: 'google', search_engine: 'google',
debug_level: 1,
random_user_agent: true, random_user_agent: true,
is_local: false, is_local: false,
html_output: false, html_output: false,

1
examples/multiple_search_engines.js
View File

@ -5,7 +5,6 @@ const se_scraper = require('./../src/node_scraper.js');
random_user_agent: true, random_user_agent: true,
write_meta_data: true, write_meta_data: true,
sleep_range: '[1,1]', sleep_range: '[1,1]',
debug_level: 1,
headless: true, headless: true,
output_file: `examples/results/multiple_search_engines.json` output_file: `examples/results/multiple_search_engines.json`
}; };

1
examples/pluggable_example.js
View File

@ -3,7 +3,6 @@ const resolve = require('path').resolve;
(async () => { (async () => {
let browser_config = { let browser_config = {
debug_level: 1,
test_evasion: false, test_evasion: false,
log_http_headers: true, log_http_headers: true,
log_ip_address: true, log_ip_address: true,

1
examples/proxies.js
View File

@ -2,7 +2,6 @@ const se_scraper = require('./../src/node_scraper.js');
(async () => { (async () => {
let browser_config = { let browser_config = {
debug_level: 1,
output_file: 'examples/results/proxyresults.json', output_file: 'examples/results/proxyresults.json',
log_ip_address: true, log_ip_address: true,
// a file with one proxy per line. Example: // a file with one proxy per line. Example:

1
examples/quickstart.js
View File

@ -2,7 +2,6 @@ const se_scraper = require('./../src/node_scraper.js');
(async () => { (async () => {
let browser_config = { let browser_config = {
debug_level: 1,
test_evasion: false, test_evasion: false,
log_http_headers: false, log_http_headers: false,
log_ip_address: false, log_ip_address: false,

1
examples/reusing.js
View File

@ -2,7 +2,6 @@ const se_scraper = require('./../src/node_scraper.js');
(async () => { (async () => {
let browser_config = { let browser_config = {
debug_level: 1,
output_file: 'examples/results/data.json', output_file: 'examples/results/data.json',
}; };

579
package-lock.json generated
View File

@ -22,15 +22,6 @@
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.7.5.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.7.5.tgz",
"integrity": "sha512-9fq4jZVhPNW8r+UYKnxF1e2HkDWOWKM5bC2/7c9wPV835I0aOrVbS/Hw/pWPk2uKrNXQqg9Z959Kz+IYDd5p3w==" "integrity": "sha512-9fq4jZVhPNW8r+UYKnxF1e2HkDWOWKM5bC2/7c9wPV835I0aOrVbS/Hw/pWPk2uKrNXQqg9Z959Kz+IYDd5p3w=="
}, },
"accepts": {
"version": "1.3.7",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz",
"integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==",
"requires": {
"mime-types": "~2.1.24",
"negotiator": "0.6.2"
}
},
"agent-base": { "agent-base": {
"version": "4.3.0", "version": "4.3.0",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz",
@ -74,17 +65,20 @@
"resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz",
"integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=" "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ="
}, },
"array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
},
"assertion-error": { "assertion-error": {
"version": "1.1.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz",
"integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==",
"dev": true "dev": true
}, },
"async": {
"version": "2.6.3",
"resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
"integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
"requires": {
"lodash": "^4.17.14"
}
},
"async-limiter": { "async-limiter": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz",
@ -95,38 +89,6 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
}, },
"body-parser": {
"version": "1.19.0",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
"integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
"requires": {
"bytes": "3.1.0",
"content-type": "~1.0.4",
"debug": "2.6.9",
"depd": "~1.1.2",
"http-errors": "1.7.2",
"iconv-lite": "0.4.24",
"on-finished": "~2.3.0",
"qs": "6.7.0",
"raw-body": "2.4.0",
"type-is": "~1.6.17"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}
}
},
"boolbase": { "boolbase": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
@ -152,11 +114,6 @@
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
}, },
"bytes": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
"integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
},
"cacheable-request": { "cacheable-request": {
"version": "6.0.0", "version": "6.0.0",
"resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.0.0.tgz", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.0.0.tgz",
@ -275,11 +232,19 @@
"integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
"dev": true "dev": true
}, },
"color": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz",
"integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==",
"requires": {
"color-convert": "^1.9.1",
"color-string": "^1.5.2"
}
},
"color-convert": { "color-convert": {
"version": "1.9.3", "version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
"dev": true,
"requires": { "requires": {
"color-name": "1.1.3" "color-name": "1.1.3"
} }
@ -287,8 +252,35 @@
"color-name": { "color-name": {
"version": "1.1.3", "version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
"integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
"dev": true },
"color-string": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.3.tgz",
"integrity": "sha512-dC2C5qeWoYkxki5UAXapdjqO672AM4vZuPGRQfO8b5HKuKGBbKWpITyDYN7TOFKvRW7kOgAn3746clDBMDJyQw==",
"requires": {
"color-name": "^1.0.0",
"simple-swizzle": "^0.2.2"
}
},
"colornames": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/colornames/-/colornames-1.1.1.tgz",
"integrity": "sha1-+IiQMGhcfE/54qVZ9Qd+t2qBb5Y="
},
"colors": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
"integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA=="
},
"colorspace": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz",
"integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==",
"requires": {
"color": "3.0.x",
"text-hex": "1.0.x"
}
}, },
"concat-map": { "concat-map": {
"version": "0.0.1", "version": "0.0.1",
@ -297,7 +289,7 @@
}, },
"concat-stream": { "concat-stream": {
"version": "1.6.2", "version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "resolved": "http://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
"requires": { "requires": {
"buffer-from": "^1.0.0", "buffer-from": "^1.0.0",
@ -308,7 +300,7 @@
"dependencies": { "dependencies": {
"readable-stream": { "readable-stream": {
"version": "2.3.6", "version": "2.3.6",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
"requires": { "requires": {
"core-util-is": "~1.0.0", "core-util-is": "~1.0.0",
@ -322,7 +314,7 @@
}, },
"string_decoder": { "string_decoder": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": { "requires": {
"safe-buffer": "~5.1.0" "safe-buffer": "~5.1.0"
@ -330,29 +322,6 @@
} }
} }
}, },
"content-disposition": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
"integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==",
"requires": {
"safe-buffer": "5.1.2"
}
},
"content-type": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
"integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
},
"cookie": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz",
"integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg=="
},
"cookie-signature": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
"integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
},
"core-util-is": { "core-util-is": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
@ -437,15 +406,15 @@
"object-keys": "^1.0.12" "object-keys": "^1.0.12"
} }
}, },
"depd": { "diagnostics": {
"version": "1.1.2", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", "resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz",
"integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" "integrity": "sha512-8wn1PmdunLJ9Tqbx+Fx/ZEuHfJf4NKSN2ZBj7SJC/OWRWha843+WsTjqMe1B5E3p28jqBlp+mJ2fPVxPyNgYKQ==",
}, "requires": {
"destroy": { "colorspace": "1.1.x",
"version": "1.0.4", "enabled": "1.0.x",
"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "kuler": "1.0.x"
"integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }
}, },
"diff": { "diff": {
"version": "3.5.0", "version": "3.5.0",
@ -503,21 +472,19 @@
"resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz",
"integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=" "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI="
}, },
"ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
},
"emoji-regex": { "emoji-regex": {
"version": "7.0.3", "version": "7.0.3",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz",
"integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==",
"dev": true "dev": true
}, },
"encodeurl": { "enabled": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", "resolved": "https://registry.npmjs.org/enabled/-/enabled-1.0.2.tgz",
"integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" "integrity": "sha1-ll9lE9LC0cX0ZStkouM5ZGf8L5M=",
"requires": {
"env-variable": "0.0.x"
}
}, },
"end-of-stream": { "end-of-stream": {
"version": "1.4.1", "version": "1.4.1",
@ -532,6 +499,11 @@
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
}, },
"env-variable": {
"version": "0.0.5",
"resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz",
"integrity": "sha512-zoB603vQReOFvTg5xMl9I1P2PnHsHQQKTEowsKKD7nseUfJq6UWzK+4YtlWUO1nhiQUxe6XMkk+JleSZD1NZFA=="
},
"es-abstract": { "es-abstract": {
"version": "1.13.0", "version": "1.13.0",
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz",
@ -564,17 +536,12 @@
}, },
"es6-promisify": { "es6-promisify": {
"version": "5.0.0", "version": "5.0.0",
"resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", "resolved": "http://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
"integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=",
"requires": { "requires": {
"es6-promise": "^4.0.3" "es6-promise": "^4.0.3"
} }
}, },
"escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
"integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
},
"escape-string-regexp": { "escape-string-regexp": {
"version": "1.0.5", "version": "1.0.5",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
@ -587,11 +554,6 @@
"integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
"dev": true "dev": true
}, },
"etag": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
"integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
},
"execa": { "execa": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz",
@ -607,58 +569,6 @@
"strip-eof": "^1.0.0" "strip-eof": "^1.0.0"
} }
}, },
"express": {
"version": "4.17.1",
"resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
"integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==",
"requires": {
"accepts": "~1.3.7",
"array-flatten": "1.1.1",
"body-parser": "1.19.0",
"content-disposition": "0.5.3",
"content-type": "~1.0.4",
"cookie": "0.4.0",
"cookie-signature": "1.0.6",
"debug": "2.6.9",
"depd": "~1.1.2",
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
"finalhandler": "~1.1.2",
"fresh": "0.5.2",
"merge-descriptors": "1.0.1",
"methods": "~1.1.2",
"on-finished": "~2.3.0",
"parseurl": "~1.3.3",
"path-to-regexp": "0.1.7",
"proxy-addr": "~2.0.5",
"qs": "6.7.0",
"range-parser": "~1.2.1",
"safe-buffer": "5.1.2",
"send": "0.17.1",
"serve-static": "1.14.1",
"setprototypeof": "1.1.1",
"statuses": "~1.5.0",
"type-is": "~1.6.18",
"utils-merge": "1.0.1",
"vary": "~1.1.2"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}
}
},
"extract-zip": { "extract-zip": {
"version": "1.6.7", "version": "1.6.7",
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz", "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz",
@ -685,6 +595,11 @@
} }
} }
}, },
"fast-safe-stringify": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz",
"integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA=="
},
"fd-slicer": { "fd-slicer": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz",
@ -693,34 +608,10 @@
"pend": "~1.2.0" "pend": "~1.2.0"
} }
}, },
"finalhandler": { "fecha": {
"version": "1.1.2", "version": "2.3.3",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz",
"integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", "integrity": "sha512-lUGBnIamTAwk4znq5BcqsDaxSmZ9nDVJaij6NvRt/Tg4R69gERA+otPKbS86ROw9nxVMw2/mp1fnaiWqbs6Sdg=="
"requires": {
"debug": "2.6.9",
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"on-finished": "~2.3.0",
"parseurl": "~1.3.3",
"statuses": "~1.5.0",
"unpipe": "~1.0.0"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}
}
}, },
"find-up": { "find-up": {
"version": "3.0.0", "version": "3.0.0",
@ -753,16 +644,6 @@
"for-in": "^1.0.1" "for-in": "^1.0.1"
} }
}, },
"forwarded": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
"integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
},
"fresh": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
"integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
},
"fs.realpath": { "fs.realpath": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@ -876,18 +757,6 @@
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz",
"integrity": "sha512-laeSTWIkuFa6lUgZAt+ic9RwOSEwbi9VDQNcCvMFO4sZiDc2Ha8DaZVCJnfpLLQCcS8rvCnIWYmz0POLxt7Dew==" "integrity": "sha512-laeSTWIkuFa6lUgZAt+ic9RwOSEwbi9VDQNcCvMFO4sZiDc2Ha8DaZVCJnfpLLQCcS8rvCnIWYmz0POLxt7Dew=="
}, },
"http-errors": {
"version": "1.7.2",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
"integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
"requires": {
"depd": "~1.1.2",
"inherits": "2.0.3",
"setprototypeof": "1.1.1",
"statuses": ">= 1.5.0 < 2",
"toidentifier": "1.0.0"
}
},
"https-proxy-agent": { "https-proxy-agent": {
"version": "3.0.1", "version": "3.0.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz",
@ -907,14 +776,6 @@
} }
} }
}, },
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
},
"inflight": { "inflight": {
"version": "1.0.6", "version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@ -935,10 +796,10 @@
"integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==", "integrity": "sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==",
"dev": true "dev": true
}, },
"ipaddr.js": { "is-arrayish": {
"version": "1.9.0", "version": "0.3.2",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.0.tgz", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
"integrity": "sha512-M4Sjn6N/+O6/IXSJseKqHoFc+5FdGJ22sXqnjTpdZweHK64MzEPAyQZyEU3R/KRv2GLoa7nNtg/C2Ev6m7z+eA==" "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="
}, },
"is-buffer": { "is-buffer": {
"version": "2.0.3", "version": "2.0.3",
@ -989,8 +850,7 @@
"is-stream": { "is-stream": {
"version": "1.1.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
"integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
"dev": true
}, },
"is-symbol": { "is-symbol": {
"version": "1.0.2", "version": "1.0.2",
@ -1055,6 +915,14 @@
} }
} }
}, },
"kuler": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/kuler/-/kuler-1.0.1.tgz",
"integrity": "sha512-J9nVUucG1p/skKul6DU3PUZrhs0LPulNaeUOox0IyXDi8S4CztTHs1gQphhuZmzXG7VOQSf6NJfKuzteQLv9gQ==",
"requires": {
"colornames": "^1.1.1"
}
},
"lazy-cache": { "lazy-cache": {
"version": "1.0.4", "version": "1.0.4",
"resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz",
@ -1098,6 +966,18 @@
"chalk": "^2.0.1" "chalk": "^2.0.1"
} }
}, },
"logform": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/logform/-/logform-2.1.2.tgz",
"integrity": "sha512-+lZh4OpERDBLqjiwDLpAWNQu6KMjnlXH2ByZwCuSqVPJletw0kTWJf5CgSNAUKn1KUkv3m2cUz/LK8zyEy7wzQ==",
"requires": {
"colors": "^1.2.1",
"fast-safe-stringify": "^2.0.4",
"fecha": "^2.3.3",
"ms": "^2.1.1",
"triple-beam": "^1.3.0"
}
},
"lowercase-keys": { "lowercase-keys": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
@ -1112,11 +992,6 @@
"p-defer": "^1.0.0" "p-defer": "^1.0.0"
} }
}, },
"media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
"integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
},
"mem": { "mem": {
"version": "4.3.0", "version": "4.3.0",
"resolved": "https://registry.npmjs.org/mem/-/mem-4.3.0.tgz", "resolved": "https://registry.npmjs.org/mem/-/mem-4.3.0.tgz",
@ -1138,34 +1013,11 @@
"kind-of": "^3.0.2" "kind-of": "^3.0.2"
} }
}, },
"merge-descriptors": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
"integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
},
"methods": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
"integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
},
"mime": { "mime": {
"version": "2.4.4", "version": "2.4.4",
"resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
"integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==" "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA=="
}, },
"mime-db": {
"version": "1.40.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz",
"integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA=="
},
"mime-types": {
"version": "2.1.24",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz",
"integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==",
"requires": {
"mime-db": "1.40.0"
}
},
"mimic-fn": { "mimic-fn": {
"version": "2.1.0", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
@ -1275,11 +1127,6 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
"integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
}, },
"negotiator": {
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
"integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
},
"nice-try": { "nice-try": {
"version": "1.0.5", "version": "1.0.5",
"resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz",
@ -1352,14 +1199,6 @@
"es-abstract": "^1.5.1" "es-abstract": "^1.5.1"
} }
}, },
"on-finished": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
"integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=",
"requires": {
"ee-first": "1.1.1"
}
},
"once": { "once": {
"version": "1.4.0", "version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
@ -1368,6 +1207,11 @@
"wrappy": "1" "wrappy": "1"
} }
}, },
"one-time": {
"version": "0.0.4",
"resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz",
"integrity": "sha1-+M33eISCb+Tf+T46nMN7HkSAdC4="
},
"os-locale": { "os-locale": {
"version": "3.1.0", "version": "3.1.0",
"resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz", "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz",
@ -1434,11 +1278,6 @@
"@types/node": "*" "@types/node": "*"
} }
}, },
"parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
"integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="
},
"path-exists": { "path-exists": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
@ -1456,11 +1295,6 @@
"integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=",
"dev": true "dev": true
}, },
"path-to-regexp": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
},
"pathval": { "pathval": {
"version": "1.1.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz",
@ -1487,15 +1321,6 @@
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
}, },
"proxy-addr": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz",
"integrity": "sha512-t/7RxHXPH6cJtP0pRG6smSr9QJidhB+3kXu0KgXnbGYMgzEnUxRQ4/LDdfOwZEMyIh3/xHb8PX3t+lfL9z+YVQ==",
"requires": {
"forwarded": "~0.1.2",
"ipaddr.js": "1.9.0"
}
},
"proxy-from-env": { "proxy-from-env": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz",
@ -1602,27 +1427,6 @@
} }
} }
}, },
"qs": {
"version": "6.7.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
"integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ=="
},
"range-parser": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
"integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
},
"raw-body": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
"integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
"requires": {
"bytes": "3.1.0",
"http-errors": "1.7.2",
"iconv-lite": "0.4.24",
"unpipe": "1.0.0"
}
},
"readable-stream": { "readable-stream": {
"version": "3.4.0", "version": "3.4.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz",
@ -1666,81 +1470,18 @@
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
}, },
"safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"semver": { "semver": {
"version": "5.7.0", "version": "5.7.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
"integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==",
"dev": true "dev": true
}, },
"send": {
"version": "0.17.1",
"resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz",
"integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
"requires": {
"debug": "2.6.9",
"depd": "~1.1.2",
"destroy": "~1.0.4",
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
"fresh": "0.5.2",
"http-errors": "~1.7.2",
"mime": "1.6.0",
"ms": "2.1.1",
"on-finished": "~2.3.0",
"range-parser": "~1.2.1",
"statuses": "~1.5.0"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
},
"dependencies": {
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}
}
},
"mime": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
}
}
},
"serve-static": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
"integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==",
"requires": {
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"parseurl": "~1.3.3",
"send": "0.17.1"
}
},
"set-blocking": { "set-blocking": {
"version": "2.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
"integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=",
"dev": true "dev": true
}, },
"setprototypeof": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
"integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw=="
},
"shallow-clone": { "shallow-clone": {
"version": "0.1.2", "version": "0.1.2",
"resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz", "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz",
@ -1793,16 +1534,24 @@
"integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
"dev": true "dev": true
}, },
"simple-swizzle": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
"integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=",
"requires": {
"is-arrayish": "^0.3.1"
}
},
"sprintf-js": { "sprintf-js": {
"version": "1.0.3", "version": "1.0.3",
"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
"integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
"dev": true "dev": true
}, },
"statuses": { "stack-trace": {
"version": "1.5.0", "version": "0.0.10",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz",
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA="
}, },
"string-width": { "string-width": {
"version": "2.1.1", "version": "2.1.1",
@ -1859,15 +1608,20 @@
"has-flag": "^3.0.0" "has-flag": "^3.0.0"
} }
}, },
"text-hex": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz",
"integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg=="
},
"to-readable-stream": { "to-readable-stream": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz",
"integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==" "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q=="
}, },
"toidentifier": { "triple-beam": {
"version": "1.0.0", "version": "1.3.0",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz",
"integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw=="
}, },
"type-detect": { "type-detect": {
"version": "4.0.8", "version": "4.0.8",
@ -1875,15 +1629,6 @@
"integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==",
"dev": true "dev": true
}, },
"type-is": {
"version": "1.6.18",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
"integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
"requires": {
"media-typer": "0.3.0",
"mime-types": "~2.1.24"
}
},
"typedarray": { "typedarray": {
"version": "0.0.6", "version": "0.0.6",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
@ -1902,11 +1647,6 @@
"underscore": "*" "underscore": "*"
} }
}, },
"unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
},
"url-parse-lax": { "url-parse-lax": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz",
@ -1929,16 +1669,6 @@
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
}, },
"utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
"integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
},
"vary": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
"integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
},
"which": { "which": {
"version": "1.3.1", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
@ -1963,6 +1693,55 @@
"string-width": "^1.0.2 || 2" "string-width": "^1.0.2 || 2"
} }
}, },
"winston": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/winston/-/winston-3.2.1.tgz",
"integrity": "sha512-zU6vgnS9dAWCEKg/QYigd6cgMVVNwyTzKs81XZtTFuRwJOcDdBg7AU0mXVyNbs7O5RH2zdv+BdNZUlx7mXPuOw==",
"requires": {
"async": "^2.6.1",
"diagnostics": "^1.1.1",
"is-stream": "^1.1.0",
"logform": "^2.1.1",
"one-time": "0.0.4",
"readable-stream": "^3.1.1",
"stack-trace": "0.0.x",
"triple-beam": "^1.3.0",
"winston-transport": "^4.3.0"
}
},
"winston-transport": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.3.0.tgz",
"integrity": "sha512-B2wPuwUi3vhzn/51Uukcao4dIduEiPOcOt9HJ3QeaXgkJ5Z7UwpBzxS4ZGNHtrxrUvTwemsQiSys0ihOf8Mp1A==",
"requires": {
"readable-stream": "^2.3.6",
"triple-beam": "^1.2.0"
},
"dependencies": {
"readable-stream": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"wrap-ansi": { "wrap-ansi": {
"version": "2.1.0", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",

4
package.json
View File

@ -24,13 +24,13 @@
"dependencies": { "dependencies": {
"cheerio": "^1.0.0-rc.3", "cheerio": "^1.0.0-rc.3",
"debug": "^4.1.1", "debug": "^4.1.1",
"express": "^4.17.1",
"got": "^9.6.0", "got": "^9.6.0",
"lodash": "^4.17.14", "lodash": "^4.17.14",
"puppeteer": "^2.0.0", "puppeteer": "^2.0.0",
"puppeteer-extra": "^2.1.3", "puppeteer-extra": "^2.1.3",
"puppeteer-extra-plugin-stealth": "^2.2.2", "puppeteer-extra-plugin-stealth": "^2.2.2",
"user-agents": "^1.0.378" "user-agents": "^1.0.378",
"winston": "^3.2.1"
}, },
"devDependencies": { "devDependencies": {
"chai": "^4.2.0", "chai": "^4.2.0",

21
src/modules/common.js
View File

@ -1,21 +0,0 @@
function log(config, loglevel, msg = null, cb = null) {
if (typeof loglevel != "number") {
throw Error('loglevel must be numeric.');
}
if (loglevel <= config.debug_level) {
if (msg) {
if (typeof msg == 'object') {
console.dir(msg, {depth: null, colors: false});
} else {
console.log('[i] ' + msg);
}
} else if (cb) {
cb();
}
}
}
module.exports = {
log: log,
};

10
src/modules/google.js
View File

@ -2,8 +2,6 @@
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const Scraper = require('./se_scraper'); const Scraper = require('./se_scraper');
const common = require('./common.js');
var log = common.log;
class GoogleScraper extends Scraper { class GoogleScraper extends Scraper {
@ -243,7 +241,7 @@ class GoogleScraper extends Scraper {
} }
} }
log(this.config, 1, 'Using startUrl: ' + startUrl); this.logger.info('Using startUrl: ' + startUrl);
this.last_response = await this.page.goto(startUrl); this.last_response = await this.page.goto(startUrl);
@ -642,7 +640,7 @@ class GoogleMapsScraper extends Scraper {
this.scrape_in_detail = this.config.google_maps_settings.scrape_in_detail || false; this.scrape_in_detail = this.config.google_maps_settings.scrape_in_detail || false;
} }
log(this.config, 1, 'Using startUrl: ' + startUrl); this.logger.info('Using startUrl: ' + startUrl);
this.last_response = await this.page.goto(startUrl); this.last_response = await this.page.goto(startUrl);
@ -681,7 +679,7 @@ class GoogleMapsScraper extends Scraper {
let last_title_last_result = this.results[this.keyword][this.page_num-1].results.slice(-1)[0].title; let last_title_last_result = this.results[this.keyword][this.page_num-1].results.slice(-1)[0].title;
log(this.config, 1, `Waiting until new last serp title differs from: "${last_title_last_result}"`); this.logger.info(`Waiting until new last serp title differs from: "${last_title_last_result}"`);
await this.page.waitForFunction((last_title) => { await this.page.waitForFunction((last_title) => {
const res = document.querySelectorAll('.section-result .section-result-title span'); const res = document.querySelectorAll('.section-result .section-result-title span');
@ -775,7 +773,7 @@ class GoogleShoppingScraper extends Scraper {
} }
} }
log(this.config, 1, 'Using startUrl: ' + startUrl); this.logger.info('Using startUrl: ' + startUrl);
this.last_response = await this.page.goto(startUrl); this.last_response = await this.page.goto(startUrl);

40
src/modules/se_scraper.js
View File

@ -1,8 +1,6 @@
'use strict'; 'use strict';
const meta = require('./metadata.js'); const meta = require('./metadata.js');
const common = require('./common.js'); const debug = require('debug')('se-scraper:Scraper');
var log = common.log;
/* /*
Get useful JS knowledge and get awesome... Get useful JS knowledge and get awesome...
@ -26,6 +24,7 @@ module.exports = class Scraper {
}; };
this.pluggable = pluggable; this.pluggable = pluggable;
this.config = config; this.config = config;
this.logger = this.config.logger;
this.context = context; this.context = context;
this.proxy = config.proxy; this.proxy = config.proxy;
@ -113,25 +112,25 @@ module.exports = class Scraper {
if (this.config.log_http_headers === true) { if (this.config.log_http_headers === true) {
this.metadata.http_headers = await meta.get_http_headers(this.page); this.metadata.http_headers = await meta.get_http_headers(this.page);
log(this.config, 2, this.metadata.http_headers); debug('this.metadata.http_headers=%O', this.metadata.http_headers);
} }
if (this.config.log_ip_address === true) { if (this.config.log_ip_address === true) {
let ipinfo = await meta.get_ip_data(this.page); let ipinfo = await meta.get_ip_data(this.page);
this.metadata.ipinfo = ipinfo; this.metadata.ipinfo = ipinfo;
log(this.config, 2, this.metadata.ipinfo); debug('this.metadata.ipinfo', this.metadata.ipinfo);
} }
// check that our proxy is working by confirming // check that our proxy is working by confirming
// that ipinfo.io sees the proxy IP address // that ipinfo.io sees the proxy IP address
if (this.proxy && this.config.log_ip_address === true) { if (this.proxy && this.config.log_ip_address === true) {
log(this.config, 3, `${this.metadata.ipinfo.ip} vs ${this.proxy}`); debug(`${this.metadata.ipinfo.ip} vs ${this.proxy}`);
// if the ip returned by ipinfo is not a substring of our proxystring, get the heck outta here // if the ip returned by ipinfo is not a substring of our proxystring, get the heck outta here
if (!this.proxy.includes(this.metadata.ipinfo.ip)) { if (!this.proxy.includes(this.metadata.ipinfo.ip)) {
throw new Error(`Proxy output ip ${this.proxy} does not match with provided one`); throw new Error(`Proxy output ip ${this.proxy} does not match with provided one`);
} else { } else {
log(this.config, 1, `Using valid Proxy: ${this.proxy}`); this.logger.info(`Using valid Proxy: ${this.proxy}`);
} }
} }
@ -179,7 +178,7 @@ module.exports = class Scraper {
do { do {
log(this.config, 1, `${this.config.search_engine_name} scrapes keyword "${keyword}" on page ${this.page_num}`); this.logger.info(`${this.config.search_engine_name} scrapes keyword "${keyword}" on page ${this.page_num}`);
await this.wait_for_results(); await this.wait_for_results();
@ -263,28 +262,21 @@ module.exports = class Scraper {
} catch (e) { } catch (e) {
console.error(`Problem with scraping ${keyword} in search engine ${this.config.search_engine_name}: ${e}`); this.logger.warn(`Problem with scraping ${keyword} in search engine ${this.config.search_engine_name}: ${e.message}`);
debug('this.last_response=%O', this.last_response);
if (this.last_response) { if (this.config.take_screenshot_on_error) {
log(this.config, 2, this.last_response); await this.page.screenshot({ path: `debug_se_scraper_${this.config.search_engine_name}_${keyword}.png` });
}
if (this.config.debug_level > 2) {
try {
// Try to save a screenshot of the error
await this.page.screenshot({path: `debug_se_scraper_${this.config.search_engine_name}_${keyword}.png`});
} catch (e) {
}
} }
this.metadata.scraping_detected = await this.detected(); this.metadata.scraping_detected = await this.detected();
if (this.metadata.scraping_detected === true) { if (this.metadata.scraping_detected === true) {
console.error(`${this.config.search_engine_name} detected the scraping!`); this.logger.warn(`${this.config.search_engine_name} detected the scraping!`);
if (this.config.is_local === true) { if (this.config.is_local === true) {
await this.sleep(this.SOLVE_CAPTCHA_TIME); await this.sleep(this.SOLVE_CAPTCHA_TIME);
console.error(`You have ${this.SOLVE_CAPTCHA_TIME}ms to enter the captcha.`); this.logger.info(`You have ${this.SOLVE_CAPTCHA_TIME}ms to enter the captcha.`);
// expect that user filled out necessary captcha // expect that user filled out necessary captcha
} else { } else {
if (this.config.throw_on_detection === true) { if (this.config.throw_on_detection === true) {
@ -318,7 +310,7 @@ module.exports = class Scraper {
baseUrl += `${key}=${settings[key]}&` baseUrl += `${key}=${settings[key]}&`
} }
log(this.config, 1, 'Using startUrl: ' + baseUrl); this.logger.info('Using startUrl: ' + baseUrl);
return baseUrl; return baseUrl;
} }
@ -335,7 +327,7 @@ module.exports = class Scraper {
async random_sleep() { async random_sleep() {
const [min, max] = this.config.sleep_range; const [min, max] = this.config.sleep_range;
let rand = Math.floor(Math.random() * (max - min + 1) + min); //Generate Random number let rand = Math.floor(Math.random() * (max - min + 1) + min); //Generate Random number
log(this.config, 1, `Sleeping for ${rand}s`); this.logger.info(`Sleeping for ${rand}s`);
await this.sleep(rand * 1000); await this.sleep(rand * 1000);
} }
@ -349,7 +341,7 @@ module.exports = class Scraper {
no_results(needles, html) { no_results(needles, html) {
for (let needle of needles) { for (let needle of needles) {
if (html.includes(needle)) { if (html.includes(needle)) {
console.log(this.config, 2, `HTML contains needle ${needle}. no_results=true`); this.logger.warn(`HTML contains needle ${needle}. no_results=true`);
return true; return true;
} }
} }

4
src/modules/yandex.js
View File

@ -1,8 +1,6 @@
'use strict'; 'use strict';
const Scraper = require('./se_scraper'); const Scraper = require('./se_scraper');
const common = require('./common.js');
var log = common.log;
class YandexScraper extends Scraper { class YandexScraper extends Scraper {
@ -75,7 +73,7 @@ class YandexScraper extends Scraper {
async load_start_page() { async load_start_page() {
let startUrl = 'https://yandex.com'; let startUrl = 'https://yandex.com';
log(this.config, 1, 'Using startUrl: ' + startUrl); this.logger.info('Using startUrl: ' + startUrl);
this.last_response = await this.page.goto(startUrl); this.last_response = await this.page.goto(startUrl);

149
src/node_scraper.js
View File

@ -1,7 +1,11 @@
'use strict'; 'use strict';
var fs = require('fs'); const fs = require('fs');
var os = require("os"); const os = require('os');
const _ = require('lodash');
const { createLogger, format, transports } = require('winston');
const { combine, timestamp, printf } = format;
const debug = require('debug')('se-scraper:ScrapeManager');
const UserAgent = require('user-agents'); const UserAgent = require('user-agents');
const google = require('./modules/google.js'); const google = require('./modules/google.js');
@ -10,8 +14,6 @@ const yandex = require('./modules/yandex.js');
const infospace = require('./modules/infospace.js'); const infospace = require('./modules/infospace.js');
const duckduckgo = require('./modules/duckduckgo.js'); const duckduckgo = require('./modules/duckduckgo.js');
const { Cluster } = require('./puppeteer-cluster/dist/index.js'); const { Cluster } = require('./puppeteer-cluster/dist/index.js');
const common = require('./modules/common.js');
var log = common.log;
const MAX_ALLOWED_BROWSERS = 6; const MAX_ALLOWED_BROWSERS = 6;
@ -63,7 +65,7 @@ class ScrapeManager {
this.scraper = null; this.scraper = null;
this.context = context; this.context = context;
this.config = { this.config = _.defaults(config, {
// the user agent to scrape with // the user agent to scrape with
user_agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3835.0 Safari/537.36', user_agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3835.0 Safari/537.36',
// if random_user_agent is set to True, a random user agent is chosen // if random_user_agent is set to True, a random user agent is chosen
@ -80,17 +82,38 @@ class ScrapeManager {
// which search engine to scrape // which search engine to scrape
search_engine: 'google', search_engine: 'google',
search_engine_name: 'google', search_engine_name: 'google',
// whether debug information should be printed logger: createLogger({
// level 0: print nothing level: 'info',
// level 1: print most important info format: combine(
// ... timestamp(),
// level 4: print all shit nobody wants to know printf(({ level, message, timestamp }) => {
debug_level: 1, return `${timestamp} [${level}] ${message}`;
})
),
transports: [
new transports.Console()
]
}),
keywords: ['nodejs rocks',], keywords: ['nodejs rocks',],
// whether to start the browser in headless mode // whether to start the browser in headless mode
headless: true, headless: true,
// specify flags passed to chrome here // specify flags passed to chrome here
chrome_flags: [], // About our defaults values https://peter.sh/experiments/chromium-command-line-switches/
chrome_flags: [
'--disable-infobars',
'--window-position=0,0',
'--ignore-certifcate-errors',
'--ignore-certifcate-errors-spki-list',
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-accelerated-2d-canvas',
'--disable-gpu',
'--window-size=1920,1040',
'--start-fullscreen',
'--hide-scrollbars',
'--disable-notifications',
],
// the number of pages to scrape for each keyword // the number of pages to scrape for each keyword
num_pages: 1, num_pages: 1,
// path to output file, data will be stored in JSON // path to output file, data will be stored in JSON
@ -115,10 +138,8 @@ class ScrapeManager {
//custom_func: resolve('examples/pluggable.js'), //custom_func: resolve('examples/pluggable.js'),
custom_func: null, custom_func: null,
throw_on_detection: false, throw_on_detection: false,
// use a proxy for all connections // List of proxies to use ['socks5://78.94.172.42:1080', 'http://localhost:1080']
// example: 'socks5://78.94.172.42:1080' proxies: [],
// example: 'http://118.174.233.10:48400'
proxy: '',
// a file with one proxy per line. Example: // a file with one proxy per line. Example:
// socks5://78.94.172.42:1080 // socks5://78.94.172.42:1080
// http://118.174.233.10:48400 // http://118.174.233.10:48400
@ -138,14 +159,9 @@ class ScrapeManager {
concurrency: Cluster.CONCURRENCY_BROWSER, concurrency: Cluster.CONCURRENCY_BROWSER,
maxConcurrency: 1, maxConcurrency: 1,
} }
}; });
this.config.proxies = []; this.logger = this.config.logger;
// overwrite default config
for (var key in config) {
this.config[key] = config[key];
}
if (config.sleep_range) { if (config.sleep_range) {
// parse an array // parse an array
@ -160,12 +176,16 @@ class ScrapeManager {
this.config.keywords = read_keywords_from_file(this.config.keyword_file); this.config.keywords = read_keywords_from_file(this.config.keyword_file);
} }
if (fs.existsSync(this.config.proxy_file)) { if (this.config.proxies && this.config.proxy_file) {
this.config.proxies = read_keywords_from_file(this.config.proxy_file); throw new Error('Either use a proxy_file or specify a proxy for all connections. Do not use both options.');
log(this.config, 1, `${this.config.proxies.length} proxies read from file.`);
} }
log(this.config, 2, this.config); if (fs.existsSync(this.config.proxy_file)) {
this.config.proxies = read_keywords_from_file(this.config.proxy_file);
this.logger.info(`${this.config.proxies.length} proxies read from file.`);
}
debug('this.config=%O', this.config);
} }
/* /*
@ -193,54 +213,16 @@ class ScrapeManager {
} }
} }
// See here: https://peter.sh/experiments/chromium-command-line-switches/ const chrome_flags = _.clone(this.config.chrome_flags);
var default_chrome_flags = [
'--disable-infobars',
'--window-position=0,0',
'--ignore-certifcate-errors',
'--ignore-certifcate-errors-spki-list',
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-accelerated-2d-canvas',
'--disable-gpu',
'--window-size=1920,1040',
'--start-fullscreen',
'--hide-scrollbars',
'--disable-notifications',
];
var chrome_flags = default_chrome_flags.slice(); // copy that
if (Array.isArray(this.config.chrome_flags) && this.config.chrome_flags.length) {
chrome_flags = this.config.chrome_flags;
}
var user_agent = null;
if (this.config.user_agent) {
user_agent = this.config.user_agent;
}
if (this.config.random_user_agent) { if (this.config.random_user_agent) {
const userAgent = new UserAgent({ deviceCategory: 'desktop' }); const userAgent = new UserAgent({ deviceCategory: 'desktop' });
user_agent = userAgent.toString(); this.config.user_agent = userAgent.toString();
} }
if (user_agent) { if (this.config.user_agent) {
chrome_flags.push( chrome_flags.push(
`--user-agent=${user_agent}` `--user-agent=${this.config.user_agent}`
)
}
if (this.config.proxy) {
if (this.config.proxies && this.config.proxies.length > 0) {
console.error('Either use a proxy_file or specify a proxy for all connections. Do not use both options.');
return false;
}
chrome_flags.push(
'--proxy-server=' + this.config.proxy,
) )
} }
@ -250,7 +232,7 @@ class ScrapeManager {
ignoreHTTPSErrors: true, ignoreHTTPSErrors: true,
}; };
log(this.config, 2, `Using the following puppeteer configuration: ${launch_args}`); debug('Using the following puppeteer configuration launch_args=%O', launch_args);
if (this.pluggable && this.pluggable.start_browser) { if (this.pluggable && this.pluggable.start_browser) {
launch_args.config = this.config; launch_args.config = this.config;
@ -259,7 +241,6 @@ class ScrapeManager {
} else { } else {
// if no custom start_browser functionality was given // if no custom start_browser functionality was given
// use puppeteer-cluster for scraping // use puppeteer-cluster for scraping
const { Cluster } = require('./puppeteer-cluster/dist/index.js');
this.numClusters = this.config.puppeteer_cluster_config.maxConcurrency; this.numClusters = this.config.puppeteer_cluster_config.maxConcurrency;
var perBrowserOptions = []; var perBrowserOptions = [];
@ -283,7 +264,7 @@ class ScrapeManager {
MAX_ALLOWED_BROWSERS MAX_ALLOWED_BROWSERS
); );
log(this.config, 1, `Using ${this.numClusters} clusters.`); this.logger.info(`Using ${this.numClusters} clusters.`);
this.config.puppeteer_cluster_config.maxConcurrency = this.numClusters; this.config.puppeteer_cluster_config.maxConcurrency = this.numClusters;
@ -306,9 +287,7 @@ class ScrapeManager {
}) })
} }
if (this.config.debug_level >= 2) { debug('perBrowserOptions=%O', perBrowserOptions)
console.dir(perBrowserOptions)
}
this.cluster = await Cluster.launch({ this.cluster = await Cluster.launch({
monitor: this.config.puppeteer_cluster_config.monitor, monitor: this.config.puppeteer_cluster_config.monitor,
@ -320,8 +299,8 @@ class ScrapeManager {
}); });
this.cluster.on('taskerror', (err, data) => { this.cluster.on('taskerror', (err, data) => {
console.log(`Error while scraping ${data}: ${err.message}`); this.logger.error(`Error while scraping ${data}: ${err.message}`);
console.log(err); debug('Error during cluster task', err);
}); });
} }
} }
@ -332,8 +311,7 @@ class ScrapeManager {
async scrape(scrape_config = {}) { async scrape(scrape_config = {}) {
if (!scrape_config.keywords && !scrape_config.keyword_file) { if (!scrape_config.keywords && !scrape_config.keyword_file) {
console.error('Either keywords or keyword_file must be supplied to scrape()'); throw new Error('Either keywords or keyword_file must be supplied to scrape()');
return false;
} }
Object.assign(this.config, scrape_config); Object.assign(this.config, scrape_config);
@ -345,10 +323,7 @@ class ScrapeManager {
this.config.search_engine_name = typeof this.config.search_engine === 'function' ? this.config.search_engine.name : this.config.search_engine; this.config.search_engine_name = typeof this.config.search_engine === 'function' ? this.config.search_engine.name : this.config.search_engine;
if (this.config.keywords && this.config.search_engine) { this.logger.info(`scrapes ${this.config.search_engine_name} with ${this.config.keywords.length} keywords on ${this.config.num_pages} pages each.`);
log(this.config, 1,
`[se-scraper] started at [${(new Date()).toUTCString()}] and scrapes ${this.config.search_engine_name} with ${this.config.keywords.length} keywords on ${this.config.num_pages} pages each.`)
}
if (this.pluggable && this.pluggable.start_browser) { if (this.pluggable && this.pluggable.start_browser) {
@ -412,8 +387,8 @@ class ScrapeManager {
let timeDelta = Date.now() - startTime; let timeDelta = Date.now() - startTime;
let ms_per_request = timeDelta/num_requests; let ms_per_request = timeDelta/num_requests;
log(this.config, 1, `Scraper took ${timeDelta}ms to perform ${num_requests} requests.`); this.logger.info(`Scraper took ${timeDelta}ms to perform ${num_requests} requests.`);
log(this.config, 1, `On average ms/request: ${ms_per_request}ms/request`); this.logger.info(`On average ms/request: ${ms_per_request}ms/request`);
if (this.pluggable && this.pluggable.handle_results) { if (this.pluggable && this.pluggable.handle_results) {
await this.pluggable.handle_results(results); await this.pluggable.handle_results(results);
@ -423,14 +398,14 @@ class ScrapeManager {
metadata.ms_per_keyword = ms_per_request.toString(); metadata.ms_per_keyword = ms_per_request.toString();
metadata.num_requests = num_requests; metadata.num_requests = num_requests;
log(this.config, 2, metadata); debug('metadata=%O', metadata);
if (this.pluggable && this.pluggable.handle_metadata) { if (this.pluggable && this.pluggable.handle_metadata) {
await this.pluggable.handle_metadata(metadata); await this.pluggable.handle_metadata(metadata);
} }
if (this.config.output_file) { if (this.config.output_file) {
log(this.config, 1, `Writing results to ${this.config.output_file}`); this.logger.info(`Writing results to ${this.config.output_file}`);
write_results(this.config.output_file, JSON.stringify(results, null, 4)); write_results(this.config.output_file, JSON.stringify(results, null, 4));
} }