mirror of
https://github.com/NikolaiT/se-scraper.git
synced 2024-11-08 00:33:58 +01:00
93 lines
3.7 KiB
JavaScript
93 lines
3.7 KiB
JavaScript
const se_scraper = require('./index.js');
|
|
|
|
let config = {
|
|
// the user agent to scrape with
|
|
user_agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36',
|
|
// if random_user_agent is set to True, a random user agent is chosen
|
|
random_user_agent: true,
|
|
// how long to sleep between requests. a random sleep interval within the range [a,b]
|
|
// is drawn before every request. empty string for no sleeping.
|
|
sleep_range: '[1,2]',
|
|
// which search engine to scrape
|
|
search_engine: 'google',
|
|
|
|
// use specific search engine parameters for various search engines
|
|
google_settings: {
|
|
google_domain: 'google.com',
|
|
gl: 'us', // The gl parameter determines the Google country to use for the query.
|
|
hl: 'us', // The hl parameter determines the Google UI language to return results.
|
|
start: 0, // Determines the results offset to use, defaults to 0.
|
|
num: 100, // Determines the number of results to show, defaults to 10. Maximum is 100.
|
|
},
|
|
|
|
// https://docs.microsoft.com/en-us/rest/api/cognitiveservices/bing-web-api-v5-reference#query-parameters
|
|
bing_settings: {
|
|
count: 50, // how many results per page
|
|
safeSearch: 'Off', // safe search (strict, moderate, off)
|
|
cc: 'us', // ISO 3166 country code
|
|
offset: 0, // The zero-based offset that indicates the number of search results to skip before returning results
|
|
},
|
|
|
|
// whether debug information should be printed
|
|
// debug info is useful for developers when debugging
|
|
debug: false,
|
|
// whether verbose program output should be printed
|
|
// this output is informational
|
|
verbose: true,
|
|
// an array of keywords to scrape
|
|
keywords: ['good news'],
|
|
// alternatively you can specify a keyword_file. this overwrites the keywords array
|
|
keyword_file: '',
|
|
// the number of pages to scrape for each keyword
|
|
num_pages: 2,
|
|
// whether to start the browser in headless mode
|
|
headless: true,
|
|
// path to output file, data will be stored in JSON
|
|
output_file: 'examples/results/advanced.json',
|
|
// whether to prevent images, css, fonts from being loaded
|
|
// will speed up scraping a great deal
|
|
block_assets: true,
|
|
// path to js module that extends functionality
|
|
// this module should export the functions:
|
|
// get_browser, handle_metadata, close_browser
|
|
// must be an absolute path to the module
|
|
//custom_func: resolve('examples/pluggable.js'),
|
|
custom_func: '',
|
|
// use a proxy for all connections
|
|
// example: 'socks5://78.94.172.42:1080'
|
|
// example: 'http://118.174.233.10:48400'
|
|
proxy: '',
|
|
// a file with one proxy per line. Example:
|
|
// socks5://78.94.172.42:1080
|
|
// http://118.174.233.10:48400
|
|
proxy_file: '',
|
|
// check if headless chrome escapes common detection techniques
|
|
// this is a quick test and should be used for debugging
|
|
test_evasion: false,
|
|
// log ip address data
|
|
log_ip_address: false,
|
|
// log http headers
|
|
log_http_headers: false,
|
|
puppeteer_cluster_config: {
|
|
timeout: 10 * 60 * 1000, // max timeout set to 10 minutes
|
|
monitor: false,
|
|
concurrency: 1, // one scraper per tab
|
|
maxConcurrency: 1, // scrape with 2 tabs
|
|
}
|
|
};
|
|
|
|
function callback(err, response) {
|
|
if (err) { console.error(err) }
|
|
|
|
/* response object has the following properties:
|
|
|
|
response.results - json object with the scraping results
|
|
response.metadata - json object with metadata information
|
|
response.statusCode - status code of the scraping process
|
|
*/
|
|
|
|
console.dir(response.results, {depth: null, colors: true});
|
|
}
|
|
|
|
se_scraper.scrape(config, callback);
|