extern/se-scraper
1
0
Fork 1
mirror of https://github.com/NikolaiT/se-scraper.git synced 2025-03-13 12:38:16 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(google): shopping top and right working now

Browse Source
This commit is contained in:
HugoPoi 2020-06-17 17:26:29 +02:00
parent c132f6e31e
commit 77b749e3fc
3 changed files with 242 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/modules/google.js
View File

@ -13,11 +13,11 @@ class GoogleScraper extends Scraper {
const results = await this.page.evaluate(() => { const results = await this.page.evaluate(() => {
let _text = (el, s) => { let _text = (el, s, onlyFirstTextNode) => {
let n = el.querySelector(s); let n = el.querySelector(s);
if (n) { if (n) {
return n.innerText; return (onlyFirstTextNode) ? n.childNodes[0].nodeValue : n.innerText;
} else { } else {
return; return;
} }
@ -153,9 +153,9 @@ class GoogleScraper extends Scraper {
tracking_link: _attr(el, '.pla-unit-title a:first-child', 'href'), tracking_link: _attr(el, '.pla-unit-title a:first-child', 'href'),
link: _attr(el, '.pla-unit-title a:nth-child(2)', 'href'), link: _attr(el, '.pla-unit-title a:nth-child(2)', 'href'),
title: _text(el, '.pla-unit-title a:nth-child(2) span'), title: _text(el, '.pla-unit-title a:nth-child(2) span'),
price: el.querySelector('.pla-unit-title + div').childNodes[0].nodeValue, price: _text(el, '.pla-unit-title + div', true),
originalPrice: _text(el, '.pla-unit-title + div > span'), originalPrice: _text(el, '.pla-unit-title + div > span'),
//shipping: _text(el, '.pla-extensions-container div:nth-of-type(1)'), // TODO get a sample page with this shipping: _text(el, '.pla-extensions-container .cYBBsb'),
vendor_link: _attr(el,'.pla-extensions-container a.FfKHB', 'href'), vendor_link: _attr(el,'.pla-extensions-container a.FfKHB', 'href'),
merchant_name: _text(el,'.LbUacb span:nth-child(1)'), merchant_name: _text(el,'.LbUacb span:nth-child(1)'),
}; };

209
test/mocks/google/shopping 2_page1.html Normal file
View File

File diff suppressed because one or more lines are too long

30
test/modules/google.js
View File

@ -120,7 +120,7 @@ describe('Module Google', function(){
}); });
}); });
it('extract google shopping', function () { it('extract google shopping on right', function () {
const googleScraper = new GoogleScraper({ const googleScraper = new GoogleScraper({
config: { config: {
search_engine_name: 'google', search_engine_name: 'google',
@ -195,6 +195,34 @@ describe('Module Google', function(){
}); });
}); });
it('extract google shopping on top', function () {
const googleScraper = new GoogleScraper({
config: {
search_engine_name: 'google',
throw_on_detection: true,
keywords: ['shopping 2'],
logger: testLogger,
scrape_from_file: '',
num_pages: 1,
}
});
googleScraper.STANDARD_TIMEOUT = 500;
return googleScraper.run({page}).then(({results, metadata, num_requests}) => {
assert.strictEqual(num_requests, 1, 'One request should be done');
assert.strictEqual(results['shopping 2']['1'].results.length, 10, 'Must have 10 organic results parsed on page 1');
assert.deepEqual(results['shopping 2']['1'].top_products[2], {
"link": "https://www.zalando.fr/lacoste-sideline-cub-chaussons-pour-bebe-whitegreen-la216f003-k11.html?size=17&allophones=0",
"merchant_name": "Zalando.fr",
"price": "31,95 €",
"rank": 3,
'shipping': 'Livraison gratuite',
"title": "Lacoste Sideline CUB Cadeau de naissance white/green, gender.kids.unisex, Taille: 17, Blanc - Imitation cuir/textile",
"tracking_link": "/aclk?sa=l&ai=DChcSEwjt7o3yj4nqAhVZhdUKHbshBNwYABASGgJ3cw&sig=AOD64_0usikwrH4jD5vqtbS7vVoCrNxMOg&ctype=5&q=&ved=2ahUKEwj0w4fyj4nqAhWZDGMBHY7HAzAQww96BAgOEFI&adurl=",
"vendor_link": "https://fr.shoptail.eu/cheap%20lacoste%20shoes",
})
});
});
it('shopping extract right one product', function () { it('shopping extract right one product', function () {
const googleScraper = new GoogleScraper({ const googleScraper = new GoogleScraper({
config: { config: {