fix(google): parser shopping top

This commit is contained in:
HugoPoi 2020-06-17 16:40:22 +02:00
parent ad8903b001
commit c132f6e31e
2 changed files with 66 additions and 22 deletions

View File

@ -19,7 +19,7 @@ class GoogleScraper extends Scraper {
if (n) { if (n) {
return n.innerText; return n.innerText;
} else { } else {
return ''; return;
} }
}; };
@ -147,26 +147,19 @@ class GoogleScraper extends Scraper {
} }
} }
// parse top main column product information // Parse Google Shopping top or left
// #tvcap .pla-unit document.querySelectorAll('.pla-unit').forEach((el) => {
document.querySelectorAll('#tvcap .pla-unit').forEach((el) => {
let top_product = { let top_product = {
tracking_link: _attr(el, '.pla-unit-title a:first-child', 'href'), tracking_link: _attr(el, '.pla-unit-title a:first-child', 'href'),
link: _attr(el, '.pla-unit-title a:nth-child(2)', 'href'), link: _attr(el, '.pla-unit-title a:nth-child(2)', 'href'),
title: _text(el, '.pla-unit-title a:nth-child(2) span'), title: _text(el, '.pla-unit-title a:nth-child(2) span'),
price: _text(el, '.pla-unit-title + div'), price: el.querySelector('.pla-unit-title + div').childNodes[0].nodeValue,
shipping: _text(el, '.pla-extensions-container div:nth-of-type(1)'), originalPrice: _text(el, '.pla-unit-title + div > span'),
vendor_link: _attr(el,'.pla-extensions-container div > a', 'href'), //shipping: _text(el, '.pla-extensions-container div:nth-of-type(1)'), // TODO get a sample page with this
vendor_link: _attr(el,'.pla-extensions-container a.FfKHB', 'href'),
merchant_name: _text(el,'.LbUacb span:nth-child(1)'),
}; };
let merchant_node = el.querySelector('.pla-unit-title');
if (merchant_node) {
let node = merchant_node.parentNode.querySelector('div > span');
if (node) {
top_product.merchant_name = node.innerText;
}
}
results.top_products.push(top_product); results.top_products.push(top_product);
}); });

View File

@ -134,12 +134,64 @@ describe('Module Google', function(){
googleScraper.STANDARD_TIMEOUT = 500; googleScraper.STANDARD_TIMEOUT = 500;
return googleScraper.run({page}).then(({results, metadata, num_requests}) => { return googleScraper.run({page}).then(({results, metadata, num_requests}) => {
assert.strictEqual(num_requests, 1, 'One request should be done'); assert.strictEqual(num_requests, 1, 'One request should be done');
assert.strictEqual(results['test keyword']['1'].results.length, 10, 'Must have 10 organic results parsed on page 1'); assert.strictEqual(results['shopping']['1'].results.length, 10, 'Must have 10 organic results parsed on page 1');
assert.strictEqual(results['test keyword']['1'].results[0].title, 'Keyword Tool (FREE) ᐈ #1 Google Keyword Planner Alternative', 'Title not matching on first organic result page 1'); assert.deepEqual(results['shopping']['1'].top_products, [
assert.strictEqual(results['test keyword']['2'].results.length, 10, 'Must have 10 organic results parsed on page 2'); {
assert.strictEqual(results['test keyword']['2'].results[0].title, 'Keyword Research | The Beginner\'s Guide to SEO - Moz', 'Title not matching on first organic result page 1'); 'link': 'https://www.laboutiqueofficielle.com/achat-baskets-basses/classic-series-baskets-317-blanc-144046.html?referer=gshopping&LGWCODE=3010559970809;160079;7403',
assert.strictEqual(results['test keyword']['3'].results.length, 10, 'Must have 10 organic results parsed on page 3'); 'merchant_name': 'LaBoutiqueOffi...',
assert.strictEqual(results['test keyword']['3'].results[0].title, 'The ACT Keyword Study Plan — NerdCoach', 'Title not matching on first organic result page 1'); 'price': '39,99 €',
'rank': 1,
'title': 'Classic Series - Baskets 317 Blanc',
'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAEGgJsZQ&sig=AOD64_1OEdvZgHU2YEMPI4JNdeTqLJTVjw&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEFU&adurl=',
'vendor_link': 'https://www.google.com/search?tbm=shop&q=cheap%20lacoste%20shoes',
},
{
'link': 'https://www.chausport.com/p/lacoste-carnaby-evo-noire-enfant-173257.html',
'merchant_name': 'Chausport',
'price': '45,00 €',
'rank': 2,
'title': 'Tennis Lacoste Carnaby Evo Noire Enfant 28',
'tracking_link': '/aclk?sa=L&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAFGgJsZQ&sig=AOD64_0lhZrLNYCENmxzquCMa5M4_D04ng&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEGA&adurl=',
'vendor_link': 'http://www.choozen.fr/nf/gs-cheap%20lacoste%20shoes.htm?kpartnerid=96955353',
},
{
'link': 'https://www.getthelabel.com/fr/p/lacoste-baskets-lerond-418/138256',
'merchant_name': 'GetTheLabel.c...',
'price': '44,99 €',
'rank': 3,
'title': 'Lacoste Baskets Lerond 418 Size 9 in Blanc pour Homme',
'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAIGgJsZQ&sig=AOD64_13MoA9It0w-yp3GqriMf13OPLI8w&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEG0&adurl=',
'vendor_link': 'https://highstreetone.com/?search=cheap%20lacoste%20shoes',
},
{
'link': 'https://www.sarenza.com/lacoste-carnaby-evo-120-2-s834061-br918-t76-p0000227925#size=39-39',
'merchant_name': 'Sarenza',
'price': '45,50 €',
'originalPrice': '65 €',
'rank': 4,
'title': 'Lacoste Carnaby Evo 120 2 Blanc - Baskets - Disponible en 39',
'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABANGgJsZQ&sig=AOD64_1Q6WUe8YXjhb-y_k0rErD2WUsTqQ&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEHk&adurl=',
'vendor_link': 'https://www.feed-price.com/search/cheap%20lacoste%20shoes',
},
{
'link': 'https://www.spartoo.com/Lacoste-CARNABY-EVO-BL-1-x4736301.php?track_id=adwo_fgl&sx=B&utm_source=froogle&utm_medium=comparateurs&utm_content=4736301&utm_campaign=adwo_fgl&size_id=158&fcsize=1&sx=B',
'merchant_name': 'Spartoo.com',
'price': '58,00 €',
'rank': 5,
'title': 'Lacoste CARNABY EVO BL 1 Baskets basses enfant (garcons)',
'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAMGgJsZQ&sig=AOD64_0NfyG0tH5Pc7kPfADKcQflx78H1g&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BQgOEIcB&adurl=',
'vendor_link': 'https://www.google.com/search?tbm=shop&q=cheap%20lacoste%20shoes',
},
{
'link': 'https://www.nike.com/fr/t/nikecourt-royale-shoe-KyTwJwgV/749747-111',
'merchant_name': 'Nike Officiel',
'price': '55,00 €',
'rank': 6,
'title': 'Chaussure Nike Court Royale pour Homme - Blanc',
'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABASGgJsZQ&sig=AOD64_2KQENuVGnvXutmSUufDSa4FnTYsw&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BQgOEJIB&adurl=',
'vendor_link': 'https://www.pricesearcher.com/css/search/?p=1&q=cheap%20lacoste%20shoes&utm_source=google&utm_medium=css',
}
])
}); });
}); });
@ -160,7 +212,6 @@ describe('Module Google', function(){
assert.strictEqual(results['shopping right product review']['1'].results.length, 9, 'Must have 9 organic results parsed on page 1'); assert.strictEqual(results['shopping right product review']['1'].results.length, 9, 'Must have 9 organic results parsed on page 1');
assert.deepEqual(results['shopping right product review']['1'].right_info, { assert.deepEqual(results['shopping right product review']['1'].right_info, {
title: 'Lacoste Lunettes', title: 'Lacoste Lunettes',
'info': '',
'num_reviews': '146 avis', 'num_reviews': '146 avis',
'review': 'Note : 4,6 sur 5', 'review': 'Note : 4,6 sur 5',
'vendors': [ 'vendors': [