From c132f6e31e2de135ca7b846653f827f8cbf98e5f Mon Sep 17 00:00:00 2001 From: HugoPoi Date: Wed, 17 Jun 2020 16:40:22 +0200 Subject: [PATCH] fix(google): parser shopping top --- src/modules/google.js | 23 ++++++--------- test/modules/google.js | 65 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 22 deletions(-) diff --git a/src/modules/google.js b/src/modules/google.js index a1ae66f..7ca9cad 100644 --- a/src/modules/google.js +++ b/src/modules/google.js @@ -19,7 +19,7 @@ class GoogleScraper extends Scraper { if (n) { return n.innerText; } else { - return ''; + return; } }; @@ -147,26 +147,19 @@ class GoogleScraper extends Scraper { } } - // parse top main column product information - // #tvcap .pla-unit - document.querySelectorAll('#tvcap .pla-unit').forEach((el) => { + // Parse Google Shopping top or left + document.querySelectorAll('.pla-unit').forEach((el) => { let top_product = { tracking_link: _attr(el, '.pla-unit-title a:first-child', 'href'), link: _attr(el, '.pla-unit-title a:nth-child(2)', 'href'), title: _text(el, '.pla-unit-title a:nth-child(2) span'), - price: _text(el, '.pla-unit-title + div'), - shipping: _text(el, '.pla-extensions-container div:nth-of-type(1)'), - vendor_link: _attr(el,'.pla-extensions-container div > a', 'href'), + price: el.querySelector('.pla-unit-title + div').childNodes[0].nodeValue, + originalPrice: _text(el, '.pla-unit-title + div > span'), + //shipping: _text(el, '.pla-extensions-container div:nth-of-type(1)'), // TODO get a sample page with this + vendor_link: _attr(el,'.pla-extensions-container a.FfKHB', 'href'), + merchant_name: _text(el,'.LbUacb span:nth-child(1)'), }; - let merchant_node = el.querySelector('.pla-unit-title'); - if (merchant_node) { - let node = merchant_node.parentNode.querySelector('div > span'); - if (node) { - top_product.merchant_name = node.innerText; - } - } - results.top_products.push(top_product); }); diff --git a/test/modules/google.js b/test/modules/google.js index 5372314..22e2285 100644 --- a/test/modules/google.js +++ b/test/modules/google.js @@ -134,12 +134,64 @@ describe('Module Google', function(){ googleScraper.STANDARD_TIMEOUT = 500; return googleScraper.run({page}).then(({results, metadata, num_requests}) => { assert.strictEqual(num_requests, 1, 'One request should be done'); - assert.strictEqual(results['test keyword']['1'].results.length, 10, 'Must have 10 organic results parsed on page 1'); - assert.strictEqual(results['test keyword']['1'].results[0].title, 'Keyword Tool (FREE) ᐈ #1 Google Keyword Planner Alternative', 'Title not matching on first organic result page 1'); - assert.strictEqual(results['test keyword']['2'].results.length, 10, 'Must have 10 organic results parsed on page 2'); - assert.strictEqual(results['test keyword']['2'].results[0].title, 'Keyword Research | The Beginner\'s Guide to SEO - Moz', 'Title not matching on first organic result page 1'); - assert.strictEqual(results['test keyword']['3'].results.length, 10, 'Must have 10 organic results parsed on page 3'); - assert.strictEqual(results['test keyword']['3'].results[0].title, 'The ACT Keyword Study Plan — NerdCoach', 'Title not matching on first organic result page 1'); + assert.strictEqual(results['shopping']['1'].results.length, 10, 'Must have 10 organic results parsed on page 1'); + assert.deepEqual(results['shopping']['1'].top_products, [ + { + 'link': 'https://www.laboutiqueofficielle.com/achat-baskets-basses/classic-series-baskets-317-blanc-144046.html?referer=gshopping&LGWCODE=3010559970809;160079;7403', + 'merchant_name': 'LaBoutiqueOffi...', + 'price': '39,99 €', + 'rank': 1, + 'title': 'Classic Series - Baskets 317 Blanc', + 'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAEGgJsZQ&sig=AOD64_1OEdvZgHU2YEMPI4JNdeTqLJTVjw&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEFU&adurl=', + 'vendor_link': 'https://www.google.com/search?tbm=shop&q=cheap%20lacoste%20shoes', + }, + { + 'link': 'https://www.chausport.com/p/lacoste-carnaby-evo-noire-enfant-173257.html', + 'merchant_name': 'Chausport', + 'price': '45,00 €', + 'rank': 2, + 'title': 'Tennis Lacoste Carnaby Evo Noire Enfant 28', + 'tracking_link': '/aclk?sa=L&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAFGgJsZQ&sig=AOD64_0lhZrLNYCENmxzquCMa5M4_D04ng&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEGA&adurl=', + 'vendor_link': 'http://www.choozen.fr/nf/gs-cheap%20lacoste%20shoes.htm?kpartnerid=96955353', + }, + { + 'link': 'https://www.getthelabel.com/fr/p/lacoste-baskets-lerond-418/138256', + 'merchant_name': 'GetTheLabel.c...', + 'price': '44,99 €', + 'rank': 3, + 'title': 'Lacoste Baskets Lerond 418 Size 9 in Blanc pour Homme', + 'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAIGgJsZQ&sig=AOD64_13MoA9It0w-yp3GqriMf13OPLI8w&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEG0&adurl=', + 'vendor_link': 'https://highstreetone.com/?search=cheap%20lacoste%20shoes', + }, + { + 'link': 'https://www.sarenza.com/lacoste-carnaby-evo-120-2-s834061-br918-t76-p0000227925#size=39-39', + 'merchant_name': 'Sarenza', + 'price': '45,50 €', + 'originalPrice': '65 €', + 'rank': 4, + 'title': 'Lacoste Carnaby Evo 120 2 Blanc - Baskets - Disponible en 39', + 'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABANGgJsZQ&sig=AOD64_1Q6WUe8YXjhb-y_k0rErD2WUsTqQ&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEHk&adurl=', + 'vendor_link': 'https://www.feed-price.com/search/cheap%20lacoste%20shoes', + }, + { + 'link': 'https://www.spartoo.com/Lacoste-CARNABY-EVO-BL-1-x4736301.php?track_id=adwo_fgl&sx=B&utm_source=froogle&utm_medium=comparateurs&utm_content=4736301&utm_campaign=adwo_fgl&size_id=158&fcsize=1&sx=B', + 'merchant_name': 'Spartoo.com', + 'price': '58,00 €', + 'rank': 5, + 'title': 'Lacoste CARNABY EVO BL 1 Baskets basses enfant (garcons)', + 'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAMGgJsZQ&sig=AOD64_0NfyG0tH5Pc7kPfADKcQflx78H1g&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BQgOEIcB&adurl=', + 'vendor_link': 'https://www.google.com/search?tbm=shop&q=cheap%20lacoste%20shoes', + }, + { + 'link': 'https://www.nike.com/fr/t/nikecourt-royale-shoe-KyTwJwgV/749747-111', + 'merchant_name': 'Nike Officiel', + 'price': '55,00 €', + 'rank': 6, + 'title': 'Chaussure Nike Court Royale pour Homme - Blanc', + 'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABASGgJsZQ&sig=AOD64_2KQENuVGnvXutmSUufDSa4FnTYsw&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BQgOEJIB&adurl=', + 'vendor_link': 'https://www.pricesearcher.com/css/search/?p=1&q=cheap%20lacoste%20shoes&utm_source=google&utm_medium=css', + } + ]) }); }); @@ -160,7 +212,6 @@ describe('Module Google', function(){ assert.strictEqual(results['shopping right product review']['1'].results.length, 9, 'Must have 9 organic results parsed on page 1'); assert.deepEqual(results['shopping right product review']['1'].right_info, { title: 'Lacoste Lunettes', - 'info': '', 'num_reviews': '146 avis', 'review': 'Note : 4,6 sur 5', 'vendors': [