mirror of
https://github.com/NikolaiT/se-scraper.git
synced 2025-06-28 21:41:27 +02:00
fix(google): parser shopping top
This commit is contained in:
parent
ad8903b001
commit
c132f6e31e
@ -19,7 +19,7 @@ class GoogleScraper extends Scraper {
|
|||||||
if (n) {
|
if (n) {
|
||||||
return n.innerText;
|
return n.innerText;
|
||||||
} else {
|
} else {
|
||||||
return '';
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -147,26 +147,19 @@ class GoogleScraper extends Scraper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// parse top main column product information
|
// Parse Google Shopping top or left
|
||||||
// #tvcap .pla-unit
|
document.querySelectorAll('.pla-unit').forEach((el) => {
|
||||||
document.querySelectorAll('#tvcap .pla-unit').forEach((el) => {
|
|
||||||
let top_product = {
|
let top_product = {
|
||||||
tracking_link: _attr(el, '.pla-unit-title a:first-child', 'href'),
|
tracking_link: _attr(el, '.pla-unit-title a:first-child', 'href'),
|
||||||
link: _attr(el, '.pla-unit-title a:nth-child(2)', 'href'),
|
link: _attr(el, '.pla-unit-title a:nth-child(2)', 'href'),
|
||||||
title: _text(el, '.pla-unit-title a:nth-child(2) span'),
|
title: _text(el, '.pla-unit-title a:nth-child(2) span'),
|
||||||
price: _text(el, '.pla-unit-title + div'),
|
price: el.querySelector('.pla-unit-title + div').childNodes[0].nodeValue,
|
||||||
shipping: _text(el, '.pla-extensions-container div:nth-of-type(1)'),
|
originalPrice: _text(el, '.pla-unit-title + div > span'),
|
||||||
vendor_link: _attr(el,'.pla-extensions-container div > a', 'href'),
|
//shipping: _text(el, '.pla-extensions-container div:nth-of-type(1)'), // TODO get a sample page with this
|
||||||
|
vendor_link: _attr(el,'.pla-extensions-container a.FfKHB', 'href'),
|
||||||
|
merchant_name: _text(el,'.LbUacb span:nth-child(1)'),
|
||||||
};
|
};
|
||||||
|
|
||||||
let merchant_node = el.querySelector('.pla-unit-title');
|
|
||||||
if (merchant_node) {
|
|
||||||
let node = merchant_node.parentNode.querySelector('div > span');
|
|
||||||
if (node) {
|
|
||||||
top_product.merchant_name = node.innerText;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
results.top_products.push(top_product);
|
results.top_products.push(top_product);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -134,12 +134,64 @@ describe('Module Google', function(){
|
|||||||
googleScraper.STANDARD_TIMEOUT = 500;
|
googleScraper.STANDARD_TIMEOUT = 500;
|
||||||
return googleScraper.run({page}).then(({results, metadata, num_requests}) => {
|
return googleScraper.run({page}).then(({results, metadata, num_requests}) => {
|
||||||
assert.strictEqual(num_requests, 1, 'One request should be done');
|
assert.strictEqual(num_requests, 1, 'One request should be done');
|
||||||
assert.strictEqual(results['test keyword']['1'].results.length, 10, 'Must have 10 organic results parsed on page 1');
|
assert.strictEqual(results['shopping']['1'].results.length, 10, 'Must have 10 organic results parsed on page 1');
|
||||||
assert.strictEqual(results['test keyword']['1'].results[0].title, 'Keyword Tool (FREE) ᐈ #1 Google Keyword Planner Alternative', 'Title not matching on first organic result page 1');
|
assert.deepEqual(results['shopping']['1'].top_products, [
|
||||||
assert.strictEqual(results['test keyword']['2'].results.length, 10, 'Must have 10 organic results parsed on page 2');
|
{
|
||||||
assert.strictEqual(results['test keyword']['2'].results[0].title, 'Keyword Research | The Beginner\'s Guide to SEO - Moz', 'Title not matching on first organic result page 1');
|
'link': 'https://www.laboutiqueofficielle.com/achat-baskets-basses/classic-series-baskets-317-blanc-144046.html?referer=gshopping&LGWCODE=3010559970809;160079;7403',
|
||||||
assert.strictEqual(results['test keyword']['3'].results.length, 10, 'Must have 10 organic results parsed on page 3');
|
'merchant_name': 'LaBoutiqueOffi...',
|
||||||
assert.strictEqual(results['test keyword']['3'].results[0].title, 'The ACT Keyword Study Plan — NerdCoach', 'Title not matching on first organic result page 1');
|
'price': '39,99 €',
|
||||||
|
'rank': 1,
|
||||||
|
'title': 'Classic Series - Baskets 317 Blanc',
|
||||||
|
'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAEGgJsZQ&sig=AOD64_1OEdvZgHU2YEMPI4JNdeTqLJTVjw&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEFU&adurl=',
|
||||||
|
'vendor_link': 'https://www.google.com/search?tbm=shop&q=cheap%20lacoste%20shoes',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'link': 'https://www.chausport.com/p/lacoste-carnaby-evo-noire-enfant-173257.html',
|
||||||
|
'merchant_name': 'Chausport',
|
||||||
|
'price': '45,00 €',
|
||||||
|
'rank': 2,
|
||||||
|
'title': 'Tennis Lacoste Carnaby Evo Noire Enfant 28',
|
||||||
|
'tracking_link': '/aclk?sa=L&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAFGgJsZQ&sig=AOD64_0lhZrLNYCENmxzquCMa5M4_D04ng&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEGA&adurl=',
|
||||||
|
'vendor_link': 'http://www.choozen.fr/nf/gs-cheap%20lacoste%20shoes.htm?kpartnerid=96955353',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'link': 'https://www.getthelabel.com/fr/p/lacoste-baskets-lerond-418/138256',
|
||||||
|
'merchant_name': 'GetTheLabel.c...',
|
||||||
|
'price': '44,99 €',
|
||||||
|
'rank': 3,
|
||||||
|
'title': 'Lacoste Baskets Lerond 418 Size 9 in Blanc pour Homme',
|
||||||
|
'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAIGgJsZQ&sig=AOD64_13MoA9It0w-yp3GqriMf13OPLI8w&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEG0&adurl=',
|
||||||
|
'vendor_link': 'https://highstreetone.com/?search=cheap%20lacoste%20shoes',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'link': 'https://www.sarenza.com/lacoste-carnaby-evo-120-2-s834061-br918-t76-p0000227925#size=39-39',
|
||||||
|
'merchant_name': 'Sarenza',
|
||||||
|
'price': '45,50 €',
|
||||||
|
'originalPrice': '65 €',
|
||||||
|
'rank': 4,
|
||||||
|
'title': 'Lacoste Carnaby Evo 120 2 Blanc - Baskets - Disponible en 39',
|
||||||
|
'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABANGgJsZQ&sig=AOD64_1Q6WUe8YXjhb-y_k0rErD2WUsTqQ&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BAgOEHk&adurl=',
|
||||||
|
'vendor_link': 'https://www.feed-price.com/search/cheap%20lacoste%20shoes',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'link': 'https://www.spartoo.com/Lacoste-CARNABY-EVO-BL-1-x4736301.php?track_id=adwo_fgl&sx=B&utm_source=froogle&utm_medium=comparateurs&utm_content=4736301&utm_campaign=adwo_fgl&size_id=158&fcsize=1&sx=B',
|
||||||
|
'merchant_name': 'Spartoo.com',
|
||||||
|
'price': '58,00 €',
|
||||||
|
'rank': 5,
|
||||||
|
'title': 'Lacoste CARNABY EVO BL 1 Baskets basses enfant (garcons)',
|
||||||
|
'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABAMGgJsZQ&sig=AOD64_0NfyG0tH5Pc7kPfADKcQflx78H1g&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BQgOEIcB&adurl=',
|
||||||
|
'vendor_link': 'https://www.google.com/search?tbm=shop&q=cheap%20lacoste%20shoes',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'link': 'https://www.nike.com/fr/t/nikecourt-royale-shoe-KyTwJwgV/749747-111',
|
||||||
|
'merchant_name': 'Nike Officiel',
|
||||||
|
'price': '55,00 €',
|
||||||
|
'rank': 6,
|
||||||
|
'title': 'Chaussure Nike Court Royale pour Homme - Blanc',
|
||||||
|
'tracking_link': '/aclk?sa=l&ai=DChcSEwjJqLX1v4bqAhXJlBgKHYRrDO4YABASGgJsZQ&sig=AOD64_2KQENuVGnvXutmSUufDSa4FnTYsw&ctype=5&q=&ved=2ahUKEwjPmK31v4bqAhXLxYUKHe8BByEQ9A56BQgOEJIB&adurl=',
|
||||||
|
'vendor_link': 'https://www.pricesearcher.com/css/search/?p=1&q=cheap%20lacoste%20shoes&utm_source=google&utm_medium=css',
|
||||||
|
}
|
||||||
|
])
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -160,7 +212,6 @@ describe('Module Google', function(){
|
|||||||
assert.strictEqual(results['shopping right product review']['1'].results.length, 9, 'Must have 9 organic results parsed on page 1');
|
assert.strictEqual(results['shopping right product review']['1'].results.length, 9, 'Must have 9 organic results parsed on page 1');
|
||||||
assert.deepEqual(results['shopping right product review']['1'].right_info, {
|
assert.deepEqual(results['shopping right product review']['1'].right_info, {
|
||||||
title: 'Lacoste Lunettes',
|
title: 'Lacoste Lunettes',
|
||||||
'info': '',
|
|
||||||
'num_reviews': '146 avis',
|
'num_reviews': '146 avis',
|
||||||
'review': 'Note : 4,6 sur 5',
|
'review': 'Note : 4,6 sur 5',
|
||||||
'vendors': [
|
'vendors': [
|
||||||
|
Loading…
x
Reference in New Issue
Block a user