diff --git a/src/modules/google.js b/src/modules/google.js
index 7ca9cad..2d10422 100644
--- a/src/modules/google.js
+++ b/src/modules/google.js
@@ -13,11 +13,11 @@ class GoogleScraper extends Scraper {
const results = await this.page.evaluate(() => {
- let _text = (el, s) => {
+ let _text = (el, s, onlyFirstTextNode) => {
let n = el.querySelector(s);
if (n) {
- return n.innerText;
+ return (onlyFirstTextNode) ? n.childNodes[0].nodeValue : n.innerText;
} else {
return;
}
@@ -153,9 +153,9 @@ class GoogleScraper extends Scraper {
tracking_link: _attr(el, '.pla-unit-title a:first-child', 'href'),
link: _attr(el, '.pla-unit-title a:nth-child(2)', 'href'),
title: _text(el, '.pla-unit-title a:nth-child(2) span'),
- price: el.querySelector('.pla-unit-title + div').childNodes[0].nodeValue,
+ price: _text(el, '.pla-unit-title + div', true),
originalPrice: _text(el, '.pla-unit-title + div > span'),
- //shipping: _text(el, '.pla-extensions-container div:nth-of-type(1)'), // TODO get a sample page with this
+ shipping: _text(el, '.pla-extensions-container .cYBBsb'),
vendor_link: _attr(el,'.pla-extensions-container a.FfKHB', 'href'),
merchant_name: _text(el,'.LbUacb span:nth-child(1)'),
};
diff --git a/test/mocks/google/shopping 2_page1.html b/test/mocks/google/shopping 2_page1.html
new file mode 100644
index 0000000..4342228
--- /dev/null
+++ b/test/mocks/google/shopping 2_page1.html
@@ -0,0 +1,209 @@
+
cheap lacoste shoes - Recherche Google
Cliquez
ici si, d'ici quelques secondes, vous n'avez pas été redirigé.
Environ 55 700 000 résultats (0,48 secondes)
Rappel concernant les règles de confidentialité de Google
Me le rappeler plus tard Lire
Afficher les produits correspondants à cheap lacoste... 39,99 €
LaBoutiqueOffic...
49,90 €
LaBoutiqueOffic...
64,95 €
LaBoutiqueOffic...
Annonces Commandez Vite sur le Site Officiel. Livraison Express 48h dès 180€ d'achat !
Note associée à lacoste.com : 4,8 - Conditions de retour: 60 jours ou plus pour la plupart des articles
Chic et intemporel, vous porterez cet essentiel en toutes occasions.
L'Elégance d'une coupe Chemise : Découvrez Le Nouveau Paris Polo.
lacoste discount
lacoste shoes homme
lacoste usa
lacoste shoes femme
get the label lacoste
Résultats de recherche Résultats de recherche à proximité Les horaires ou les services proposés peuvent varier Résultats Web Browse cheap Lacoste trainers at low prices. Find smart shoes to look good and feel great. Get big brand footwear for less, now!
Get huge savings on our great range of Lacoste trainers , polos, t-shirts and more for men and women at MandM Direct. But hurry, once it's gone, it's gone!
Discover our collections on the official Lacoste online store: Clothes, shoes , bags and accessories for men, women and kids.
114 items - Save up to 75% on our huge range of Lacoste clothing including polos, t shirts and trainers for men, women & kids. Shop now and get great deals for ...
Cheap Lacoste trainer at Soletrader Outlet with at least 30% off the range, free delivery on orders over £50 and easy returns.
289 items - Free shipping BOTH ways on Lacoste , Shoes , Men from our vast selection of styles. Fast delivery, and 24/7/365 real-person service with a smile.
Lacoste Mens trainers sale ✅ now on with up to 60% off ✅ Huge discounts on High tops, Plimsolls and more ✅ from the biggest online sales & clearance outlet.
Discounted shoes , clothing, accessories and more at 6pm.com! ... 6pm - Your Premier Destination for Discount Fashion ... René Lacoste entered the legend of tennis when he and his teammates "The Musketeers", stole the Davis Cup away ... Results 1 - 43 of 43 - Incredible savings with up to 75% off in the Mens Lacoste Trainers Sale! The best prices in the UK on Cheap Lacoste Trainers and Shoes ...
2411 products - Find Lacoste sales up to 93% off from 38 stores. Shop the best Lacoste sale from the most popular stores.
Annonces Nouvelle Collection - Vêtements - Chaussures - Urban Sport - Accessoires Jusqu'à -70%. Choisissez les modèles qui conviennent le votre style, Vaste choix de tailles & modèles . Manches courtes. Coupe classique. Coupe slim. Types: Polo, T-Shirts, Sweatshirts.
Lacoste Chaussures 200 Modèles à Prix Promo. Déstockage - Lacoste Chaussures à prix web ! Modèles de Lacoste Chaussures à Prix Super Réduits. Vite: Lacoste Chaussures à Saisir ! Comparez plus de prix. Faites des économies. Enchères et prix fixe.
diff --git a/test/modules/google.js b/test/modules/google.js
index 22e2285..d008e7a 100644
--- a/test/modules/google.js
+++ b/test/modules/google.js
@@ -120,7 +120,7 @@ describe('Module Google', function(){
});
});
- it('extract google shopping', function () {
+ it('extract google shopping on right', function () {
const googleScraper = new GoogleScraper({
config: {
search_engine_name: 'google',
@@ -195,6 +195,34 @@ describe('Module Google', function(){
});
});
+ it('extract google shopping on top', function () {
+ const googleScraper = new GoogleScraper({
+ config: {
+ search_engine_name: 'google',
+ throw_on_detection: true,
+ keywords: ['shopping 2'],
+ logger: testLogger,
+ scrape_from_file: '',
+ num_pages: 1,
+ }
+ });
+ googleScraper.STANDARD_TIMEOUT = 500;
+ return googleScraper.run({page}).then(({results, metadata, num_requests}) => {
+ assert.strictEqual(num_requests, 1, 'One request should be done');
+ assert.strictEqual(results['shopping 2']['1'].results.length, 10, 'Must have 10 organic results parsed on page 1');
+ assert.deepEqual(results['shopping 2']['1'].top_products[2], {
+ "link": "https://www.zalando.fr/lacoste-sideline-cub-chaussons-pour-bebe-whitegreen-la216f003-k11.html?size=17&allophones=0",
+ "merchant_name": "Zalando.fr",
+ "price": "31,95 €",
+ "rank": 3,
+ 'shipping': 'Livraison gratuite',
+ "title": "Lacoste Sideline CUB Cadeau de naissance white/green, gender.kids.unisex, Taille: 17, Blanc - Imitation cuir/textile",
+ "tracking_link": "/aclk?sa=l&ai=DChcSEwjt7o3yj4nqAhVZhdUKHbshBNwYABASGgJ3cw&sig=AOD64_0usikwrH4jD5vqtbS7vVoCrNxMOg&ctype=5&q=&ved=2ahUKEwj0w4fyj4nqAhWZDGMBHY7HAzAQww96BAgOEFI&adurl=",
+ "vendor_link": "https://fr.shoptail.eu/cheap%20lacoste%20shoes",
+ })
+ });
+ });
+
it('shopping extract right one product', function () {
const googleScraper = new GoogleScraper({
config: {