2023-07-21 21:53:54 +02:00
|
|
|
const axios = require('axios');
|
|
|
|
const xml2js = require('xml2js');
|
2023-08-09 23:33:36 +02:00
|
|
|
const middleware = require('./_common/middleware');
|
2023-07-21 21:53:54 +02:00
|
|
|
|
2023-08-09 23:33:36 +02:00
|
|
|
const fetchSitemapHandler = async (url) => {
|
2023-07-21 21:53:54 +02:00
|
|
|
let sitemapUrl;
|
|
|
|
|
|
|
|
try {
|
|
|
|
// Fetch robots.txt
|
|
|
|
const robotsRes = await axios.get(`${url}/robots.txt`);
|
|
|
|
const robotsTxt = robotsRes.data.split('\n');
|
|
|
|
|
|
|
|
for (let line of robotsTxt) {
|
|
|
|
if (line.startsWith('Sitemap:')) {
|
|
|
|
sitemapUrl = line.split(' ')[1];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!sitemapUrl) {
|
2023-08-09 23:33:36 +02:00
|
|
|
throw new Error('Sitemap not found in robots.txt');
|
2023-07-21 21:53:54 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Fetch sitemap
|
|
|
|
const sitemapRes = await axios.get(sitemapUrl);
|
|
|
|
const sitemap = await xml2js.parseStringPromise(sitemapRes.data);
|
|
|
|
|
2023-08-09 23:33:36 +02:00
|
|
|
return sitemap;
|
2023-07-21 21:53:54 +02:00
|
|
|
} catch (error) {
|
2023-08-09 23:33:36 +02:00
|
|
|
throw new Error(error.message);
|
2023-07-21 21:53:54 +02:00
|
|
|
}
|
|
|
|
};
|
2023-08-09 23:33:36 +02:00
|
|
|
|
|
|
|
exports.handler = middleware(fetchSitemapHandler);
|