extern/web-check
1
0
Fork 0
mirror of https://github.com/Lissy93/web-check.git synced 2025-01-24 23:28:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Re-wrote sitemap lambda function

Browse Source
This commit is contained in:
Alicia Sykes 2023-07-29 09:38:25 +01:00
parent abff9283dc
commit df24445ac6
1 changed files with 43 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
api/sitemap.js
View File

@ -2,40 +2,60 @@ const axios = require('axios');
const xml2js = require('xml2js');
exports.handler = async (event) => {
const baseUrl = event.queryStringParameters.url.replace(/^(?:https?:\/\/)?/i, "");
const url = baseUrl.startsWith('http') ? baseUrl : `http://${baseUrl}`;
let sitemapUrl;
const url = event.queryStringParameters.url;
let sitemapUrl = `${url}/sitemap.xml`;
try {
// Fetch robots.txt
const robotsRes = await axios.get(`${url}/robots.txt`);
const robotsTxt = robotsRes.data.split('\n');
// Try to fetch sitemap directly
let sitemapRes;
try {
sitemapRes = await axios.get(sitemapUrl, { timeout: 5000 });
} catch (error) {
if (error.response && error.response.status === 404) {
// If sitemap not found, try to fetch it from robots.txt
const robotsRes = await axios.get(`${url}/robots.txt`, { timeout: 5000 });
const robotsTxt = robotsRes.data.split('\n');
for (let line of robotsTxt) {
if (line.startsWith('Sitemap:')) {
sitemapUrl = line.split(' ')[1];
for (let line of robotsTxt) {
if (line.toLowerCase().startsWith('sitemap:')) {
sitemapUrl = line.split(' ')[1].trim();
break;
}
}
if (!sitemapUrl) {
return {
statusCode: 404,
body: JSON.stringify({ skipped: 'No sitemap found' }),
};
}
sitemapRes = await axios.get(sitemapUrl, { timeout: 5000 });
} else {
throw error; // If other error, throw it
}
}
if (!sitemapUrl) {
return {
statusCode: 404,
body: JSON.stringify({ error: 'Sitemap not found in robots.txt' }),
};
}
// Fetch sitemap
const sitemapRes = await axios.get(sitemapUrl);
const sitemap = await xml2js.parseStringPromise(sitemapRes.data);
const parser = new xml2js.Parser();
const sitemap = await parser.parseStringPromise(sitemapRes.data);
return {
statusCode: 200,
body: JSON.stringify(sitemap),
};
} catch (error) {
return {
statusCode: 500,
body: JSON.stringify({ error: error.message }),
};
// If error occurs
console.log(error.message);
if (error.code === 'ECONNABORTED') {
return {
statusCode: 500,
body: JSON.stringify({ error: 'Request timed out' }),
};
} else {
return {
statusCode: 500,
body: JSON.stringify({ error: error.message }),
};
}
}
};