web-check/api/read-robots-txt.js

46 lines
1.0 KiB
JavaScript

const axios = require('axios');
exports.handler = async function(event, context) {
const siteURL = (event.queryStringParameters || event.query).url;
if (!siteURL) {
return {
statusCode: 400,
body: JSON.stringify({ error: 'Missing url query parameter' }),
};
}
let parsedURL;
try {
parsedURL = new URL(siteURL);
} catch (error) {
return {
statusCode: 400,
body: JSON.stringify({ error: 'Invalid url query parameter' }),
};
}
const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`;
try {
const response = await axios.get(robotsURL);
if (response.status === 200) {
return {
statusCode: 200,
body: response.data,
};
} else {
return {
statusCode: response.status,
body: JSON.stringify({ error: 'Failed to fetch robots.txt', statusCode: response.status }),
};
}
} catch (error) {
return {
statusCode: 500,
body: JSON.stringify({ error: `Error fetching robots.txt: ${error.message}` }),
};
}
};