extern/web-check
1
0
Fork 0
mirror of https://github.com/Lissy93/web-check.git synced 2025-06-18 10:56:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

Skip check if robots.txt not present

Browse Source
This commit is contained in:
Alicia Sykes 2023-08-18 19:27:38 +01:00
parent 519d2f0f79
commit dbcbd36874
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/robots-txt.js
View File

@ -48,7 +48,11 @@ const handler = async function(url) {
const response = await axios.get(robotsURL); const response = await axios.get(robotsURL);
if (response.status === 200) { if (response.status === 200) {
return parseRobotsTxt(response.data); const parsedData = parseRobotsTxt(response.data);
if (!parsedData.robots || parsedData.robots.length === 0) {
return { skipped: 'No robots.txt file present, unable to continue' };
}
return parsedData;
} else { } else {
return { return {
statusCode: response.status, statusCode: response.status,