From dbcbd36874545129d5bfbf6c3da6411cefaade2f Mon Sep 17 00:00:00 2001
From: Alicia Sykes <alicia@omg.lol>
Date: Fri, 18 Aug 2023 19:27:38 +0100
Subject: [PATCH] Skip check if robots.txt not present

---
 api/robots-txt.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/api/robots-txt.js b/api/robots-txt.js
index c5007ba..c21088f 100644
--- a/api/robots-txt.js
+++ b/api/robots-txt.js
@@ -48,7 +48,11 @@ const handler = async function(url) {
     const response = await axios.get(robotsURL);
 
     if (response.status === 200) {
-      return parseRobotsTxt(response.data);
+      const parsedData = parseRobotsTxt(response.data);
+      if (!parsedData.robots || parsedData.robots.length === 0) {
+        return { skipped: 'No robots.txt file present, unable to continue' };
+      }
+      return parsedData;
     } else {
       return {
         statusCode: response.status,