From 9e426ed55e5e18de15b6272567ae0705623d57f7 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Thu, 10 Aug 2023 19:30:21 +0100 Subject: [PATCH] Rename endpoints to be the same as job IDs --- api/_common/middleware.js | 1 - api/{get-carbon.js => carbon.js} | 0 api/{get-cookies.js => cookies.js} | 0 api/{get-dns.js => dns.js} | 0 api/{dns-sec.js => dnssec.js} | 0 api/{site-features.js => features.js} | 0 api/{find-url-ip.js => get-ip.js} | 0 api/{get-headers.js => headers.js} | 0 api/{check-hsts.js => hsts.js} | 0 api/{content-links.js => linked-pages.js} | 0 api/{check-ports.js => ports.js} | 0 api/{lighthouse-report.js => quality.js} | 0 api/read-robots-txt.js | 45 ---------------- api/{follow-redirects.js => redirects.js} | 0 api/robots-txt.js | 66 +++++++++++++++++++++++ api/{ssl-check.js => ssl.js} | 0 api/{server-status.js => status.js} | 0 api/{get-txt.js => txt-records.js} | 0 api/{whois-lookup.js => whois.js} | 0 src/pages/Results.tsx | 37 +++++++------ 20 files changed, 84 insertions(+), 65 deletions(-) rename api/{get-carbon.js => carbon.js} (100%) rename api/{get-cookies.js => cookies.js} (100%) rename api/{get-dns.js => dns.js} (100%) rename api/{dns-sec.js => dnssec.js} (100%) rename api/{site-features.js => features.js} (100%) rename api/{find-url-ip.js => get-ip.js} (100%) rename api/{get-headers.js => headers.js} (100%) rename api/{check-hsts.js => hsts.js} (100%) rename api/{content-links.js => linked-pages.js} (100%) rename api/{check-ports.js => ports.js} (100%) rename api/{lighthouse-report.js => quality.js} (100%) delete mode 100644 api/read-robots-txt.js rename api/{follow-redirects.js => redirects.js} (100%) create mode 100644 api/robots-txt.js rename api/{ssl-check.js => ssl.js} (100%) rename api/{server-status.js => status.js} (100%) rename api/{get-txt.js => txt-records.js} (100%) rename api/{whois-lookup.js => whois.js} (100%) diff --git a/api/_common/middleware.js b/api/_common/middleware.js index 434c498..7c9c53e 100644 --- a/api/_common/middleware.js +++ b/api/_common/middleware.js @@ -27,7 +27,6 @@ const commonMiddleware = (handler) => { }); } } catch (error) { - console.log(error); callback(null, { statusCode: 500, body: JSON.stringify({ error: error.message }), diff --git a/api/get-carbon.js b/api/carbon.js similarity index 100% rename from api/get-carbon.js rename to api/carbon.js diff --git a/api/get-cookies.js b/api/cookies.js similarity index 100% rename from api/get-cookies.js rename to api/cookies.js diff --git a/api/get-dns.js b/api/dns.js similarity index 100% rename from api/get-dns.js rename to api/dns.js diff --git a/api/dns-sec.js b/api/dnssec.js similarity index 100% rename from api/dns-sec.js rename to api/dnssec.js diff --git a/api/site-features.js b/api/features.js similarity index 100% rename from api/site-features.js rename to api/features.js diff --git a/api/find-url-ip.js b/api/get-ip.js similarity index 100% rename from api/find-url-ip.js rename to api/get-ip.js diff --git a/api/get-headers.js b/api/headers.js similarity index 100% rename from api/get-headers.js rename to api/headers.js diff --git a/api/check-hsts.js b/api/hsts.js similarity index 100% rename from api/check-hsts.js rename to api/hsts.js diff --git a/api/content-links.js b/api/linked-pages.js similarity index 100% rename from api/content-links.js rename to api/linked-pages.js diff --git a/api/check-ports.js b/api/ports.js similarity index 100% rename from api/check-ports.js rename to api/ports.js diff --git a/api/lighthouse-report.js b/api/quality.js similarity index 100% rename from api/lighthouse-report.js rename to api/quality.js diff --git a/api/read-robots-txt.js b/api/read-robots-txt.js deleted file mode 100644 index 50b7449..0000000 --- a/api/read-robots-txt.js +++ /dev/null @@ -1,45 +0,0 @@ -const axios = require('axios'); - -exports.handler = async function(event, context) { - const siteURL = (event.queryStringParameters || event.query).url; - - if (!siteURL) { - return { - statusCode: 400, - body: JSON.stringify({ error: 'Missing url query parameter' }), - }; - } - - let parsedURL; - try { - parsedURL = new URL(siteURL); - } catch (error) { - return { - statusCode: 400, - body: JSON.stringify({ error: 'Invalid url query parameter' }), - }; - } - - const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`; - - try { - const response = await axios.get(robotsURL); - - if (response.status === 200) { - return { - statusCode: 200, - body: response.data, - }; - } else { - return { - statusCode: response.status, - body: JSON.stringify({ error: 'Failed to fetch robots.txt', statusCode: response.status }), - }; - } - } catch (error) { - return { - statusCode: 500, - body: JSON.stringify({ error: `Error fetching robots.txt: ${error.message}` }), - }; - } -}; diff --git a/api/follow-redirects.js b/api/redirects.js similarity index 100% rename from api/follow-redirects.js rename to api/redirects.js diff --git a/api/robots-txt.js b/api/robots-txt.js new file mode 100644 index 0000000..c5007ba --- /dev/null +++ b/api/robots-txt.js @@ -0,0 +1,66 @@ +const axios = require('axios'); +const middleware = require('./_common/middleware'); + +const parseRobotsTxt = (content) => { + const lines = content.split('\n'); + const rules = []; + + lines.forEach(line => { + line = line.trim(); // This removes trailing and leading whitespaces + + let match = line.match(/^(Allow|Disallow):\s*(\S*)$/i); + if (match) { + const rule = { + lbl: match[1], + val: match[2], + }; + + rules.push(rule); + } else { + match = line.match(/^(User-agent):\s*(\S*)$/i); + if (match) { + const rule = { + lbl: match[1], + val: match[2], + }; + + rules.push(rule); + } + } + }); + return { robots: rules }; +} + +const handler = async function(url) { + let parsedURL; + try { + parsedURL = new URL(url); + } catch (error) { + return { + statusCode: 400, + body: JSON.stringify({ error: 'Invalid url query parameter' }), + }; + } + + const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`; + + try { + const response = await axios.get(robotsURL); + + if (response.status === 200) { + return parseRobotsTxt(response.data); + } else { + return { + statusCode: response.status, + body: JSON.stringify({ error: 'Failed to fetch robots.txt', statusCode: response.status }), + }; + } + } catch (error) { + return { + statusCode: 500, + body: JSON.stringify({ error: `Error fetching robots.txt: ${error.message}` }), + }; + } +}; + +exports.handler = middleware(handler); diff --git a/api/ssl-check.js b/api/ssl.js similarity index 100% rename from api/ssl-check.js rename to api/ssl.js diff --git a/api/server-status.js b/api/status.js similarity index 100% rename from api/server-status.js rename to api/status.js diff --git a/api/get-txt.js b/api/txt-records.js similarity index 100% rename from api/get-txt.js rename to api/txt-records.js diff --git a/api/whois-lookup.js b/api/whois.js similarity index 100% rename from api/whois-lookup.js rename to api/whois.js diff --git a/src/pages/Results.tsx b/src/pages/Results.tsx index 5042b91..f025289 100644 --- a/src/pages/Results.tsx +++ b/src/pages/Results.tsx @@ -165,7 +165,7 @@ const Results = (): JSX.Element => { jobId: 'get-ip', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/find-url-ip?url=${address}`) + fetchRequest: () => fetch(`${api}/get-ip?url=${address}`) .then(res => parseJson(res)) .then(res => res.ip), }); @@ -175,7 +175,7 @@ const Results = (): JSX.Element => { jobId: 'ssl', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/ssl-check?url=${address}`).then((res) => parseJson(res)), + fetchRequest: () => fetch(`${api}/ssl?url=${address}`).then((res) => parseJson(res)), }); // Fetch and parse cookies info @@ -183,7 +183,7 @@ const Results = (): JSX.Element => { jobId: 'cookies', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/get-cookies?url=${address}`) + fetchRequest: () => fetch(`${api}/cookies?url=${address}`) .then(res => parseJson(res)) .then(res => parseCookies(res.cookies)), }); @@ -193,9 +193,8 @@ const Results = (): JSX.Element => { jobId: 'robots-txt', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/read-robots-txt?url=${address}`) - .then(res => res.text()) - .then(res => parseRobotsTxt(res)), + fetchRequest: () => fetch(`${api}/robots-txt?url=${address}`) + .then(res => parseJson(res)), }); // Fetch and parse headers @@ -203,7 +202,7 @@ const Results = (): JSX.Element => { jobId: 'headers', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/get-headers?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/headers?url=${address}`).then(res => parseJson(res)), }); // Fetch and parse DNS records @@ -211,7 +210,7 @@ const Results = (): JSX.Element => { jobId: 'dns', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/get-dns?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/dns?url=${address}`).then(res => parseJson(res)), }); // Fetch and parse Lighthouse performance data @@ -219,7 +218,7 @@ const Results = (): JSX.Element => { jobId: 'quality', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/lighthouse-report?url=${address}`) + fetchRequest: () => fetch(`${api}/quality?url=${address}`) .then(res => parseJson(res)) .then(res => res?.lighthouseResult || { error: 'No Data'}), }); @@ -249,7 +248,7 @@ const Results = (): JSX.Element => { jobId: 'ports', updateLoadingJobs, addressInfo: { address: ipAddress, addressType: 'ipV4', expectedAddressTypes: ['ipV4', 'ipV6'] }, - fetchRequest: () => fetch(`${api}/check-ports?url=${ipAddress}`) + fetchRequest: () => fetch(`${api}/ports?url=${ipAddress}`) .then(res => parseJson(res)), }); @@ -268,7 +267,7 @@ const Results = (): JSX.Element => { jobId: 'txt-records', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/get-txt?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/txt-records?url=${address}`).then(res => parseJson(res)), }); // Fetches URL redirects @@ -276,7 +275,7 @@ const Results = (): JSX.Element => { jobId: 'redirects', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/follow-redirects?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/redirects?url=${address}`).then(res => parseJson(res)), }); // Get current status and response time of server @@ -284,7 +283,7 @@ const Results = (): JSX.Element => { jobId: 'status', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/server-status?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/status?url=${address}`).then(res => parseJson(res)), }); // Get current status and response time of server @@ -308,7 +307,7 @@ const Results = (): JSX.Element => { jobId: 'carbon', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/get-carbon?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/carbon?url=${address}`).then(res => parseJson(res)), }); // Check if a site is on the HSTS preload list @@ -316,7 +315,7 @@ const Results = (): JSX.Element => { jobId: 'hsts', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/check-hsts?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/hsts?url=${address}`).then(res => parseJson(res)), }); // Get a websites listed pages, from sitemap @@ -356,7 +355,7 @@ const Results = (): JSX.Element => { jobId: 'features', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/site-features?url=${address}`) + fetchRequest: () => fetch(`${api}/features?url=${address}`) .then(res => parseJson(res)) .then(res => { if (res.Errors && res.Errors.length > 0) { @@ -371,7 +370,7 @@ const Results = (): JSX.Element => { jobId: 'dnssec', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/dns-sec?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/dnssec?url=${address}`).then(res => parseJson(res)), }); // Run a manual whois lookup on the domain @@ -379,7 +378,7 @@ const Results = (): JSX.Element => { jobId: 'domain', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/whois-lookup?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/whois?url=${address}`).then(res => parseJson(res)), }); // Get the DNS server(s) for a domain, and test DoH/DoT support @@ -395,7 +394,7 @@ const Results = (): JSX.Element => { jobId: 'linked-pages', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/content-links?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/linked-pages?url=${address}`).then(res => parseJson(res)), }); // Get mail config for server, based on DNS records