From 8e1bc7a97e088a18d30ae7eff4b3a189456204e0 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sat, 22 Jul 2023 17:53:33 +0100 Subject: [PATCH] Moves API handlers from server/lambda to /api --- {server/lambda => api}/check-hsts.js | 0 {server/lambda => api}/check-ports.js | 0 {server/lambda => api}/dns-sec.js | 0 {server/lambda => api}/dns-server.js | 0 {server/lambda => api}/find-url-ip.js | 0 {server/lambda => api}/follow-redirects.js | 0 {server/lambda => api}/get-carbon.js | 0 {server/lambda => api}/get-cookies.js | 0 {server/lambda => api}/get-dns.js | 0 {server/lambda => api}/get-headers.js | 0 {server/lambda => api}/get-txt.js | 0 {server/lambda => api}/lighthouse-report.js | 0 {server/lambda => api}/read-robots-txt.js | 0 {server/lambda => api}/screenshot.js | 0 {server/lambda => api}/server-status.js | 0 {server/lambda => api}/site-features.js | 0 {server/lambda => api}/sitemap.js | 0 {server/lambda => api}/ssl-check.js | 0 {server/lambda => api}/tech-stack.js | 0 {server/lambda => api}/trace-route.js | 0 {server/lambda => api}/whois-lookup.js | 0 netlify.toml | 145 +++----------------- package.json | 30 ++-- src/pages/Results.tsx | 51 +++---- 24 files changed, 64 insertions(+), 162 deletions(-) rename {server/lambda => api}/check-hsts.js (100%) rename {server/lambda => api}/check-ports.js (100%) rename {server/lambda => api}/dns-sec.js (100%) rename {server/lambda => api}/dns-server.js (100%) rename {server/lambda => api}/find-url-ip.js (100%) rename {server/lambda => api}/follow-redirects.js (100%) rename {server/lambda => api}/get-carbon.js (100%) rename {server/lambda => api}/get-cookies.js (100%) rename {server/lambda => api}/get-dns.js (100%) rename {server/lambda => api}/get-headers.js (100%) rename {server/lambda => api}/get-txt.js (100%) rename {server/lambda => api}/lighthouse-report.js (100%) rename {server/lambda => api}/read-robots-txt.js (100%) rename {server/lambda => api}/screenshot.js (100%) rename {server/lambda => api}/server-status.js (100%) rename {server/lambda => api}/site-features.js (100%) rename {server/lambda => api}/sitemap.js (100%) rename {server/lambda => api}/ssl-check.js (100%) rename {server/lambda => api}/tech-stack.js (100%) rename {server/lambda => api}/trace-route.js (100%) rename {server/lambda => api}/whois-lookup.js (100%) diff --git a/server/lambda/check-hsts.js b/api/check-hsts.js similarity index 100% rename from server/lambda/check-hsts.js rename to api/check-hsts.js diff --git a/server/lambda/check-ports.js b/api/check-ports.js similarity index 100% rename from server/lambda/check-ports.js rename to api/check-ports.js diff --git a/server/lambda/dns-sec.js b/api/dns-sec.js similarity index 100% rename from server/lambda/dns-sec.js rename to api/dns-sec.js diff --git a/server/lambda/dns-server.js b/api/dns-server.js similarity index 100% rename from server/lambda/dns-server.js rename to api/dns-server.js diff --git a/server/lambda/find-url-ip.js b/api/find-url-ip.js similarity index 100% rename from server/lambda/find-url-ip.js rename to api/find-url-ip.js diff --git a/server/lambda/follow-redirects.js b/api/follow-redirects.js similarity index 100% rename from server/lambda/follow-redirects.js rename to api/follow-redirects.js diff --git a/server/lambda/get-carbon.js b/api/get-carbon.js similarity index 100% rename from server/lambda/get-carbon.js rename to api/get-carbon.js diff --git a/server/lambda/get-cookies.js b/api/get-cookies.js similarity index 100% rename from server/lambda/get-cookies.js rename to api/get-cookies.js diff --git a/server/lambda/get-dns.js b/api/get-dns.js similarity index 100% rename from server/lambda/get-dns.js rename to api/get-dns.js diff --git a/server/lambda/get-headers.js b/api/get-headers.js similarity index 100% rename from server/lambda/get-headers.js rename to api/get-headers.js diff --git a/server/lambda/get-txt.js b/api/get-txt.js similarity index 100% rename from server/lambda/get-txt.js rename to api/get-txt.js diff --git a/server/lambda/lighthouse-report.js b/api/lighthouse-report.js similarity index 100% rename from server/lambda/lighthouse-report.js rename to api/lighthouse-report.js diff --git a/server/lambda/read-robots-txt.js b/api/read-robots-txt.js similarity index 100% rename from server/lambda/read-robots-txt.js rename to api/read-robots-txt.js diff --git a/server/lambda/screenshot.js b/api/screenshot.js similarity index 100% rename from server/lambda/screenshot.js rename to api/screenshot.js diff --git a/server/lambda/server-status.js b/api/server-status.js similarity index 100% rename from server/lambda/server-status.js rename to api/server-status.js diff --git a/server/lambda/site-features.js b/api/site-features.js similarity index 100% rename from server/lambda/site-features.js rename to api/site-features.js diff --git a/server/lambda/sitemap.js b/api/sitemap.js similarity index 100% rename from server/lambda/sitemap.js rename to api/sitemap.js diff --git a/server/lambda/ssl-check.js b/api/ssl-check.js similarity index 100% rename from server/lambda/ssl-check.js rename to api/ssl-check.js diff --git a/server/lambda/tech-stack.js b/api/tech-stack.js similarity index 100% rename from server/lambda/tech-stack.js rename to api/tech-stack.js diff --git a/server/lambda/trace-route.js b/api/trace-route.js similarity index 100% rename from server/lambda/trace-route.js rename to api/trace-route.js diff --git a/server/lambda/whois-lookup.js b/api/whois-lookup.js similarity index 100% rename from server/lambda/whois-lookup.js rename to api/whois-lookup.js diff --git a/netlify.toml b/netlify.toml index da4cad1..6ff0bce 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1,152 +1,39 @@ -# Essential site config +# Build settings and site core config [build] base = "/" command = "yarn build" publish = "build" - functions = "server/lambda" + functions = "api" # Environmental variables and optioanl secrets -# [build.environment] -# NODE_VERSION = "16.16.0" -# GOOGLE_CLOUD_API_KEY='' -# SHODAN_API_KEY='' -# REACT_APP_SHODAN_API_KEY='' -# WHO_API_KEY='' -# REACT_APP_WHO_API_KEY='' -# SECURITY_TRAILS_API_KEY='' -# BUILT_WITH_API_KEY='' -# CI=false +[build.environment] + # Build configuration env vars (uncomment if you want to conigure these) + CI=false # Set CI to false, to prevent warnings from exiting the build + # CHROME_PATH='/usr/bin/chromium' # Path to Chromium binary + # NODE_VERSION = "16.16.0" # Set the version of Node.js to use -# Site info, used for the 1-Click deploy page -[template.environment] - STATUSKIT_PAGE_TITLE = "Web Check" - STATUSKIT_COMPANY_LOGO = "https://raw.githubusercontent.com/Lissy93/web-check/master/public/apple-touch-icon.png" - STATUSKIT_SUPPORT_CONTACT_LINK = "https://github.com/lissy93/web-check" + # Optional secrets and API keys (uncomment if you want to add these) + # GOOGLE_CLOUD_API_KEY='' # Google Cloud API key, for running Lighthouse scans + # BUILT_WITH_API_KEY='' # BuiltWith API key, for detecting site features + # REACT_APP_SHODAN_API_KEY='' # Shodan API key, for using Shodan scan API + # REACT_APP_WHO_API_KEY='' # WhoAPI key, for iniiating client-side whois lookup -# Redirect the Node endpoints to serverless functions +# Redirect the /api/* path to the lambda functions [[redirects]] - from = "/find-url-ip" - to = "/.netlify/functions/find-url-ip" - status = 301 - force = true -[[redirects]] - from = "/lighthouse-report" - to = "/.netlify/functions/lighthouse-report" - status = 301 - force = true -[[redirects]] - from = "/ssl-check" - to = "/.netlify/functions/ssl-check" - status = 301 - force = true -[[redirects]] - from = "/get-headers" - to = "/.netlify/functions/get-headers" - status = 301 - force = true -[[redirects]] - from = "/get-cookies" - to = "/.netlify/functions/get-cookies" - status = 301 - force = true -[[redirects]] - from = "/get-dns" - to = "/.netlify/functions/get-dns" - status = 301 - force = true -[[redirects]] - from = "/read-robots-txt" - to = "/.netlify/functions/read-robots-txt" - status = 301 - force = true -[[redirects]] - from = "/follow-redirects" - to = "/.netlify/functions/follow-redirects" - status = 301 - force = true -[[redirects]] - from = "/get-txt" - to = "/.netlify/functions/get-txt" - status = 301 - force = true -[[redirects]] - from = "/server-status" - to = "/.netlify/functions/server-status" - status = 301 - force = true -[[redirects]] - from = "/check-ports" - to = "/.netlify/functions/check-ports" - status = 301 - force = true -[[redirects]] - from = "/trace-route" - to = "/.netlify/functions/trace-route" - status = 301 - force = true -[[redirects]] - from = "/get-carbon" - to = "/.netlify/functions/get-carbon" - status = 301 - force = true -[[redirects]] - from = "/site-features" - to = "/.netlify/functions/site-features" - status = 301 - force = true -[[redirects]] - from = "/dns-sec" - to = "/.netlify/functions/dns-sec" - status = 301 - force = true -[[redirects]] - from = "/check-hsts" - to = "/.netlify/functions/check-hsts" - status = 301 - force = true -[[redirects]] - from = "/whois-lookup" - to = "/.netlify/functions/whois-lookup" - status = 301 - force = true -[[redirects]] - from = "/dns-server" - to = "/.netlify/functions/dns-server" - status = 301 - force = true -[[redirects]] - from = "/tech-stack" - to = "/.netlify/functions/tech-stack" - status = 301 - force = true -[[redirects]] - from = "/sitemap" - to = "/.netlify/functions/sitemap" - status = 301 - force = true -[[redirects]] - from = "/screenshot" - to = "/.netlify/functions/screenshot" + from = "/api/*" + to = "/.netlify/functions/:splat" status = 301 force = true +# Plugins [[plugins]] package = "netlify-plugin-chromium" [plugins.inputs] packageManager = "yarn" -# For router history mode, ensure pages land on index -[[redirects]] - from = "/*" - to = "/index.html" - status = 200 - # Set any security headers here [[headers]] for = "/*" [headers.values] # Uncomment to enable Netlify user control. Requires premium plan. # Basic-Auth = "someuser:somepassword anotheruser:anotherpassword" - - - \ No newline at end of file diff --git a/package.json b/package.json index 3cfce36..2d36584 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,26 @@ { "name": "web-check", - "version": "0.1.0", - "private": true, + "version": "1.0.0", + "private": false, + "description": "All-in-one OSINT tool for analyzing any website", + "repository": "github:lissy93/web-check", + "homepage": "https://web-check.as93.net", + "license": "MIT", + "author": { + "name": "Alicia Sykes", + "email": "alicia@omg.lol" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Lissy93" + }, + "scripts": { + "dev": "netlify dev", + "start": "react-scripts start", + "build": "react-scripts build", + "test": "react-scripts test", + "eject": "react-scripts eject" + }, "dependencies": { "@netlify/functions": "^1.6.0", "@testing-library/jest-dom": "^5.17.0", @@ -38,13 +57,6 @@ "web-vitals": "^3.4.0", "xml2js": "^0.6.0" }, - "scripts": { - "dev": "netlify dev", - "start": "react-scripts start", - "build": "react-scripts build", - "test": "react-scripts test", - "eject": "react-scripts eject" - }, "eslintConfig": { "extends": [ "react-app", diff --git a/src/pages/Results.tsx b/src/pages/Results.tsx index 4ded93f..20ba495 100644 --- a/src/pages/Results.tsx +++ b/src/pages/Results.tsx @@ -155,9 +155,10 @@ const Results = (): JSX.Element => { response.json() .then(data => resolve(data)) .catch(error => resolve( - { error: `Failed to get a valid response 😢. - This is likely due the target not exposing the required data, or limitations in how Netlify executes lambda functions, such as the 10-sec timeout. - Error info: ${error}`} + { error: `Failed to get a valid response 😢\n` + + `This is likely due the target not exposing the required data, ` + + `or limitations in how Netlify executes lambda functions, such as the 10-sec timeout.\n\n` + + `Error info:\n${error}`} )); }); }; @@ -173,12 +174,14 @@ const Results = (): JSX.Element => { const urlTypeOnly = ['url'] as AddressType[]; // Many jobs only run with these address types + const api = '/api'; + // Fetch and parse IP address for given URL const [ipAddress, setIpAddress] = useMotherHook({ jobId: 'get-ip', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/find-url-ip?url=${address}`) + fetchRequest: () => fetch(`${api}/find-url-ip?url=${address}`) .then(res => parseJson(res)) .then(res => res.ip), }); @@ -188,7 +191,7 @@ const Results = (): JSX.Element => { jobId: 'ssl', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/ssl-check?url=${address}`).then((res) => parseJson(res)), + fetchRequest: () => fetch(`${api}/ssl-check?url=${address}`).then((res) => parseJson(res)), }); // Fetch and parse cookies info @@ -196,7 +199,7 @@ const Results = (): JSX.Element => { jobId: 'cookies', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/get-cookies?url=${address}`) + fetchRequest: () => fetch(`${api}/get-cookies?url=${address}`) .then(res => parseJson(res)) .then(res => parseCookies(res.cookies)), }); @@ -206,7 +209,7 @@ const Results = (): JSX.Element => { jobId: 'robots-txt', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/read-robots-txt?url=${address}`) + fetchRequest: () => fetch(`${api}/read-robots-txt?url=${address}`) .then(res => res.text()) .then(res => parseRobotsTxt(res)), }); @@ -216,7 +219,7 @@ const Results = (): JSX.Element => { jobId: 'headers', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/get-headers?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/get-headers?url=${address}`).then(res => parseJson(res)), }); // Fetch and parse DNS records @@ -224,7 +227,7 @@ const Results = (): JSX.Element => { jobId: 'dns', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/get-dns?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/get-dns?url=${address}`).then(res => parseJson(res)), }); // Fetch and parse Lighthouse performance data @@ -232,7 +235,7 @@ const Results = (): JSX.Element => { jobId: 'quality', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/lighthouse-report?url=${address}`) + fetchRequest: () => fetch(`${api}/lighthouse-report?url=${address}`) .then(res => parseJson(res)) .then(res => res?.lighthouseResult || { error: 'No Data'}), }); @@ -262,7 +265,7 @@ const Results = (): JSX.Element => { jobId: 'ports', updateLoadingJobs, addressInfo: { address: ipAddress, addressType: 'ipV4', expectedAddressTypes: ['ipV4', 'ipV6'] }, - fetchRequest: () => fetch(`/check-ports?url=${ipAddress}`) + fetchRequest: () => fetch(`${api}/check-ports?url=${ipAddress}`) .then(res => parseJson(res)), }); @@ -281,7 +284,7 @@ const Results = (): JSX.Element => { jobId: 'txt-records', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/get-txt?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/get-txt?url=${address}`).then(res => parseJson(res)), }); // Fetches URL redirects @@ -289,7 +292,7 @@ const Results = (): JSX.Element => { jobId: 'redirects', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/follow-redirects?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/follow-redirects?url=${address}`).then(res => parseJson(res)), }); // Get current status and response time of server @@ -297,7 +300,7 @@ const Results = (): JSX.Element => { jobId: 'status', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/server-status?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/server-status?url=${address}`).then(res => parseJson(res)), }); // Get current status and response time of server @@ -305,7 +308,7 @@ const Results = (): JSX.Element => { jobId: 'tech-stack', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/tech-stack?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/tech-stack?url=${address}`).then(res => parseJson(res)), }); // Get trace route for a given hostname @@ -313,7 +316,7 @@ const Results = (): JSX.Element => { jobId: 'trace-route', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/trace-route?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/trace-route?url=${address}`).then(res => parseJson(res)), }); // Fetch carbon footprint data for a given site @@ -321,7 +324,7 @@ const Results = (): JSX.Element => { jobId: 'carbon', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/get-carbon?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/get-carbon?url=${address}`).then(res => parseJson(res)), }); // Check if a site is on the HSTS preload list @@ -329,7 +332,7 @@ const Results = (): JSX.Element => { jobId: 'hsts', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/check-hsts?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/check-hsts?url=${address}`).then(res => parseJson(res)), }); // Get a websites listed pages, from sitemap @@ -337,7 +340,7 @@ const Results = (): JSX.Element => { jobId: 'sitemap', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/sitemap?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/sitemap?url=${address}`).then(res => parseJson(res)), }); // Get a websites listed pages, from sitemap @@ -345,7 +348,7 @@ const Results = (): JSX.Element => { jobId: 'screenshot', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/screenshot?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/screenshot?url=${address}`).then(res => parseJson(res)), }); // Get site features from BuiltWith @@ -353,7 +356,7 @@ const Results = (): JSX.Element => { jobId: 'features', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/site-features?url=${address}`) + fetchRequest: () => fetch(`${api}/site-features?url=${address}`) .then(res => parseJson(res)) .then(res => { if (res.Errors && res.Errors.length > 0) { @@ -368,7 +371,7 @@ const Results = (): JSX.Element => { jobId: 'dnssec', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/dns-sec?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/dns-sec?url=${address}`).then(res => parseJson(res)), }); // Run a manual whois lookup on the domain @@ -376,7 +379,7 @@ const Results = (): JSX.Element => { jobId: 'domain', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/whois-lookup?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/whois-lookup?url=${address}`).then(res => parseJson(res)), }); // Get the DNS server(s) for a domain, and test DoH/DoT support @@ -384,7 +387,7 @@ const Results = (): JSX.Element => { jobId: 'dns-server', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`/dns-server?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/dns-server?url=${address}`).then(res => parseJson(res)), }); /* Cancel remaining jobs after 10 second timeout */