Removal of node-fetch, updates all functions accordingly, use of custom chromium driver for netlify support, plus update of all older packages

netlify.toml

@@ -24,11 +24,6 @@
   STATUSKIT_SUPPORT_CONTACT_LINK = "https://github.com/lissy93/web-check"
 
 # Redirect the Node endpoints to serverless functions
-[[redirects]]
-  from = "/api"
-  to = "/.netlify/functions/web-check"
-  status = 301
-  force = true
 [[redirects]]
   from = "/find-url-ip"
   to = "/.netlify/functions/find-url-ip"
@@ -134,7 +129,12 @@
   to = "/.netlify/functions/screenshot"
   status = 301
   force = true
-  
+
+[[plugins]]
+package = "netlify-plugin-chromium"
+  [plugins.inputs]
+  packageManager = ["yarn"]
+
 # For router history mode, ensure pages land on index
 [[redirects]]
   from = "/*"

package.json

@@ -4,37 +4,37 @@
   "private": true,
   "dependencies": {
     "@netlify/functions": "^1.6.0",
-    "@testing-library/jest-dom": "^5.16.4",
-    "@testing-library/react": "^13.3.0",
-    "@testing-library/user-event": "^13.5.0",
-    "@types/jest": "^27.5.2",
-    "@types/node": "^16.11.39",
-    "@types/react": "^18.0.12",
-    "@types/react-dom": "^18.0.5",
+    "@testing-library/jest-dom": "^5.17.0",
+    "@testing-library/react": "^14.0.0",
+    "@testing-library/user-event": "^14.4.3",
+    "@types/jest": "^29.5.3",
+    "@types/node": "^20.4.4",
+    "@types/react": "^18.2.15",
+    "@types/react-dom": "^18.2.7",
     "@types/react-router-dom": "^5.3.3",
-    "@types/styled-components": "^5.1.25",
+    "@types/styled-components": "^5.1.26",
     "axios": "^1.4.0",
     "chrome-aws-lambda": "^10.1.0",
     "flatted": "^3.2.7",
     "got": "^13.0.0",
-    "jest-styled-components": "^7.0.8",
+    "jest-styled-components": "^7.1.1",
     "netlify-cli": "^15.9.1",
-    "node-fetch": "2.6.1",
+    "netlify-plugin-chromium": "^1.1.4",
     "perf_hooks": "^0.0.1",
     "psl": "^1.9.0",
     "puppeteer-core": "^20.9.0",
-    "react": "^18.1.0",
-    "react-dom": "^18.1.0",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
     "react-masonry-css": "^1.0.16",
-    "react-router-dom": "^6.3.0",
+    "react-router-dom": "^6.14.2",
     "react-scripts": "5.0.1",
-    "react-simple-maps": "^2.3.0",
+    "react-simple-maps": "^3.0.0",
     "react-toastify": "^9.1.3",
-    "styled-components": "^5.3.5",
+    "styled-components": "^6.0.5",
     "traceroute": "^1.0.0",
-    "typescript": "^4.7.3",
+    "typescript": "^5.1.6",
     "wappalyzer": "^6.10.63",
-    "web-vitals": "^2.1.4",
+    "web-vitals": "^3.4.0",
     "xml2js": "^0.6.0"
   },
   "scripts": {
@@ -63,7 +63,7 @@
     ]
   },
   "devDependencies": {
-    "@types/react-simple-maps": "^1.0.8"
+    "@types/react-simple-maps": "^3.0.0"
   },
   "compilerOptions": {
     "allowJs": true,

server/lambda/get-cookies.js

@@ -1,4 +1,4 @@
-const fetch = require('node-fetch');
+const axios = require('axios');
 
 exports.handler = async function(event, context) {
   const { url } = event.queryStringParameters;
@@ -11,9 +11,8 @@ exports.handler = async function(event, context) {
   }
 
   try {
-    const response = await fetch(url);
-    const cookies = response.headers.get('set-cookie');
-
+    const response = await axios.get(url, {withCredentials: true});
+    const cookies = response.headers['set-cookie'];
     return {
       statusCode: 200,
       body: JSON.stringify({ cookies }),

server/lambda/get-headers.js

@@ -1,4 +1,4 @@
-const fetch = require('node-fetch');
+const axios = require('axios');
 
 exports.handler = async function(event, context) {
   const { url } = event.queryStringParameters;
@@ -11,14 +11,18 @@ exports.handler = async function(event, context) {
   }
 
   try {
-    const response = await fetch(url);
-    const headers = response.headers.raw();
+    const response = await axios.get(url, {
+      validateStatus: function (status) {
+        return status >= 200 && status < 600; // Resolve only if the status code is less than 600
+      },
+    });
 
     return {
       statusCode: 200,
-      body: JSON.stringify(headers),
+      body: JSON.stringify(response.headers),
     };
   } catch (error) {
+    console.log(error);
     return {
       statusCode: 500,
       body: JSON.stringify({ error: error.message }),

server/lambda/lighthouse-report.js

@@ -1,4 +1,4 @@
-const fetch = require('node-fetch');
+const axios = require('axios');
 
 exports.handler = function(event, context, callback) {
   const { url } = event.queryStringParameters;
@@ -12,13 +12,13 @@ exports.handler = function(event, context, callback) {
 
   const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
   const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile&key=${apiKey}`;
-  fetch(endpoint)
-  .then((res) => res.json() )
+  
+  axios.get(endpoint)
   .then(
-    (data) => {
+    (response) => {
       callback(null, {
         statusCode: 200,
-        body: JSON.stringify(data),
+        body: JSON.stringify(response.data),
       });
     }
   ).catch(
@@ -30,4 +30,3 @@ exports.handler = function(event, context, callback) {
     }
   );
 };
-

server/lambda/read-robots-txt.js

@@ -1,8 +1,8 @@
-const fetch = require('node-fetch');
+const axios = require('axios');
 
 exports.handler = async function(event, context) {
   const siteURL = event.queryStringParameters.url;
-  
+
   if (!siteURL) {
     return {
       statusCode: 400,
@@ -23,13 +23,12 @@ exports.handler = async function(event, context) {
   const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`;
 
   try {
-    const response = await fetch(robotsURL);
-    const text = await response.text();
+    const response = await axios.get(robotsURL);
 
-    if (response.ok) {
+    if (response.status === 200) {
       return {
         statusCode: 200,
-        body: text,
+        body: response.data,
       };
     } else {
       return {

server/lambda/screenshot.js

@@ -31,9 +31,10 @@ exports.handler = async (event, context, callback) => {
       browser = await puppeteer.launch({
       args: chromium.args,
       defaultViewport: { width: 800, height: 600 },
-      executablePath: process.env.CHROME_EXECUTABLE_PATH || await chromium.executablePath,
+      executablePath: process.env.CHROME_PATH || await chromium.executablePath,
       headless: chromium.headless,
       ignoreHTTPSErrors: true,
+      ignoreDefaultArgs: ['--disable-extensions'],
     });
 
     let page = await browser.newPage();

server/lambda/web-check.js

@@ -1,71 +0,0 @@
-const fetch = require('node-fetch');
-
-
-const endpoints = {
-  dnsRecords: 'https://api.securitytrails.com/v1/domain/{address}',
-  subDomains: 'https://api.securitytrails.com/v1/domain/{address}/subdomains?include_inactive=true',
-};
-
-const apiKeys = {
-  shodan: process.env.SHODAN_API_KEY,
-  securityTrails: process.env.SECURITY_TRAILS_API_KEY,
-};
-
-const makeEndpoint = (endpoint, address) => endpoint.replace('{address}', address);
-
-const errorObject = (msg, status) => {
-  return {
-    statusCode: status || 500,
-    body: JSON.stringify({ error: true, errorMsg: msg }),
-  }
-};
-
-/* Fetches list of DNS records for a given URL from SecurityTrails API */
-const getDnsInfo = async (url) => {
-  const stOptions = {
-    method: 'GET',
-    headers: { Accept: 'application/json', APIKEY: apiKeys.securityTrails }
-  };
-  const dnsInfoRequest = await fetch(makeEndpoint(endpoints.dnsRecords, url), stOptions);
-  const dnsInfoResponse = await dnsInfoRequest.json();
-  if (dnsInfoResponse) return null;
-  return dnsInfoResponse?.current_dns;
-};
-
-/* Fetches a list of a domain's sub-domains from SecurityTrails API */
-const getSubDomains = async (url) => {
-  const stOptions = {
-    method: 'GET',
-    headers: { Accept: 'application/json', APIKEY: apiKeys.securityTrails }
-  };
-  const subDomainRequest = await fetch(makeEndpoint(endpoints.subDomains, url), stOptions);
-  const subDomainResponse = await subDomainRequest.json();
-  if (!subDomainResponse.subdomains) return null;
-  return {
-    count: subDomainResponse?.subdomain_count,
-    list: subDomainResponse?.subdomains,
-  };
-};
-
-exports.handler = async (event, context) => {
-  const addressParam = event.queryStringParameters.address;
-  if (!addressParam) return errorObject('An address must be specified');
-  const address = decodeURIComponent(addressParam)
-    .replaceAll('https://', '')
-    .replaceAll('http://', '');
-
-  try {
-    const results = {};
-    if (apiKeys.securityTrails) {
-      results.dnsRecords = await getDnsInfo(address);
-      results.subDomains = await getSubDomains(address);
-    }
-
-    return {
-      statusCode: 200,
-      body: JSON.stringify(results),
-    };
-  } catch (error) {
-    return errorObject('Failed to fetch data');
-  }
-};

src/utils/result-processor.ts

@@ -140,20 +140,24 @@ export type Cookie = {
   attributes: Record<string, string>;
 };
 
-export const parseCookies = (cookiesHeader: string): {cookies: Cookie[]} => {
-  if (!cookiesHeader) return {cookies: []};
-  const cookies = cookiesHeader.split(/,(?=\s[A-Za-z0-9]+=)/).map(cookieString => {
-    const [nameValuePair, ...attributePairs] = cookieString.split('; ').map(part => part.trim());
-    const [name, value] = nameValuePair.split('=');
-    const attributes: Record<string, string> = {};
-    attributePairs.forEach(pair => {
-      const [attributeName, attributeValue = ''] = pair.split('=');
-      attributes[attributeName] = attributeValue;
+export const parseCookies = (cookiesHeader: string[]): {cookies: Cookie[]} => {
+  if (!cookiesHeader || !cookiesHeader.length) return {cookies: []};
+
+  const cookies = cookiesHeader.flatMap(cookieHeader => {
+    return cookieHeader.split(/,(?=\s[A-Za-z0-9]+=)/).map(cookieString => {
+      const [nameValuePair, ...attributePairs] = cookieString.split('; ').map(part => part.trim());
+      const [name, value] = nameValuePair.split('=');
+      const attributes: Record<string, string> = {};
+      attributePairs.forEach(pair => {
+        const [attributeName, attributeValue = ''] = pair.split('=');
+        attributes[attributeName] = attributeValue;
+      });
+      return { name, value, attributes };
     });
-    return { name, value, attributes };
   });
-  return { cookies }
-}
+
+  return { cookies };
+};
 
 export const parseRobotsTxt = (content: string): { robots: RowProps[] } => {
   const lines = content.split('\n');