Merge pull request #31 from Lissy93/FEAT/node-server

[FEAT] New Server Arch

.gitignore

@@ -1,5 +1,5 @@
 
-# Keys
+# keys
 .env
 
 # dependencies
@@ -7,22 +7,21 @@
 /.pnp
 .pnp.js
 
+# logs
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
 # testing
 /coverage
 
 # production
 /build
 
-# misc
+# Random AWS and Netlify crap
-.DS_Store
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
-
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-
-# Local Netlify folder
 .netlify
+.serverless
+.webpack
+
+# OS generated files
+.DS_Store

Dockerfile

@@ -7,6 +7,7 @@ RUN apt-get update && \
     chmod 755 /usr/bin/chromium && \
     rm -rf /var/lib/apt/lists/*
 RUN npm install --force
-EXPOSE 8888
+RUN npm run build
+EXPOSE ${PORT:-3000}
 ENV CHROME_PATH='/usr/bin/chromium'
-CMD ["npm", "run", "serve"]
+CMD ["npm", "start"]

api/_common/aws-webpack.config.js

@@ -0,0 +1,51 @@
+const path = require('path');
+const nodeExternals = require('webpack-node-externals');
+
+module.exports = {
+    target: 'node',
+    mode: 'production',
+    entry: {
+        'carbon': './api/carbon.js',
+        'cookies': './api/cookies.js',
+        'dns-server': './api/dns-server.js',
+        'dns': './api/dns.js',
+        'dnssec': './api/dnssec.js',
+        'features': './api/features.js',
+        'get-ip': './api/get-ip.js',
+        'headers': './api/headers.js',
+        'hsts': './api/hsts.js',
+        'linked-pages': './api/linked-pages.js',
+        'mail-config': './api/mail-config.js',
+        'ports': './api/ports.js',
+        'quality': './api/quality.js',
+        'redirects': './api/redirects.js',
+        'robots-txt': './api/robots-txt.js',
+        'screenshot': './api/screenshot.js',
+        'security-txt': './api/security-txt.js',
+        'sitemap': './api/sitemap.js',
+        'social-tags': './api/social-tags.js',
+        'ssl': './api/ssl.js',
+        'status': './api/status.js',
+        'tech-stack': './api/tech-stack.js',
+        'trace-route': './api/trace-route.js',
+        'txt-records': './api/txt-records.js',
+        'whois': './api/whois.js',
+    },
+    externals: [nodeExternals()],
+    output: {
+        filename: '[name].js',
+        path: path.resolve(__dirname, '.webpack'),
+        libraryTarget: 'commonjs2'
+    },
+    module: {
+        rules: [
+            {
+                test: /\.js$/,
+                use: {
+                    loader: 'babel-loader'
+                },
+                exclude: /node_modules/,
+            }
+        ]
+    }
+};

api/_common/middleware.js

@@ -0,0 +1,38 @@
+const normalizeUrl = (url) => {
+  return url.startsWith('http') ? url : `https://${url}`;
+};
+
+const commonMiddleware = (handler) => {
+  return async (event, context, callback) => {
+    const queryParams = event.queryStringParameters || event.query || {};
+    const rawUrl = queryParams.url;
+    
+    if (!rawUrl) {
+      callback(null, {
+        statusCode: 500,
+        body: JSON.stringify({ error: 'No URL specified' }),
+      });
+    }
+
+    const url = normalizeUrl(rawUrl);
+
+    try {
+      const response = await handler(url, event, context);
+      if (response.body && response.statusCode) {
+      callback(null, response);
+      } else {
+        callback(null, {
+          statusCode: 200,
+          body: typeof response === 'object' ? JSON.stringify(response) : response,
+        });
+      }
+    } catch (error) {
+      callback(null, {
+        statusCode: 500,
+        body: JSON.stringify({ error: error.message }),
+      });
+    }
+  };
+};
+
+module.exports = commonMiddleware;

api/carbon.js

@@ -1,14 +1,7 @@
 const https = require('https');
+const middleware = require('./_common/middleware');
 
-exports.handler = async (event, context) => {
+const handler = async (url) => {
-  const { url } = event.queryStringParameters;
-
-  if (!url) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'url query parameter is required' }),
-    };
-  }
 
   // First, get the size of the website's HTML
   const getHtmlSize = (url) => new Promise((resolve, reject) => {
@@ -49,14 +42,10 @@ exports.handler = async (event, context) => {
     }
 
     carbonData.scanUrl = url;
-    return {
+    return carbonData;
-      statusCode: 200,
-      body: JSON.stringify(carbonData),
-    };
   } catch (error) {
-    return {
+    throw new Error(`Error: ${error.message}`);
-      statusCode: 500,
-      body: JSON.stringify({ error: `Error: ${error.message}` }),
-    };
   }
 };
+
+exports.handler = middleware(handler);

api/content-links.js

@@ -1,60 +0,0 @@
-const axios = require('axios');
-const cheerio = require('cheerio');
-const urlLib = require('url');
-
-exports.handler = async (event, context) => {
-  let url = event.queryStringParameters.url;
-  
-  // Check if url includes protocol
-  if (!url.startsWith('http://') && !url.startsWith('https://')) {
-    url = 'http://' + url;
-  }
-  
-  try {
-    const response = await axios.get(url);
-    const html = response.data;
-    const $ = cheerio.load(html);
-    const internalLinksMap = new Map();
-    const externalLinksMap = new Map();
-
-    $('a[href]').each((i, link) => {
-      const href = $(link).attr('href');
-      const absoluteUrl = urlLib.resolve(url, href);
-      
-      if (absoluteUrl.startsWith(url)) {
-        const count = internalLinksMap.get(absoluteUrl) || 0;
-        internalLinksMap.set(absoluteUrl, count + 1);
-      } else if (href.startsWith('http://') || href.startsWith('https://')) {
-        const count = externalLinksMap.get(absoluteUrl) || 0;
-        externalLinksMap.set(absoluteUrl, count + 1);
-      }
-    });
-
-    // Convert maps to sorted arrays
-    const internalLinks = [...internalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
-    const externalLinks = [...externalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
-
-    if (internalLinks.length === 0 && externalLinks.length === 0) {
-      return {
-        statusCode: 400,
-        body: JSON.stringify({
-          skipped: 'No internal or external links found. '
-            + 'This may be due to the website being dynamically rendered, using a client-side framework (like React), and without SSR enabled. '
-            + 'That would mean that the static HTML returned from the HTTP request doesn\'t contain any meaningful content for Web-Check to analyze. '
-            + 'You can rectify this by using a headless browser to render the page instead.',
-          }),
-      };
-    }
-
-    return {
-      statusCode: 200,
-      body: JSON.stringify({ internal: internalLinks, external: externalLinks }),
-    };
-  } catch (error) {
-    console.log(error);
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: 'Failed fetching data' }),
-    };
-  }
-};

api/cookies.js

@@ -0,0 +1,14 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const handler = async (url, event, context) => {
+  try {
+    const response = await axios.get(url, {withCredentials: true});
+    const cookies = response.headers['set-cookie'];
+    return { cookies };
+  } catch (error) {
+    throw new Error(error.message);
+  }
+};
+
+exports.handler = middleware(handler);

api/dns-server.js

@@ -1,11 +1,11 @@
 const dns = require('dns');
 const dnsPromises = dns.promises;
-// const https = require('https');
 const axios = require('axios');
+const commonMiddleware = require('./_common/middleware');
 
-exports.handler = async (event) => {
+const handler = async (url) => {
-  const domain = event.queryStringParameters.url.replace(/^(?:https?:\/\/)?/i, "");
   try {
+    const domain = url.replace(/^(?:https?:\/\/)?/i, "");
     const addresses = await dnsPromises.resolve4(domain);
     const results = await Promise.all(addresses.map(async (address) => {
       const hostname = await dnsPromises.reverse(address).catch(() => null);
@@ -22,6 +22,7 @@ exports.handler = async (event) => {
         dohDirectSupports,
       };
     }));
+
     // let dohMozillaSupport = false;
     // try {
     //   const mozillaList = await axios.get('https://firefox.settings.services.mozilla.com/v1/buckets/security-state/collections/onecrl/records');
@@ -29,20 +30,15 @@ exports.handler = async (event) => {
     // } catch (error) {
     //   console.error(error);
     // }
+
     return {
-      statusCode: 200,
+      domain,
-      body: JSON.stringify({
+      dns: results,
-        domain,
+      // dohMozillaSupport,
-        dns: results,
-        // dohMozillaSupport,
-      }),
     };
   } catch (error) {
-    return {
+    throw new Error(`An error occurred while resolving DNS. ${error.message}`); // This will be caught and handled by the commonMiddleware
-      statusCode: 500,
-      body: JSON.stringify({
-        error: `An error occurred while resolving DNS. ${error.message}`,
-      }),
-    };
   }
 };
+
+exports.handler = commonMiddleware(handler);

api/dns.js

@@ -1,8 +1,9 @@
 const dns = require('dns');
 const util = require('util');
+const middleware = require('./_common/middleware');
 
-exports.handler = async function(event, context) {
+const handler = async (url) => {
-  let hostname = event.queryStringParameters.url;
+  let hostname = url;
 
   // Handle URLs by extracting hostname
   if (hostname.startsWith('http://') || hostname.startsWith('https://')) {
@@ -35,25 +36,19 @@ exports.handler = async function(event, context) {
     ]);
 
     return {
-      statusCode: 200,
+      A: a,
-      body: JSON.stringify({
+      AAAA: aaaa,
-        A: a,
+      MX: mx,
-        AAAA: aaaa,
+      TXT: txt,
-        MX: mx,
+      NS: ns,
-        TXT: txt,
+      CNAME: cname,
-        NS: ns,
+      SOA: soa,
-        CNAME: cname,
+      SRV: srv,
-        SOA: soa,
+      PTR: ptr
-        SRV: srv,
-        PTR: ptr
-      })
     };
   } catch (error) {
-    return {
+    throw new Error(error.message);
-      statusCode: 500,
-      body: JSON.stringify({
-        error: error.message
-      })
-    };
   }
 };
+
+exports.handler = middleware(handler);

api/dnssec.js

@@ -1,16 +1,7 @@
 const https = require('https');
+const commonMiddleware = require('./_common/middleware'); // Make sure this path is correct
 
-exports.handler = async function(event, context) {
+const fetchDNSRecords = async (domain, event, context) => {
-  let { url } = event.queryStringParameters;
-
-  if (!url) {
-    return errorResponse('URL query parameter is required.');
-  }
-
-  // Extract hostname from URL
-  const parsedUrl = new URL(url);
-  const domain = parsedUrl.hostname;
-
   const dnsTypes = ['DNSKEY', 'DS', 'RRSIG'];
   const records = {};
 
@@ -48,22 +39,14 @@ exports.handler = async function(event, context) {
       if (dnsResponse.Answer) {
         records[type] = { isFound: true, answer: dnsResponse.Answer, response: dnsResponse.Answer };
       } else {
-        records[type] = { isFound: false, answer: null, response: dnsResponse};
+        records[type] = { isFound: false, answer: null, response: dnsResponse };
       }
     } catch (error) {
-      return errorResponse(`Error fetching ${type} record: ${error.message}`);
+      throw new Error(`Error fetching ${type} record: ${error.message}`); // This will be caught and handled by the commonMiddleware
     }
   }
 
-  return {
+  return records;
-    statusCode: 200,
-    body: JSON.stringify(records),
-  };
 };
 
-const errorResponse = (message, statusCode = 444) => {
+exports.handler = commonMiddleware(fetchDNSRecords);
-  return {
-    statusCode: statusCode,
-    body: JSON.stringify({ error: message }),
-  };
-};

api/features.js

@@ -1,22 +1,15 @@
 const https = require('https');
+const middleware = require('./_common/middleware');
 
-exports.handler = async function (event, context) {
+const builtWithHandler = async (url) => {
-  const { url } = event.queryStringParameters;
   const apiKey = process.env.BUILT_WITH_API_KEY;
 
-  const errorResponse = (message, statusCode = 500) => {
-    return {
-      statusCode: statusCode,
-      body: JSON.stringify({ error: message }),
-    };
-  };
-
   if (!url) {
-    return errorResponse('URL query parameter is required', 400);
+    throw new Error('URL query parameter is required');
   }
 
   if (!apiKey) {
-    return errorResponse('Missing BuiltWith API key in environment variables', 500);
+    throw new Error('Missing BuiltWith API key in environment variables');
   }
 
   const apiUrl = `https://api.builtwith.com/free1/api.json?KEY=${apiKey}&LOOKUP=${encodeURIComponent(url)}`;
@@ -46,11 +39,10 @@ exports.handler = async function (event, context) {
       req.end();
     });
 
-    return {
+    return response;
-      statusCode: 200,
-      body: response,
-    };
   } catch (error) {
-    return errorResponse(`Error making request: ${error.message}`);
+    throw new Error(`Error making request: ${error.message}`);
   }
 };
+
+exports.handler = middleware(builtWithHandler);

api/find-url-ip.js

@@ -1,33 +0,0 @@
-const dns = require('dns');
-
-/* Lambda function to fetch the IP address of a given URL */
-exports.handler = function (event, context, callback) {
-  const addressParam = event.queryStringParameters.url;
-  
-  if (!addressParam) {
-    callback(null, errorResponse('Address parameter is missing.'));
-    return;
-  }
-
-  const address = decodeURIComponent(addressParam)
-    .replaceAll('https://', '')
-    .replaceAll('http://', '');
-
-  dns.lookup(address, (err, ip, family) => {
-    if (err) {
-      callback(null, errorResponse(err.message));
-    } else {
-      callback(null, {
-        statusCode: 200,
-        body: JSON.stringify({ ip, family }),
-      });
-    }
-  });
-};
-
-const errorResponse = (message, statusCode = 444) => {
-  return {
-    statusCode: statusCode,
-    body: JSON.stringify({ error: message }),
-  };
-};

api/follow-redirects.js

@@ -1,35 +0,0 @@
-exports.handler = async (event) => {
-  const { url } = event.queryStringParameters;
-  const redirects = [url];
-
-  try {
-    const got = await import('got');
-    await got.default(url, {
-      followRedirect: true,
-      maxRedirects: 12,
-      hooks: {
-        beforeRedirect: [
-          (options, response) => {
-            redirects.push(response.headers.location);
-          },
-        ],
-      },
-    });
-
-    return {
-      statusCode: 200,
-      body: JSON.stringify({
-        redirects: redirects,
-      }),
-    };
-  } catch (error) {
-    return errorResponse(`Error: ${error.message}`);
-  }
-};
-
-const errorResponse = (message, statusCode = 444) => {
-  return {
-    statusCode: statusCode,
-    body: JSON.stringify({ error: message }),
-  };
-};

api/get-cookies.js

@@ -1,26 +0,0 @@
-const axios = require('axios');
-
-exports.handler = async function(event, context) {
-  const { url } = event.queryStringParameters;
-
-  if (!url) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ message: 'url query string parameter is required' }),
-    };
-  }
-
-  try {
-    const response = await axios.get(url, {withCredentials: true});
-    const cookies = response.headers['set-cookie'];
-    return {
-      statusCode: 200,
-      body: JSON.stringify({ cookies }),
-    };
-  } catch (error) {
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: error.message }),
-    };
-  }
-};

api/get-headers.js

@@ -1,31 +0,0 @@
-const axios = require('axios');
-
-exports.handler = async function(event, context) {
-  const { url } = event.queryStringParameters;
-
-  if (!url) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'url query string parameter is required' }),
-    };
-  }
-
-  try {
-    const response = await axios.get(url, {
-      validateStatus: function (status) {
-        return status >= 200 && status < 600; // Resolve only if the status code is less than 600
-      },
-    });
-
-    return {
-      statusCode: 200,
-      body: JSON.stringify(response.headers),
-    };
-  } catch (error) {
-    console.log(error);
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: error.message }),
-    };
-  }
-};

api/get-ip.js

@@ -0,0 +1,21 @@
+const dns = require('dns');
+const middleware = require('./_common/middleware');
+
+const lookupAsync = (address) => {
+  return new Promise((resolve, reject) => {
+    dns.lookup(address, (err, ip, family) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve({ ip, family });
+      }
+    });
+  });
+};
+
+const handler = async (url) => {
+  const address = url.replaceAll('https://', '').replaceAll('http://', '');
+  return await lookupAsync(address);
+};
+
+exports.handler = middleware(handler);

api/headers.js

@@ -0,0 +1,18 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const handler = async (url, event, context) => {
+  try {
+    const response = await axios.get(url, {
+      validateStatus: function (status) {
+        return status >= 200 && status < 600; // Resolve only if the status code is less than 600
+      },
+    });
+
+    return response.headers;
+  } catch (error) {
+    throw new Error(error.message);
+  }
+};
+
+exports.handler = middleware(handler);

api/hsts.js

@@ -1,8 +1,7 @@
 const https = require('https');
+const middleware = require('./_common/middleware');
 
-exports.handler = async function(event, context) {
+exports.handler = middleware(async (url, event, context) => {
-  const siteURL = event.queryStringParameters.url;
-
   const errorResponse = (message, statusCode = 500) => {
     return {
       statusCode: statusCode,
@@ -16,15 +15,9 @@ exports.handler = async function(event, context) {
     };
   };
 
-  if (!siteURL) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'URL parameter is missing!' }),
-    };
-  }
 
   return new Promise((resolve, reject) => {
-    const req = https.request(siteURL, res => {
+    const req = https.request(url, res => {
       const headers = res.headers;
       const hstsHeader = headers['strict-transport-security'];
 
@@ -60,4 +53,5 @@ exports.handler = async function(event, context) {
 
     req.end();
   });
-};
+});
+

api/lighthouse-report.js

@@ -1,40 +0,0 @@
-const axios = require('axios');
-
-exports.handler = function(event, context, callback) {
-  const { url } = event.queryStringParameters;
-
-  if (!url) {
-    callback(null, {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'URL param is required'}),
-    });
-  }
-
-  const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
-
-  if (!apiKey) {
-    callback(null, {
-      statusCode: 500,
-      body: JSON.stringify({ error: 'API key (GOOGLE_CLOUD_API_KEY) not set'}),
-    });
-  }
-
-  const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile&key=${apiKey}`;
-  
-  axios.get(endpoint)
-  .then(
-    (response) => {
-      callback(null, {
-        statusCode: 200,
-        body: JSON.stringify(response.data),
-      });
-    }
-  ).catch(
-    () => {
-      callback(null, {
-        statusCode: 500,
-        body: JSON.stringify({ error: 'Error running Lighthouse'}),
-      });
-    }
-  );
-};

api/linked-pages.js

@@ -0,0 +1,48 @@
+const axios = require('axios');
+const cheerio = require('cheerio');
+const urlLib = require('url');
+const commonMiddleware = require('./_common/middleware');
+
+const handler = async (url) => {
+  const response = await axios.get(url);
+  const html = response.data;
+  const $ = cheerio.load(html);
+  const internalLinksMap = new Map();
+  const externalLinksMap = new Map();
+
+  // Get all links on the page
+  $('a[href]').each((i, link) => {
+    const href = $(link).attr('href');
+    const absoluteUrl = urlLib.resolve(url, href);
+    
+    // Check if absolute / relative, append to appropriate map or increment occurrence count
+    if (absoluteUrl.startsWith(url)) {
+      const count = internalLinksMap.get(absoluteUrl) || 0;
+      internalLinksMap.set(absoluteUrl, count + 1);
+    } else if (href.startsWith('http://') || href.startsWith('https://')) {
+      const count = externalLinksMap.get(absoluteUrl) || 0;
+      externalLinksMap.set(absoluteUrl, count + 1);
+    }
+  });
+
+  // Sort by most occurrences, remove supplicates, and convert to array
+  const internalLinks = [...internalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
+  const externalLinks = [...externalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
+
+  // If there were no links, then mark as skipped and show reasons
+  if (internalLinks.length === 0 && externalLinks.length === 0) {
+    return {
+      statusCode: 400,
+      body: JSON.stringify({
+        skipped: 'No internal or external links found. '
+          + 'This may be due to the website being dynamically rendered, using a client-side framework (like React), and without SSR enabled. '
+          + 'That would mean that the static HTML returned from the HTTP request doesn\'t contain any meaningful content for Web-Check to analyze. '
+          + 'You can rectify this by using a headless browser to render the page instead.',
+        }),
+    };
+  }
+
+  return { internal: internalLinks, external: externalLinks };
+};
+
+exports.handler = commonMiddleware(handler);

api/mail-config.js

@@ -1,11 +1,11 @@
+const commonMiddleware = require('./_common/middleware');
+
 const dns = require('dns').promises;
 const URL = require('url-parse');
 
-exports.handler = async (event, context) => {
+const handler = async (url, event, context) => {
   try {
-    let domain = event.queryStringParameters.url;
+    const domain = new URL(url).hostname || new URL(url).pathname;
-    const parsedUrl = new URL(domain);
-    domain = parsedUrl.hostname || parsedUrl.pathname;
 
     // Get MX records
     const mxRecords = await dns.resolveMx(domain);
@@ -56,19 +56,13 @@ exports.handler = async (event, context) => {
     }
 
     return {
-      statusCode: 200,
-      body: JSON.stringify({
         mxRecords,
         txtRecords: emailTxtRecords,
         mailServices,
-      }),
+      };
-    };
   } catch (error) {
     if (error.code === 'ENOTFOUND' || error.code === 'ENODATA') {
-      return {
+      return { skipped: 'No mail server in use on this domain' };
-        statusCode: 200,
-        body: JSON.stringify({ skipped: 'No mail server in use on this domain' }),
-      };
     } else {
       return {
         statusCode: 500,
@@ -77,3 +71,5 @@ exports.handler = async (event, context) => {
     }
   }
 };
+
+module.exports.handler = commonMiddleware(handler);

api/ports.js

@@ -1,4 +1,5 @@
 const net = require('net');
+const middleware = require('./_common/middleware');
 
 // A list of commonly used ports.
 const PORTS = [
@@ -12,7 +13,7 @@ async function checkPort(port, domain) {
     return new Promise((resolve, reject) => {
         const socket = new net.Socket();
 
-        socket.setTimeout(1500); // you may want to adjust the timeout
+        socket.setTimeout(1500);
 
         socket.once('connect', () => {
             socket.destroy();
@@ -33,12 +34,8 @@ async function checkPort(port, domain) {
     });
 }
 
-exports.handler = async (event, context) => {
+const handler = async (url, event, context) => {
-  const domain = event.queryStringParameters.url;
+  const domain = url.replace(/(^\w+:|^)\/\//, '');
-  
-  if (!domain) {
-    return errorResponse('Missing domain parameter.');
-  }
   
   const delay = ms => new Promise(res => setTimeout(res, ms));
   const timeout = delay(9000);
@@ -88,3 +85,5 @@ const errorResponse = (message, statusCode = 444) => {
     body: JSON.stringify({ error: message }),
   };
 };
+
+exports.handler = middleware(handler);

api/quality.js

@@ -0,0 +1,22 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const handler = async (url, event, context) => {
+  const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
+
+  if (!url) {
+    throw new Error('URL param is required');
+  }
+
+  if (!apiKey) {
+    throw new Error('API key (GOOGLE_CLOUD_API_KEY) not set');
+  }
+
+  const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile&key=${apiKey}`;
+
+  const response = await axios.get(endpoint);
+  
+  return response.data;
+};
+
+exports.handler = middleware(handler);

api/read-robots-txt.js

@@ -1,45 +0,0 @@
-const axios = require('axios');
-
-exports.handler = async function(event, context) {
-  const siteURL = event.queryStringParameters.url;
-
-  if (!siteURL) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'Missing url query parameter' }),
-    };
-  }
-
-  let parsedURL;
-  try {
-    parsedURL = new URL(siteURL);
-  } catch (error) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'Invalid url query parameter' }),
-    };
-  }
-
-  const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`;
-
-  try {
-    const response = await axios.get(robotsURL);
-
-    if (response.status === 200) {
-      return {
-        statusCode: 200,
-        body: response.data,
-      };
-    } else {
-      return {
-        statusCode: response.status,
-        body: JSON.stringify({ error: 'Failed to fetch robots.txt', statusCode: response.status }),
-      };
-    }
-  } catch (error) {
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: `Error fetching robots.txt: ${error.message}` }),
-    };
-  }
-};

api/redirects.js

@@ -0,0 +1,27 @@
+const handler = async (url) => {
+  const redirects = [url];
+  const got = await import('got');
+
+  try {
+    await got.default(url, {
+      followRedirect: true,
+      maxRedirects: 12,
+      hooks: {
+        beforeRedirect: [
+          (options, response) => {
+            redirects.push(response.headers.location);
+          },
+        ],
+      },
+    });
+
+    return {
+      redirects: redirects,
+    };
+  } catch (error) {
+    throw new Error(`Error: ${error.message}`);
+  }
+};
+
+const middleware = require('./_common/middleware');
+exports.handler = middleware(handler);

api/robots-txt.js

@@ -0,0 +1,66 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const parseRobotsTxt = (content) => {
+  const lines = content.split('\n');
+  const rules = [];
+
+  lines.forEach(line => {
+    line = line.trim();  // This removes trailing and leading whitespaces
+
+    let match = line.match(/^(Allow|Disallow):\s*(\S*)$/i);
+    if (match) {
+      const rule = {
+        lbl: match[1],
+        val: match[2],
+      };
+      
+      rules.push(rule);
+    } else {
+      match = line.match(/^(User-agent):\s*(\S*)$/i);
+      if (match) {
+        const rule = {
+          lbl: match[1],
+          val: match[2],
+        };
+        
+        rules.push(rule);
+      }
+    }
+  });
+  return { robots: rules };
+}
+
+const handler = async function(url) {
+  let parsedURL;
+  try {
+    parsedURL = new URL(url);
+  } catch (error) {
+    return {
+      statusCode: 400,
+      body: JSON.stringify({ error: 'Invalid url query parameter' }),
+    };
+  }
+
+  const robotsURL = `${parsedURL.protocol}//${parsedURL.hostname}/robots.txt`;
+
+  try {
+    const response = await axios.get(robotsURL);
+
+    if (response.status === 200) {
+      return parseRobotsTxt(response.data);
+    } else {
+      return {
+        statusCode: response.status,
+        body: JSON.stringify({ error: 'Failed to fetch robots.txt', statusCode: response.status }),
+      };
+    }
+  } catch (error) {
+    return {
+      statusCode: 500,
+      body: JSON.stringify({ error: `Error fetching robots.txt: ${error.message}` }),
+    };
+  }
+};
+
+exports.handler = middleware(handler);

api/screenshot.js

@@ -1,16 +1,11 @@
 const puppeteer = require('puppeteer-core');
 const chromium = require('chrome-aws-lambda');
+const middleware = require('./_common/middleware');
 
-exports.handler = async (event, context, callback) => {
+const screenshotHandler = async (targetUrl) => {
-  let browser = null;
-  let targetUrl = event.queryStringParameters.url;
 
   if (!targetUrl) {
-    callback(null, {
+    throw new Error('URL is missing from queryStringParameters');
-      statusCode: 400,
-      body: JSON.stringify({ error: 'URL is missing from queryStringParameters' }),
-    });
-    return;
   }
 
   if (!targetUrl.startsWith('http://') && !targetUrl.startsWith('https://')) {
@@ -20,13 +15,10 @@ exports.handler = async (event, context, callback) => {
   try {
     new URL(targetUrl);
   } catch (error) {
-    callback(null, {
+    throw new Error('URL provided is invalid');
-      statusCode: 400,
-      body: JSON.stringify({ error: 'URL provided is invalid' }),
-    });
-    return;
   }
 
+  let browser = null;
   try {
       browser = await puppeteer.launch({
       args: chromium.args,
@@ -40,9 +32,7 @@ exports.handler = async (event, context, callback) => {
     let page = await browser.newPage();
 
     await page.emulateMediaFeatures([{ name: 'prefers-color-scheme', value: 'dark' }]);
-
     page.setDefaultNavigationTimeout(8000);
-
     await page.goto(targetUrl, { waitUntil: 'domcontentloaded' });
 
     await page.evaluate(() => {
@@ -57,24 +47,15 @@ exports.handler = async (event, context, callback) => {
     });
 
     const screenshotBuffer = await page.screenshot();
-
     const base64Screenshot = screenshotBuffer.toString('base64');
 
-    const response = {
+    return { image: base64Screenshot };
-      statusCode: 200,
-      body: JSON.stringify({ image: base64Screenshot }),
-    };
 
-    callback(null, response);
-  } catch (error) {
-    console.log(error);
-    callback(null, {
-      statusCode: 500,
-      body: JSON.stringify({ error: `An error occurred: ${error.message}` }),
-    });
   } finally {
     if (browser !== null) {
       await browser.close();
     }
   }
 };
+
+exports.handler = middleware(screenshotHandler);

api/security-txt.js

@@ -1,5 +1,6 @@
 const { https } = require('follow-redirects');
 const { URL } = require('url');
+const middleware = require('./_common/middleware');
 
 const SECURITY_TXT_PATHS = [
   '/security.txt',
@@ -37,59 +38,39 @@ const isPgpSigned = (result) => {
   return false;
 };
 
-exports.handler = async (event, context) => {
+const securityTxtHandler = async (urlParam) => {
-  const urlParam = event.queryStringParameters.url;
-  if (!urlParam) {
-    return { 
-      statusCode: 400, 
-      body: JSON.stringify({ error: 'Missing url parameter' }) 
-    };
-  }
 
   let url;
   try {
     url = new URL(urlParam.includes('://') ? urlParam : 'https://' + urlParam);
   } catch (error) {
-    return {
+    throw new Error('Invalid URL format');
-      statusCode: 500,
-      body: JSON.stringify({ error: 'Invalid URL format' }),
-    };
   }
   url.pathname = '';
   
   for (let path of SECURITY_TXT_PATHS) {
     try {
       const result = await fetchSecurityTxt(url, path);
-      if (result && result.includes('<html')) return {
+      if (result && result.includes('<html')) return { isPresent: false };
-        statusCode: 200,
-        body: JSON.stringify({ isPresent: false }),
-      };
       if (result) {
         return {
-          statusCode: 200,
+          isPresent: true,
-          body: JSON.stringify({
+          foundIn: path,
-            isPresent: true,
+          content: result,
-            foundIn: path,
+          isPgpSigned: isPgpSigned(result),
-            content: result,
+          fields: parseResult(result),
-            isPgpSigned: isPgpSigned(result),
-            fields: parseResult(result),
-          }),
         };
       }
     } catch (error) {
-      return {
+      throw new Error(error.message);
-        statusCode: 500,
-        body: JSON.stringify({ error: error.message }),
-      };
     }
   }
   
-  return {
+  return { isPresent: false };
-    statusCode: 404,
-    body: JSON.stringify({ isPresent: false }),
-  };
 };
 
+exports.handler = middleware(securityTxtHandler);
+
 async function fetchSecurityTxt(baseURL, path) {
   return new Promise((resolve, reject) => {
     const url = new URL(path, baseURL);

api/sitemap.js

@@ -1,8 +1,9 @@
+const commonMiddleware = require('./_common/middleware');
+
 const axios = require('axios');
 const xml2js = require('xml2js');
 
-exports.handler = async (event) => {
+const handler = async (url) => {
-  const url = event.queryStringParameters.url;
   let sitemapUrl = `${url}/sitemap.xml`;
 
   try {
@@ -59,3 +60,5 @@ exports.handler = async (event) => {
     }
   }
 };
+
+exports.handler = commonMiddleware(handler);

api/social-tags.js

@@ -1,8 +1,9 @@
+const commonMiddleware = require('./_common/middleware');
+
 const axios = require('axios');
 const cheerio = require('cheerio');
 
-exports.handler = async (event, context) => {
+const handler = async (url) => {
-  let url = event.queryStringParameters.url;
   
   // Check if url includes protocol
   if (!url.startsWith('http://') && !url.startsWith('https://')) {
@@ -66,3 +67,5 @@ exports.handler = async (event, context) => {
     };
   }
 };
+
+exports.handler = commonMiddleware(handler);

api/ssl-check.js

@@ -1,50 +0,0 @@
-const https = require('https');
-
-exports.handler = async function (event, context) {
-  const { url } = event.queryStringParameters;
-
-  const errorResponse = (message, statusCode = 500) => {
-    return {
-      statusCode: statusCode,
-      body: JSON.stringify({ error: message }),
-    };
-  };
-
-  if (!url) {
-    return errorResponse('URL query parameter is required', 400);
-  }
-
-  try {
-    const response = await new Promise((resolve, reject) => {
-      const req = https.request(url, res => {
-
-        // Check if the SSL handshake was authorized
-        if (!res.socket.authorized) {
-          resolve(errorResponse(`SSL handshake not authorized. Reason: ${res.socket.authorizationError}`));
-        } else {
-          let cert = res.socket.getPeerCertificate(true);
-          if (!cert || Object.keys(cert).length === 0) {
-            resolve(errorResponse("No certificate presented by the server."));
-          } else {
-            // omit the raw and issuerCertificate fields
-            const { raw, issuerCertificate, ...certWithoutRaw } = cert;
-            resolve({
-              statusCode: 200,
-              body: JSON.stringify(certWithoutRaw),
-            });
-          }
-        }
-      });
-
-      req.on('error', error => {
-        resolve(errorResponse(`Error fetching site certificate: ${error.message}`));
-      });
-
-      req.end();
-    });
-
-    return response;
-  } catch (error) {
-    return errorResponse(`Unexpected error occurred: ${error.message}`);
-  }
-};

api/ssl.js

@@ -0,0 +1,37 @@
+const https = require('https');
+const middleware = require('./_common/middleware');
+
+const fetchSiteCertificateHandler = async (url) => {
+  try {
+    const response = await new Promise((resolve, reject) => {
+      const req = https.request(url, res => {
+        
+        // Check if the SSL handshake was authorized
+        if (!res.socket.authorized) {
+          reject(new Error(`SSL handshake not authorized. Reason: ${res.socket.authorizationError}`));
+        } else {
+          let cert = res.socket.getPeerCertificate(true);
+          if (!cert || Object.keys(cert).length === 0) {
+            reject(new Error("No certificate presented by the server."));
+          } else {
+            // omit the raw and issuerCertificate fields
+            const { raw, issuerCertificate, ...certWithoutRaw } = cert;
+            resolve(certWithoutRaw);
+          }
+        }
+      });
+
+      req.on('error', error => {
+        reject(new Error(`Error fetching site certificate: ${error.message}`));
+      });
+
+      req.end();
+    });
+
+    return response;
+  } catch (error) {
+    throw new Error(error.message);
+  }
+};
+
+exports.handler = middleware(fetchSiteCertificateHandler);

api/status.js

@@ -1,14 +1,10 @@
 const https = require('https');
 const { performance, PerformanceObserver } = require('perf_hooks');
+const middleware = require('./_common/middleware');
 
-exports.handler = async function(event, context) {
+const checkURLHandler = async (url) => {
-  const { url } = event.queryStringParameters;
-
   if (!url) {
-    return {
+    throw new Error('You must provide a URL query parameter!');
-      statusCode: 400,
-      body: JSON.stringify({ error: 'You must provide a URL query parameter!' }),
-    };
   }
 
   let dnsLookupTime;
@@ -43,10 +39,7 @@ exports.handler = async function(event, context) {
     });
 
     if (responseCode < 200 || responseCode >= 400) {
-      return {
+      throw new Error(`Received non-success response code: ${responseCode}`);
-        statusCode: 200,
-        body: JSON.stringify({ error: `Received non-success response code: ${responseCode}` }),
-      };
     }
 
     performance.mark('B');
@@ -54,16 +47,12 @@ exports.handler = async function(event, context) {
     let responseTime = performance.now() - startTime;
     obs.disconnect();
 
-    return {
+    return { isUp: true, dnsLookupTime, responseTime, responseCode };
-      statusCode: 200,
-      body: JSON.stringify({ isUp: true, dnsLookupTime, responseTime, responseCode }),
-    };
 
   } catch (error) {
     obs.disconnect();
-    return {
+    throw error;
-      statusCode: 200,
-      body: JSON.stringify({ error: `Error during operation: ${error.message}` }),
-    };
   }
 };
+
+exports.handler = middleware(checkURLHandler);

api/tech-stack.js

@@ -1,69 +1,30 @@
 const Wappalyzer = require('wappalyzer');
+const middleware = require('./_common/middleware');
 
-const analyze = async (url) => {
+const analyzeSiteTechnologies = async (url) => {
-
   const options = {};
 
   const wappalyzer = new Wappalyzer(options);
-  return (async function() {
-    try {
-      await wappalyzer.init()
-      const headers = {}
-      const storage = {
-        local: {},
-        session: {},
-      }
-      const site = await wappalyzer.open(url, headers, storage)
-      const results = await site.analyze()
-      return results;
-    } catch (error) {
-      return error;
-    } finally {
-      await wappalyzer.destroy()
-    }
-  })();
-}
-
-exports.handler = async (event, context, callback) => {
-  // Validate URL parameter
-  if (!event.queryStringParameters || !event.queryStringParameters.url) {
-    return {
-      statusCode: 400,
-      body: JSON.stringify({ error: 'Missing url parameter' }),
-    };
-  }
-
-  // Get URL from param
-  let url = event.queryStringParameters.url;
-  if (!/^https?:\/\//i.test(url)) {
-    url = 'http://' + url;
-  }
 
   try {
-    return analyze(url).then(
+    await wappalyzer.init();
-      (results) => {
+    const headers = {};
-        if (!results.technologies || results.technologies.length === 0) {
+    const storage = {
-          return {
+      local: {},
-            statusCode: 200,
+      session: {},
-            body: JSON.stringify({ error: 'Unable to find any technologies for site' }),
-          };
-        }
-        return {
-          statusCode: 200,
-          body: JSON.stringify(results),
-        }
-      }
-    )
-    .catch((error) => {
-      return {
-      statusCode: 500,
-      body: JSON.stringify({ error: error.message }),
-      };
-    });
-  } catch (error) {
-    return {
-      statusCode: 500,
-      body: JSON.stringify({ error: error.message }),
     };
+    const site = await wappalyzer.open(url, headers, storage);
+    const results = await site.analyze();
+
+    if (!results.technologies || results.technologies.length === 0) {
+      throw new Error('Unable to find any technologies for site');
+    }
+    return results;
+  } catch (error) {
+    throw new Error(error.message);
+  } finally {
+    await wappalyzer.destroy();
   }
 };
+
+exports.handler = middleware(analyzeSiteTechnologies);

api/trace-route.js

@@ -1,55 +1,31 @@
 const traceroute = require('traceroute');
 const url = require('url');
+const middleware = require('./_common/middleware');
 
-exports.handler = async function(event, context) {
+const executeTraceroute = async (urlString, context) => {
-  const urlString = event.queryStringParameters.url;
+  // Parse the URL and get the hostname
+  const urlObject = url.parse(urlString);
+  const host = urlObject.hostname;
 
-  try {
+  if (!host) {
-    if (!urlString) {
+    throw new Error('Invalid URL provided');
-      throw new Error('URL parameter is missing!');
+  }
-    }
 
-    // Parse the URL and get the hostname
+  // Traceroute with callback
-    const urlObject = url.parse(urlString);
+  const result = await new Promise((resolve, reject) => {
-    const host = urlObject.hostname;
+    traceroute.trace(host, (err, hops) => {
-
+      if (err || !hops) {
-    if (!host) {
+        reject(err || new Error('No hops found'));
-      throw new Error('Invalid URL provided');
+      } else {
-    }
+        resolve(hops);
-
-    // Traceroute with callback
-    const result = await new Promise((resolve, reject) => {
-      traceroute.trace(host, (err, hops) => {
-        if (err || !hops) {
-          reject(err || new Error('No hops found'));
-        } else {
-          resolve(hops);
-        }
-      });
-
-      // Check if remaining time is less than 8.8 seconds, then reject promise
-      if (context.getRemainingTimeInMillis() < 8800) {
-        reject(new Error('Lambda is about to timeout'));
       }
     });
+  });
 
-    return {
+  return {
-      statusCode: 200,
+    message: "Traceroute completed!",
-      body: JSON.stringify({
+    result,
-        message: "Traceroute completed!",
+  };
-        result,
-      }),
-    };
-  } catch (err) {
-    const message = err.code === 'ENOENT'
-      ? 'Traceroute command is not installed on the host.'
-      : err.message;
-
-    return {
-      statusCode: 500,
-      body: JSON.stringify({
-        error: message,
-      }),
-    };
-  }
 };
+
+exports.handler = middleware(executeTraceroute);

api/txt-records.js

@@ -1,9 +1,11 @@
 const dns = require('dns').promises;
+const middleware = require('./_common/middleware');
 
-exports.handler = async (event) => {
+const handler = async (url, event, context) => {
-  const url = new URL(event.queryStringParameters.url);
   try {
-    const txtRecords = await dns.resolveTxt(url.hostname);
+    const parsedUrl = new URL(url);
+    
+    const txtRecords = await dns.resolveTxt(parsedUrl.hostname);
 
     // Parsing and formatting TXT records into a single object
     const readableTxtRecords = txtRecords.reduce((acc, recordArray) => {
@@ -16,15 +18,15 @@ exports.handler = async (event) => {
       return { ...acc, ...recordObject };
     }, {});
 
-    return {
+    return readableTxtRecords;
-      statusCode: 200,
+
-      body: JSON.stringify(readableTxtRecords),
-    };
   } catch (error) {
-    console.error('Error:', error);
+    if (error.code === 'ERR_INVALID_URL') {
-    return {
+      throw new Error(`Invalid URL ${error}`);
-      statusCode: 500,
+    } else {
-      body: JSON.stringify({ error: error.message }),
+      throw error;
-    };
+    }
   }
 };
+
+exports.handler = middleware(handler);

api/whois.js

@@ -1,14 +1,6 @@
 const net = require('net');
 const psl = require('psl');
-// const { URL } = require('url');
+const middleware = require('./_common/middleware');
-
-const errorResponse = (message, statusCode = 444) => {
-  return {
-    statusCode: statusCode,
-    body: JSON.stringify({ error: message }),
-  };
-};
-
 
 const getBaseDomain = (url) => {
   let protocol = '';
@@ -22,55 +14,7 @@ const getBaseDomain = (url) => {
   return protocol + parsed.domain;
 };
 
-
-exports.handler = async function(event, context) {
-  let url = event.queryStringParameters.url;
-
-  if (!url) {
-    return errorResponse('URL query parameter is required.', 400);
-  }
-
-  if (!url.startsWith('http://') && !url.startsWith('https://')) {
-    url = 'http://' + url;
-  }
-
-  let hostname;
-  try {
-    hostname = getBaseDomain(new URL(url).hostname);
-  } catch (error) {
-    return errorResponse(`Unable to parse URL: ${error}`, 400);
-  }
-
-  return new Promise((resolve, reject) => {
-    const client = net.createConnection({ port: 43, host: 'whois.internic.net' }, () => {
-      client.write(hostname + '\r\n');
-    });
-
-    let data = '';
-    client.on('data', (chunk) => {
-      data += chunk;
-    });
-
-    client.on('end', () => {
-      try {
-        const parsedData = parseWhoisData(data);
-        resolve({
-          statusCode: 200,
-          body: JSON.stringify(parsedData),
-        });
-      } catch (error) {
-        resolve(errorResponse(error.message));
-      }
-    });
-
-    client.on('error', (err) => {
-      resolve(errorResponse(err.message, 500));
-    });
-  });
-};
-
 const parseWhoisData = (data) => {
-
   if (data.includes('No match for')) {
     return { error: 'No matches found for domain in internic database'};
   }
@@ -100,3 +44,41 @@ const parseWhoisData = (data) => {
   return parsedData;
 };
 
+const fetchWhoisData = async (url) => {
+  if (!url.startsWith('http://') && !url.startsWith('https://')) {
+    url = 'http://' + url;
+  }
+
+  let hostname;
+  try {
+    hostname = getBaseDomain(new URL(url).hostname);
+  } catch (error) {
+    throw new Error(`Unable to parse URL: ${error}`);
+  }
+
+  return new Promise((resolve, reject) => {
+    const client = net.createConnection({ port: 43, host: 'whois.internic.net' }, () => {
+      client.write(hostname + '\r\n');
+    });
+
+    let data = '';
+    client.on('data', (chunk) => {
+      data += chunk;
+    });
+
+    client.on('end', () => {
+      try {
+        const parsedData = parseWhoisData(data);
+        resolve(parsedData);
+      } catch (error) {
+        reject(error);
+      }
+    });
+
+    client.on('error', (err) => {
+      reject(err);
+    });
+  });
+};
+
+exports.handler = middleware(fetchWhoisData);

package.json

@@ -17,7 +17,7 @@
   "scripts": {
     "dev": "netlify dev",
     "serve": "netlify serve --offline",
-    "start": "react-scripts start",
+    "start": "node server",
     "build": "react-scripts build",
     "test": "react-scripts test",
     "eject": "react-scripts eject"
@@ -35,9 +35,13 @@
     "@types/react-router-dom": "^5.3.3",
     "@types/react-simple-maps": "^3.0.0",
     "@types/styled-components": "^5.1.26",
+    "aws-serverless-express": "^3.4.0",
     "axios": "^1.4.0",
     "cheerio": "^1.0.0-rc.12",
     "chrome-aws-lambda": "^10.1.0",
+    "chromium": "^3.0.3",
+    "connect-history-api-fallback": "^2.0.0",
+    "dotenv": "^16.3.1",
     "flatted": "^3.2.7",
     "follow-redirects": "^1.15.2",
     "got": "^13.0.0",
@@ -46,7 +50,7 @@
     "perf_hooks": "^0.0.1",
     "psl": "^1.9.0",
     "puppeteer": "^20.9.0",
-    "puppeteer-core": "^20.9.0",
+    "puppeteer-core": "^21.0.3",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-masonry-css": "^1.0.16",
@@ -57,9 +61,9 @@
     "styled-components": "^6.0.5",
     "traceroute": "^1.0.0",
     "typescript": "^5.1.6",
-    "wappalyzer": "^6.10.63",
+    "wappalyzer": "^6.10.65",
     "web-vitals": "^3.4.0",
-    "xml2js": "^0.6.0"
+    "xml2js": "^0.6.2"
   },
   "eslintConfig": {
     "extends": [
@@ -82,5 +86,12 @@
   "compilerOptions": {
     "allowJs": true,
     "outDir": "./dist"
+  },
+  "devDependencies": {
+    "serverless-domain-manager": "^7.1.1",
+    "serverless-offline": "^12.0.4",
+    "serverless-webpack": "^5.13.0",
+    "webpack": "^5.88.2",
+    "webpack-node-externals": "^3.0.0"
   }
 }

server.js

@@ -0,0 +1,116 @@
+const express = require('express');
+const awsServerlessExpress = require('aws-serverless-express');
+const fs = require('fs');
+const path = require('path');
+const historyApiFallback = require('connect-history-api-fallback');
+require('dotenv').config();
+
+const app = express();
+
+const API_DIR = '/api'; // Name of the dir containing the lambda functions
+const dirPath = path.join(__dirname, API_DIR); // Path to the lambda functions dir
+const guiPath = path.join(__dirname, 'build');
+
+// Execute the lambda function
+const executeHandler = async (handler, req) => {
+  return new Promise((resolve, reject) => {
+    const callback = (err, response) => err ? reject(err) : resolve(response);
+    const promise = handler(req, {}, callback);
+
+    if (promise && typeof promise.then === 'function') {
+      promise.then(resolve).catch(reject);
+    }
+  });
+};
+
+// Array of all the lambda function file names
+const fileNames = fs.readdirSync(dirPath, { withFileTypes: true })
+  .filter(dirent => dirent.isFile() && dirent.name.endsWith('.js'))
+  .map(dirent => dirent.name);
+
+const handlers = {};
+
+fileNames.forEach(file => {
+  const routeName = file.split('.')[0];
+  const route = `${API_DIR}/${routeName}`;
+  const handler = require(path.join(dirPath, file)).handler;
+
+  handlers[route] = handler;
+  
+  app.get(route, async (req, res) => {
+    try {
+      const { statusCode = 200, body = '' } = await executeHandler(handler, req);
+      res.status(statusCode).json(JSON.parse(body));
+    } catch (err) {
+      res.status(500).json({ error: err.message });
+    }
+  });
+});
+
+const timeout = (ms, jobName = null) => {
+  return new Promise((_, reject) => {
+    setTimeout(() => {
+      reject(new Error(`Timed out after the ${ms/1000} second limit${jobName ? `, when executing the ${jobName} task` : ''}`));
+    }, ms);
+  });
+}
+
+app.get('/api', async (req, res) => {
+  const results = {};
+  const { url } = req.query;
+  const maxExecutionTime = process.env.API_TIMEOUT_LIMIT || 15000;
+
+  const handlerPromises = Object.entries(handlers).map(async ([route, handler]) => {
+    const routeName = route.replace(`${API_DIR}/`, '');
+
+    try {
+      const result = await Promise.race([
+        executeHandler(handler, { query: { url } }),
+        timeout(maxExecutionTime, routeName)
+      ]);
+      results[routeName] = JSON.parse((result || {}).body);
+      
+    } catch (err) {
+      results[routeName] = { error: err.message };
+    }
+  });
+
+  await Promise.all(handlerPromises);
+  res.json(results);
+});
+
+// Handle SPA routing
+app.use(historyApiFallback({
+  rewrites: [
+    { from: /^\/api\/.*$/, to: function(context) { return context.parsedUrl.path; } },
+  ]
+}));
+
+// Serve up the GUI - if build dir exists, and GUI feature enabled
+if (process.env.DISABLE_GUI && process.env.DISABLE_GUI !== 'false') {
+  app.get('*', (req, res) => {
+      res.status(500).send(
+        'Welcome to Web-Check!<br />Access the API endpoints at '
+        +'<a href="/api"><code>/api</code></a>'
+      );
+  });
+} else if (!fs.existsSync(guiPath)) {
+  app.get('*', (req, res) => {
+    res.status(500).send(
+      'Welcome to Web-Check!<br />Looks like the GUI app has not yet been compiled, '
+      +'run <code>yarn build</code> to continue, then restart the server.'
+    );
+});
+} else { // GUI enabled, and build files present, let's go!!
+  app.use(express.static(guiPath));
+}
+
+// Create serverless express server
+const port = process.env.PORT || 3000;
+const server = awsServerlessExpress.createServer(app).listen(port, () => {
+  console.log(`Server is running on port ${port}`);
+});
+
+exports.handler = (event, context) => {
+  awsServerlessExpress.proxy(server, event, context);
+};

serverless.yml

@@ -0,0 +1,179 @@
+service: web-check-api
+
+provider:
+  name: aws
+  runtime: nodejs14.x
+  region: us-east-1
+
+functions:
+  dnssec:
+    handler: api/dnssec.handler
+    events:
+      - http:
+          path: api/dnssec
+          method: get
+  linkedPages:
+    handler: api/linked-pages.handler
+    events:
+      - http:
+          path: api/linked-pages
+          method: get
+  robotsTxt:
+    handler: api/robots-txt.handler
+    events:
+      - http:
+          path: api/robots-txt
+          method: get
+  ssl:
+    handler: api/ssl.handler
+    events:
+      - http:
+          path: api/ssl
+          method: get
+  whois:
+    handler: api/whois.handler
+    events:
+      - http:
+          path: api/whois
+          method: get
+  carbon:
+    handler: api/carbon.handler
+    events:
+      - http:
+          path: api/carbon
+          method: get
+  features:
+    handler: api/features.handler
+    events:
+      - http:
+          path: api/features
+          method: get
+  mailConfig:
+    handler: api/mail-config.handler
+    events:
+      - http:
+          path: api/mail-config
+          method: get
+  screenshot:
+    handler: api/screenshot.handler
+    events:
+      - http:
+          path: api/screenshot
+          method: get
+  status:
+    handler: api/status.handler
+    events:
+      - http:
+          path: api/status
+          method: get
+  cookies:
+    handler: api/cookies.handler
+    events:
+      - http:
+          path: api/cookies
+          method: get
+  getIp:
+    handler: api/get-ip.handler
+    events:
+      - http:
+          path: api/get-ip
+          method: get
+  ports:
+    handler: api/ports.handler
+    events:
+      - http:
+          path: api/ports
+          method: get
+  securityTxt:
+    handler: api/security-txt.handler
+    events:
+      - http:
+          path: api/security-txt
+          method: get
+  techStack:
+    handler: api/tech-stack.handler
+    events:
+      - http:
+          path: api/tech-stack
+          method: get
+  dnsServer:
+    handler: api/dns-server.handler
+    events:
+      - http:
+          path: api/dns-server
+          method: get
+  headers:
+    handler: api/headers.handler
+    events:
+      - http:
+          path: api/headers
+          method: get
+  quality:
+    handler: api/quality.handler
+    events:
+      - http:
+          path: api/quality
+          method: get
+  sitemap:
+    handler: api/sitemap.handler
+    events:
+      - http:
+          path: api/sitemap
+          method: get
+  traceRoute:
+    handler: api/trace-route.handler
+    events:
+      - http:
+          path: api/trace-route
+          method: get
+  dns:
+    handler: api/dns.handler
+    events:
+      - http:
+          path: api/dns
+          method: get
+  hsts:
+    handler: api/hsts.handler
+    events:
+      - http:
+          path: api/hsts
+          method: get
+  redirects:
+    handler: api/redirects.handler
+    events:
+      - http:
+          path: api/redirects
+          method: get
+  socialTags:
+    handler: api/social-tags.handler
+    events:
+      - http:
+          path: api/social-tags
+          method: get
+  txtRecords:
+    handler: api/txt-records.handler
+    events:
+      - http:
+          path: api/txt-records
+          method: get
+
+
+plugins:
+  # - serverless-webpack
+  - serverless-domain-manager
+  - serverless-offline
+
+custom:
+  webpack:
+    webpackConfig: 'api/_common/aws-webpack.config.js'
+    includeModules: true
+
+  customDomain:
+    domainName: example.com
+    basePath: 'api'
+    stage: ${self:provider.stage}
+    createRoute53Record: true
+
+  serverless-offline:
+    prefix: ''
+    httpPort: 3000

src/pages/NotFound.tsx

@@ -20,6 +20,10 @@ const AboutContainer = styled.div`
     color: ${colors.primary};
   }
   .im-drink { font-size: 6rem; }
+  header {
+    width: auto;
+    margin: 1rem;
+  }
 `;
 
 const HeaderLinkContainer = styled.nav`

src/pages/Results.tsx

@@ -149,14 +149,14 @@ const Results = (): JSX.Element => {
 
   const urlTypeOnly = ['url'] as AddressType[]; // Many jobs only run with these address types
 
-  const api = '/api';
+  const api = process.env.REACT_APP_API_ENDPOINT || '/api'; // Where is the API hosted?
 
   // Fetch and parse IP address for given URL
   const [ipAddress, setIpAddress] = useMotherHook({
     jobId: 'get-ip',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/find-url-ip?url=${address}`)
+    fetchRequest: () => fetch(`${api}/get-ip?url=${address}`)
     .then(res => parseJson(res))
     .then(res => res.ip),
   });
@@ -175,7 +175,7 @@ const Results = (): JSX.Element => {
     jobId: 'ssl',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/ssl-check?url=${address}`).then((res) => parseJson(res)),
+    fetchRequest: () => fetch(`${api}/ssl?url=${address}`).then((res) => parseJson(res)),
   });
 
   // Fetch and parse cookies info
@@ -183,7 +183,7 @@ const Results = (): JSX.Element => {
     jobId: 'cookies',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/get-cookies?url=${address}`)
+    fetchRequest: () => fetch(`${api}/cookies?url=${address}`)
       .then(res => parseJson(res))
       .then(res => parseCookies(res.cookies)),
   });
@@ -193,9 +193,8 @@ const Results = (): JSX.Element => {
     jobId: 'robots-txt',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/read-robots-txt?url=${address}`)
+    fetchRequest: () => fetch(`${api}/robots-txt?url=${address}`)
-      .then(res => res.text())
+      .then(res => parseJson(res)),
-      .then(res => parseRobotsTxt(res)),
   });
 
   // Fetch and parse headers
@@ -203,7 +202,7 @@ const Results = (): JSX.Element => {
     jobId: 'headers',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/get-headers?url=${address}`).then(res => parseJson(res)),
+    fetchRequest: () => fetch(`${api}/headers?url=${address}`).then(res => parseJson(res)),
   });
 
   // Fetch and parse DNS records
@@ -211,7 +210,7 @@ const Results = (): JSX.Element => {
     jobId: 'dns',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/get-dns?url=${address}`).then(res => parseJson(res)),
+    fetchRequest: () => fetch(`${api}/dns?url=${address}`).then(res => parseJson(res)),
   });
 
   // Fetch and parse Lighthouse performance data
@@ -219,7 +218,7 @@ const Results = (): JSX.Element => {
     jobId: 'quality',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/lighthouse-report?url=${address}`)
+    fetchRequest: () => fetch(`${api}/quality?url=${address}`)
       .then(res => parseJson(res))
       .then(res => res?.lighthouseResult || { error: 'No Data'}),
   });
@@ -249,7 +248,7 @@ const Results = (): JSX.Element => {
     jobId: 'ports',
     updateLoadingJobs,
     addressInfo: { address: ipAddress, addressType: 'ipV4', expectedAddressTypes: ['ipV4', 'ipV6'] },
-    fetchRequest: () => fetch(`${api}/check-ports?url=${ipAddress}`)
+    fetchRequest: () => fetch(`${api}/ports?url=${ipAddress}`)
       .then(res => parseJson(res)),
   });
 
@@ -268,7 +267,7 @@ const Results = (): JSX.Element => {
     jobId: 'txt-records',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/get-txt?url=${address}`).then(res => parseJson(res)),
+    fetchRequest: () => fetch(`${api}/txt-records?url=${address}`).then(res => parseJson(res)),
   });
 
   // Fetches URL redirects
@@ -276,7 +275,7 @@ const Results = (): JSX.Element => {
     jobId: 'redirects',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/follow-redirects?url=${address}`).then(res => parseJson(res)),
+    fetchRequest: () => fetch(`${api}/redirects?url=${address}`).then(res => parseJson(res)),
   });
 
   // Get current status and response time of server
@@ -284,7 +283,7 @@ const Results = (): JSX.Element => {
     jobId: 'status',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/server-status?url=${address}`).then(res => parseJson(res)),
+    fetchRequest: () => fetch(`${api}/status?url=${address}`).then(res => parseJson(res)),
   });
 
   // Get current status and response time of server
@@ -308,7 +307,7 @@ const Results = (): JSX.Element => {
     jobId: 'carbon',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/get-carbon?url=${address}`).then(res => parseJson(res)),
+    fetchRequest: () => fetch(`${api}/carbon?url=${address}`).then(res => parseJson(res)),
   });
 
   // Check if a site is on the HSTS preload list
@@ -316,7 +315,7 @@ const Results = (): JSX.Element => {
     jobId: 'hsts',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/check-hsts?url=${address}`).then(res => parseJson(res)),
+    fetchRequest: () => fetch(`${api}/hsts?url=${address}`).then(res => parseJson(res)),
   });
 
   // Get a websites listed pages, from sitemap
@@ -356,7 +355,7 @@ const Results = (): JSX.Element => {
     jobId: 'features',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/site-features?url=${address}`)
+    fetchRequest: () => fetch(`${api}/features?url=${address}`)
     .then(res => parseJson(res))
     .then(res => {
       if (res.Errors && res.Errors.length > 0) {
@@ -371,7 +370,7 @@ const Results = (): JSX.Element => {
     jobId: 'dnssec',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/dns-sec?url=${address}`).then(res => parseJson(res)),
+    fetchRequest: () => fetch(`${api}/dnssec?url=${address}`).then(res => parseJson(res)),
   });
 
   // Run a manual whois lookup on the domain
@@ -379,7 +378,7 @@ const Results = (): JSX.Element => {
     jobId: 'domain',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/whois-lookup?url=${address}`).then(res => parseJson(res)),
+    fetchRequest: () => fetch(`${api}/whois?url=${address}`).then(res => parseJson(res)),
   });
 
   // Get the DNS server(s) for a domain, and test DoH/DoT support
@@ -395,7 +394,7 @@ const Results = (): JSX.Element => {
     jobId: 'linked-pages',
     updateLoadingJobs,
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
-    fetchRequest: () => fetch(`${api}/content-links?url=${address}`).then(res => parseJson(res)),
+    fetchRequest: () => fetch(`${api}/linked-pages?url=${address}`).then(res => parseJson(res)),
   });
 
   // Get mail config for server, based on DNS records