Merge pull request #118 from Lissy93/FEAT/override-timeout

FEAT: Override timeout

api/_common/middleware.js

@@ -2,13 +2,46 @@ const normalizeUrl = (url) => {
   return url.startsWith('http') ? url : `https://${url}`;
 };
 
+
+// If present, set a shorter timeout for API requests
+const TIMEOUT = parseInt(process.env.API_TIMEOUT_LIMIT, 10) || 60000;
+
+// If present, set CORS allowed origins for responses
+const ALLOWED_ORIGINS = process.env.API_CORS_ORIGIN || '*';
+
+// Set the platform currently being used
+let PLATFORM = 'NETLIFY';
+if (process.env.PLATFORM) { PLATFORM = process.env.PLATFORM.toUpperCase(); }
+else if (process.env.VERCEL) { PLATFORM = 'VERCEL'; }
+else if (process.env.WC_SERVER) { PLATFORM = 'NODE'; }
+
+// Define the headers to be returned with each response
 const headers = {
-  'Access-Control-Allow-Origin': process.env.API_CORS_ORIGIN || '*',
+  'Access-Control-Allow-Origin': ALLOWED_ORIGINS,
   'Access-Control-Allow-Credentials': true,
   'Content-Type': 'application/json;charset=UTF-8',
 };
 
+
+// A middleware function used by all API routes on all platforms
 const commonMiddleware = (handler) => {
+
+  // Create a timeout promise, to throw an error if a request takes too long
+  const createTimeoutPromise = (timeoutMs) => {
+    return new Promise((_, reject) => {
+      setTimeout(() => {
+        const howToResolve = 'You can re-trigger this request, by clicking "Retry"\n'
+        + 'If you\'re running your own instance of Web Check, then you can '
+        + 'resolve this issue, by increasing the timeout limit in the '
+        + '`API_TIMEOUT_LIMIT` environmental variable to a higher value (in milliseconds). \n\n'
+        + `The public instance currently has a lower timeout of ${timeoutMs}ms `
+        + 'in order to keep running costs affordable, so that Web Check can '
+        + 'remain freely available for everyone.';
+        reject(new Error(`Request timed-out after ${timeoutMs} ms.\n\n${howToResolve}`));
+      }, timeoutMs);
+    });
+  };
+
   // Vercel
   const vercelHandler = async (request, response) => {
     const queryParams = request.query || {};
@@ -21,7 +54,12 @@ const commonMiddleware = (handler) => {
     const url = normalizeUrl(rawUrl);
 
     try {
-      const handlerResponse = await handler(url, request);
+      // Race the handler against the timeout
+      const handlerResponse = await Promise.race([
+        handler(url, request),
+        createTimeoutPromise(TIMEOUT)
+      ]);
+
       if (handlerResponse.body && handlerResponse.statusCode) {
         response.status(handlerResponse.statusCode).json(handlerResponse.body);
       } else {
@@ -30,7 +68,11 @@ const commonMiddleware = (handler) => {
         );
       }
     } catch (error) {
-      response.status(500).json({ error: error.message });
+      let errorCode = 500;
+      if (error.message.includes('timed-out')) {
+        errorCode = 408;
+      }
+      response.status(errorCode).json({ error: error.message });
     }
   };
 
@@ -51,7 +93,12 @@ const commonMiddleware = (handler) => {
     const url = normalizeUrl(rawUrl);
 
     try {
-      const handlerResponse = await handler(url, event, context);
+      // Race the handler against the timeout
+      const handlerResponse = await Promise.race([
+        handler(url, event, context),
+        createTimeoutPromise(TIMEOUT)
+      ]);
+
       if (handlerResponse.body && handlerResponse.statusCode) {
         callback(null, handlerResponse);
       } else {
@@ -71,11 +118,7 @@ const commonMiddleware = (handler) => {
   };
 
   // The format of the handlers varies between platforms
-  // E.g. Netlify + AWS expect Lambda functions, but Vercel or Node needs standard handler
-  const platformEnv = (process.env.PLATFORM || '').toUpperCase(); // Has user set platform manually?
-
-  const nativeMode = (['VERCEL', 'NODE'].includes(platformEnv) || process.env.VERCEL || process.env.WC_SERVER);
-
+  const nativeMode = (['VERCEL', 'NODE'].includes(PLATFORM));
   return nativeMode ? vercelHandler : netlifyHandler;
 };
 

api/sitemap.js

@@ -6,15 +6,17 @@ const xml2js = require('xml2js');
 const handler = async (url) => {
   let sitemapUrl = `${url}/sitemap.xml`;
 
+  const hardTimeOut = 5000;
+
   try {
     // Try to fetch sitemap directly
     let sitemapRes;
     try {
-      sitemapRes = await axios.get(sitemapUrl, { timeout: 5000 });
+      sitemapRes = await axios.get(sitemapUrl, { timeout: hardTimeOut });
     } catch (error) {
       if (error.response && error.response.status === 404) {
         // If sitemap not found, try to fetch it from robots.txt
-        const robotsRes = await axios.get(`${url}/robots.txt`, { timeout: 5000 });
+        const robotsRes = await axios.get(`${url}/robots.txt`, { timeout: hardTimeOut });
         const robotsTxt = robotsRes.data.split('\n');
 
         for (let line of robotsTxt) {
@@ -28,7 +30,7 @@ const handler = async (url) => {
           return { skipped: 'No sitemap found' };
         }
 
-        sitemapRes = await axios.get(sitemapUrl, { timeout: 5000 });
+        sitemapRes = await axios.get(sitemapUrl, { timeout: hardTimeOut });
       } else {
         throw error; // If other error, throw it
       }
@@ -40,7 +42,7 @@ const handler = async (url) => {
     return sitemap;
   } catch (error) {
     if (error.code === 'ECONNABORTED') {
-      return { error: 'Request timed out after 5000ms' };
+      return { error: `Request timed-out after ${hardTimeOut}ms` };
     } else {
       return { error: error.message };
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-check",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "private": false,
   "description": "All-in-one OSINT tool for analyzing any website",
   "repository": "github:lissy93/web-check",

src/components/Form/Row.tsx

@@ -159,7 +159,7 @@ export const ExpandableRow = (props: RowProps) => {
           { rowList?.map((row: RowProps, index: number) => {
             return (
               <SubRow key={`${row.lbl}-${index}`}>
-                <span className="lbl" title={row.title}>{row.lbl}</span>
+                <span className="lbl" title={row.title?.toString()}>{row.lbl}</span>
                 <span className="val" title={row.val} onClick={() => copyToClipboard(row.val)}>
                   {formatValue(row.val)}
                 </span>
@@ -199,7 +199,7 @@ const Row = (props: RowProps) => {
   if (children) return <StyledRow key={`${lbl}-${val}`}>{children}</StyledRow>;
   return (
   <StyledRow key={`${lbl}-${val}`}>
-    { lbl && <span className="lbl" title={title}>{lbl}</span> }
+    { lbl && <span className="lbl" title={title?.toString()}>{lbl}</span> }
     <span className="val" title={val} onClick={() => copyToClipboard(val)}>
       {formatValue(val)}
     </span>

src/components/Results/BlockLists.tsx

@@ -6,11 +6,13 @@ const BlockListsCard = (props: {data: any, title: string, actionButtons: any }):
   const blockLists = props.data.blocklists;
   return (
     <Card heading={props.title} actionButtons={props.actionButtons}>
-      { blockLists.map((blocklist: any) => (
+      { blockLists.map((blocklist: any, blockIndex: number) => (
         <Row
           title={blocklist.serverIp}
           lbl={blocklist.server}
-          val={blocklist.isBlocked ? '❌ Blocked' : '✅ Not Blocked'} />
+          val={blocklist.isBlocked ? '❌ Blocked' : '✅ Not Blocked'}
+          key={`blocklist-${blockIndex}-${blocklist.serverIp}`}
+        />
       ))}
     </Card>
   );

src/components/Results/DnsServer.tsx

@@ -18,12 +18,12 @@ const DnsServerCard = (props: {data: any, title: string, actionButtons: any }):
   return (
     <Card heading={props.title} actionButtons={props.actionButtons} styles={cardStyles}>
       {dnsSecurity.dns.map((dns: any, index: number) => {
-        return (<>
+        return (<div key={`dns-${index}`}>
           { dnsSecurity.dns.length > 1 && <Heading as="h4" size="small" color={colors.primary}>DNS Server #{index+1}</Heading> }
           <Row lbl="IP Address" val={dns.address} key={`ip-${index}`} />
           { dns.hostname && <Row lbl="Hostname" val={dns.hostname}  key={`host-${index}`} /> }
           <Row lbl="DoH Support" val={dns.dohDirectSupports ? '✅ Yes*' : '❌ No*'} key={`doh-${index}`} />
-        </>);
+        </div>);
       })}
       {dnsSecurity.dns.length > 0 && (<small>
         * DoH Support is determined by the DNS server's response to a DoH query.

src/components/Results/ServerStatus.tsx

@@ -13,7 +13,7 @@ span.val {
 const ServerStatusCard = (props: { data: any, title: string, actionButtons: any }): JSX.Element => {
   const serverStatus = props.data;
   return (
-    <Card heading={props.title} actionButtons={props.actionButtons} styles={cardStyles}>
+    <Card heading={props.title.toString()} actionButtons={props.actionButtons} styles={cardStyles}>
       <Row lbl="" val="">
         <span className="lbl">Is Up?</span>
         { serverStatus.isUp ? <span className="val up">✅ Online</span> : <span className="val down">❌ Offline</span>}

src/components/Results/TlsCipherSuites.tsx

@@ -52,7 +52,7 @@ const TlsCard = (props: {data: any, title: string, actionButtons: any }): JSX.El
     <Card heading={props.title} actionButtons={props.actionButtons}>
       { cipherSuites.length && cipherSuites.map((cipherSuite: any, index: number) => {
         return (
-          <ExpandableRow lbl={cipherSuite.title} val="" rowList={cipherSuite.fields} />
+          <ExpandableRow key={`tls-${index}`} lbl={cipherSuite.title} val="" rowList={cipherSuite.fields} />
         );
       })}
       { !cipherSuites.length && (

src/components/Results/TlsClientSupport.tsx

@@ -59,8 +59,15 @@ const TlsCard = (props: {data: any, title: string, actionButtons: any }): JSX.El
   const scanId = props.data?.id;
   return (
     <Card heading={props.title} actionButtons={props.actionButtons}>
-      {clientSupport.map((support: any) => {
-        return (<ExpandableRow lbl={support.title} val={support.value || '?'} rowList={support.fields} />)
+      {clientSupport.map((support: any, index: number) => {
+        return (
+        <ExpandableRow
+          key={`tls-client-${index}`}
+          lbl={support.title}
+          val={support.value || '?'}
+          rowList={support.fields}
+        />
+      )
       })}
       { !clientSupport.length && (
         <div>

src/components/Results/TlsIssueAnalysis.tsx

@@ -99,7 +99,13 @@ const TlsCard = (props: {data: any, title: string, actionButtons: any }): JSX.El
     <Card heading={props.title} actionButtons={props.actionButtons}>
       { tlsResults.length > 0 && tlsResults.map((row: any, index: number) => {
         return (
-          <Row lbl={row.lbl} val={row.val} plaintext={row.plaintext} listResults={row.list} />
+          <Row
+            lbl={row.lbl}
+            val={row.val}
+            plaintext={row.plaintext}
+            listResults={row.list}
+            key={`tls-issues-${index}`}
+          />
         );
       })}
       <Expandable>

src/components/misc/ErrorBoundary.tsx

@@ -7,6 +7,7 @@ import colors from 'styles/colors';
 interface Props {
   children: ReactNode;
   title?: string;
+  key?: string;
 }
 
 interface State {

src/components/misc/ProgressBar.tsx

@@ -224,6 +224,52 @@ const jobNames = [
   'carbon',
 ] as const;
 
+interface JobListItemProps {
+  job: LoadingJob;
+  showJobDocs: (name: string) => void;
+  showErrorModal: (name: string, state: LoadingState, timeTaken: number | undefined, error: string, isInfo?: boolean) => void;
+  barColors: Record<LoadingState, [string, string]>;
+}
+
+const getStatusEmoji = (state: LoadingState): string => {
+  switch (state) {
+    case 'success':
+      return '✅';
+    case 'loading':
+      return '🔄';
+    case 'error':
+      return '❌';
+    case 'timed-out':
+      return '⏸️';
+    case 'skipped':
+      return '⏭️';
+    default:
+      return '❓';
+  }
+};
+
+const JobListItem: React.FC<JobListItemProps> = ({ job, showJobDocs, showErrorModal, barColors }) => {
+  const { name, state, timeTaken, retry, error } = job;
+  const actionButton = retry && state !== 'success' && state !== 'loading' ? 
+    <FailedJobActionButton onClick={retry}>↻ Retry</FailedJobActionButton> : null;
+    
+  const showModalButton = error && ['error', 'timed-out', 'skipped'].includes(state) &&
+    <FailedJobActionButton onClick={() => showErrorModal(name, state, timeTaken, error, state === 'skipped')}> 
+      {state === 'timed-out' ? '■ Show Timeout Reason' : '■ Show Error'} 
+    </FailedJobActionButton>;
+
+  return (
+    <li key={name}>
+      <b onClick={() => showJobDocs(name)}>{getStatusEmoji(state)} {name}</b>
+      <span style={{color: barColors[state][0]}}> ({state})</span>.
+      <i>{timeTaken && state !== 'loading' ? ` Took ${timeTaken} ms` : ''}</i>
+      {actionButton}
+      {showModalButton}
+    </li>
+  );
+};
+
+
 export const initialJobs = jobNames.map((job: string) => {
   return {
     name: job,
@@ -239,9 +285,9 @@ export const calculateLoadingStatePercentages = (loadingJobs: LoadingJob[]): Rec
   const stateCount: Record<LoadingState, number> = {
     'success': 0,
     'loading': 0,
-    'skipped': 0,
-    'error': 0,
     'timed-out': 0,
+    'error': 0,
+    'skipped': 0,
   };
 
   // Count the number of each state
@@ -253,9 +299,9 @@ export const calculateLoadingStatePercentages = (loadingJobs: LoadingJob[]): Rec
   const statePercentage: Record<LoadingState, number> = {
     'success': (stateCount['success'] / totalJobs) * 100,
     'loading': (stateCount['loading'] / totalJobs) * 100,
-    'skipped': (stateCount['skipped'] / totalJobs) * 100,
-    'error': (stateCount['error'] / totalJobs) * 100,
     'timed-out': (stateCount['timed-out'] / totalJobs) * 100,
+    'error': (stateCount['error'] / totalJobs) * 100,
+    'skipped': (stateCount['skipped'] / totalJobs) * 100,
   };
 
   return statePercentage;
@@ -353,26 +399,9 @@ const ProgressLoader = (props: { loadStatus: LoadingJob[], showModal: (err: Reac
   const barColors: Record<LoadingState | string, [string, string]> = {
     'success': isDone ? makeBarColor(colors.primary) : makeBarColor(colors.success),
     'loading': makeBarColor(colors.info),
-    'skipped': makeBarColor(colors.warning),
     'error': makeBarColor(colors.danger),
-    'timed-out': makeBarColor(colors.neutral),
-  };
-
-  const getStatusEmoji = (state: LoadingState): string => {
-    switch (state) {
-      case 'success':
-        return '✅';
-      case 'loading':
-        return '🔄';
-      case 'skipped':
-        return '⏭️';
-      case 'error':
-        return '❌';
-      case 'timed-out':
-        return '⏸️';
-      default:
-        return '❓';
-    }
+    'timed-out': makeBarColor(colors.warning),
+    'skipped': makeBarColor(colors.neutral),
   };
 
   const showErrorModal = (name: string, state: LoadingState, timeTaken: number | undefined, error: string, isInfo?: boolean) => {
@@ -416,20 +445,9 @@ const ProgressLoader = (props: { loadStatus: LoadingJob[], showModal: (err: Reac
     <Details>
       <summary>Show Details</summary>
       <ul>
-        {
-          loadStatus.map(({ name, state, timeTaken, retry, error }: LoadingJob) => {
-            return (
-              <li key={name}>
-                <b onClick={() => props.showJobDocs(name)}>{getStatusEmoji(state)} {name}</b>
-                <span style={{color : barColors[state][0]}}> ({state})</span>.
-                <i>{(timeTaken && state !== 'loading') ? ` Took ${timeTaken} ms` : '' }</i>
-                { (retry && state !== 'success' && state !== 'loading') && <FailedJobActionButton onClick={retry}>↻ Retry</FailedJobActionButton> }
-                { (error && state === 'error') && <FailedJobActionButton onClick={() => showErrorModal(name, state, timeTaken, error)}>■ Show Error</FailedJobActionButton> }
-                { (error && state === 'skipped') && <FailedJobActionButton onClick={() => showErrorModal(name, state, timeTaken, error, true)}>■ Show Reason</FailedJobActionButton> }
-              </li>
-            );
-          })
-        }
+        {loadStatus.map((job: LoadingJob) => (
+          <JobListItem key={job.name} job={job} showJobDocs={props.showJobDocs} showErrorModal={showErrorModal} barColors={barColors} />
+        ))}
       </ul>
       { loadStatus.filter((val: LoadingJob) => val.state === 'error').length > 0 &&
         <p className="error">

src/hooks/motherOfAllHooks.ts

@@ -35,16 +35,21 @@ const useMotherOfAllHooks = <ResultType = any>(params: UseIpAddressProps<ResultT
   const [result, setResult] = useState<ResultType>();
 
   // Fire off the HTTP fetch request, then set results and update loading / error state
+
   const doTheFetch = () => {
     return fetchRequest()
     .then((res: any) => {
       if (!res) { // No response :(
-        updateLoadingJobs(jobId, 'error', res.error || 'No response', reset);
+        updateLoadingJobs(jobId, 'error', 'No response', reset);
       } else if (res.error) { // Response returned an error message
-        updateLoadingJobs(jobId, 'error', res.error, reset);
+        if (res.error.includes("timed-out")) { // Specific handling for timeout errors
+          updateLoadingJobs(jobId, 'timed-out', res.error, reset);
+        } else {
+          updateLoadingJobs(jobId, 'error', res.error, reset);
+        }
       } else if (res.skipped) { // Response returned a skipped message
         updateLoadingJobs(jobId, 'skipped', res.skipped, reset);
-      }  else { // Yay, everything went to plan :)
+      } else { // Yay, everything went to plan :)
         setResult(res);
         updateLoadingJobs(jobId, 'success', '', undefined, res);
       }

src/pages/Results.tsx

@@ -208,12 +208,24 @@ const Results = (): JSX.Element => {
         console.log(
           `%cFetch Error - ${job}%c\n\n${timeString}%c The ${job} job failed `
           +`after ${timeTaken}ms, with the following error:%c\n${error}`,
-          `background: ${colors.danger}; padding: 4px 8px; font-size: 16px;`,
+          `background: ${colors.danger}; color:${colors.background}; padding: 4px 8px; font-size: 16px;`,
           `font-weight: bold; color: ${colors.danger};`,
           `color: ${colors.danger};`,
           `color: ${colors.warning};`,
         );
       }
+
+      if (newState === 'timed-out') {
+        console.log(
+          `%cFetch Timeout - ${job}%c\n\n${timeString}%c The ${job} job timed out `
+          +`after ${timeTaken}ms, with the following error:%c\n${error}`,
+          `background: ${colors.info}; color:${colors.background}; padding: 4px 8px; font-size: 16px;`,
+          `font-weight: bold; color: ${colors.info};`,
+          `color: ${colors.info};`,
+          `color: ${colors.warning};`,
+        );
+      }
+
       return newJobs;
     });
   });
@@ -225,8 +237,9 @@ const Results = (): JSX.Element => {
           .then(data => resolve(data))
           .catch(error => resolve(
             { error: `Failed to get a valid response 😢\n`
-            + `This is likely due the target not exposing the required data, `
-            + `or limitations in how Netlify executes lambda functions, such as the 10-sec timeout.\n\n`
+            + 'This is likely due the target not exposing the required data, '
+            + 'or limitations in imposed by the infrastructure this instance '
+            + 'of Web Check is running on.\n\n'
             + `Error info:\n${error}`}
           ));
     });
@@ -910,7 +923,7 @@ const Results = (): JSX.Element => {
               && title.toLowerCase().includes(searchTerm.toLowerCase())
               && (result && !result.error);
               return show ? (
-                <ErrorBoundary title={title}>
+                <ErrorBoundary title={title} key={`eb-${index}`}>
                   <Component
                     key={`${title}-${index}`}
                     data={{...result}}