Writes get robots, refactors row to be reusable, adds loading progress logic

netlify.toml

@@ -59,6 +59,16 @@
   to = "/.netlify/functions/get-cookies"
   status = 301
   force = true
+[[redirects]]
+  from = "/get-dns"
+  to = "/.netlify/functions/get-dns"
+  status = 301
+  force = true
+[[redirects]]
+  from = "/read-robots-txt"
+  to = "/.netlify/functions/read-robots-txt"
+  status = 301
+  force = true
   
 # For router history mode, ensure pages land on index
 [[redirects]]

src/components/Form/Heading.tsx

@@ -17,10 +17,14 @@ const StyledHeading = styled.h1<HeadingProps>`
   display: flex;
   gap: 1rem;
   align-items: center;
-  img {
+  img { // Some titles have an icon
     width: 2rem;
     border-radius: 4px;
   }
+  a { // If a title is a link, keep title styles
+    color: inherit;
+    text-decoration: none;
+  }
   ${props => {
     switch (props.size) {
       case 'xSmall': return `font-size: ${TextSizes.small};`;

src/components/Form/Row.tsx

@@ -1,21 +1,23 @@
 import styled from 'styled-components';
 import colors from 'styles/colors';
 
-interface RowProps {
+export interface RowProps {
   lbl: string,
   val: string,
   key?: string,
+  children?: JSX.Element[],
   rowList?: RowProps[],
+  title?: string,
 }
 
-const StyledRow = styled.div`
+export const StyledRow = styled.div`
   display: flex;
   justify-content: space-between;
   padding: 0.25rem;
   &:not(:last-child) { border-bottom: 1px solid ${colors.primary}; }
   span.lbl { font-weight: bold; }
   span.val {
-    max-width: 200px;
+    max-width: 16rem;
     white-space: nowrap;
     overflow: hidden;
     text-overflow: ellipsis;
@@ -64,7 +66,6 @@ const formatDate = (dateString: string): string => {
 const formatValue = (value: any): string => {
   const isValidDate = (date: any) => date instanceof Date && !isNaN(date as any as number);
   if (isValidDate(new Date(value))) return formatDate(value);
-  if (typeof value === 'object') return JSON.stringify(value);
   if (typeof value === 'boolean') return value ? '✅' : '❌';
   return value;
 };
@@ -74,11 +75,11 @@ const copyToClipboard = (text: string) => {
 }
 
 export const ExpandableRow = (props: RowProps) => {
-  const { lbl, val, rowList } = props;
+  const { lbl, val, key, title, rowList } = props;
   return (
     <Details>
-      <StyledExpandableRow>
+      <StyledExpandableRow key={key}>
-        <span className="lbl">{lbl}</span>
+        <span className="lbl" title={title}>{lbl}</span>
         <span className="val" title={val}>{val}</span>
       </StyledExpandableRow>
       { rowList &&
@@ -86,7 +87,7 @@ export const ExpandableRow = (props: RowProps) => {
           { rowList?.map((row: RowProps, index: number) => {
             return (
               <SubRow key={row.key || `${row.lbl}-${index}`}>
-                <span className="lbl">{row.lbl}</span>
+                <span className="lbl" title={row.title}>{row.lbl}</span>
                 <span className="val" title={row.val} onClick={() => copyToClipboard(row.val)}>
                   {formatValue(row.val)}
                 </span>
@@ -100,10 +101,11 @@ export const ExpandableRow = (props: RowProps) => {
 };
 
 const Row = (props: RowProps) => {
-  const { lbl, val, key } = props;
+  const { lbl, val, key, title, children } = props;
+  if (children) return <StyledRow key={key}>{children}</StyledRow>;
   return (
   <StyledRow key={key}>
-    <span className="lbl">{lbl}</span>
+    <span className="lbl" title={title}>{lbl}</span>
     <span className="val" title={val} onClick={() => copyToClipboard(val)}>
       {formatValue(val)}
     </span>

src/components/Results/Cookies.tsx

@@ -3,17 +3,18 @@ import styled from 'styled-components';
 import colors from 'styles/colors';
 import Card from 'components/Form/Card';
 import Heading from 'components/Form/Heading';
-import Row, { ExpandableRow } from 'components/Form/Row';
+import { ExpandableRow } from 'components/Form/Row';
 
 const Outer = styled(Card)``;
 
 const CookiesCard = (props: { cookies: any }): JSX.Element => {
   const cookies = props.cookies;
-  console.log('COOKIES: ', cookies);
   return (
     <Outer>
       <Heading as="h3" size="small" align="left" color={colors.primary}>Cookies</Heading>
-      {/* { subject && <DataRow lbl="Subject" val={subject?.CN} /> } */}
+      {
+        cookies.length === 0 && <p>No cookies found.</p>
+      }
       {
         cookies.map((cookie: any, index: number) => {
           const attributes = Object.keys(cookie.attributes).map((key: string) => {

src/components/Results/Headers.tsx

@@ -3,35 +3,12 @@ import styled from 'styled-components';
 import colors from 'styles/colors';
 import Card from 'components/Form/Card';
 import Heading from 'components/Form/Heading';
+import Row from 'components/Form/Row';
 
 const Outer = styled(Card)`
   grid-row: span 2;
 `;
 
-const Row = styled.div`
-  display: flex;
-  justify-content: space-between;
-  padding: 0.25rem;
-  &:not(:last-child) { border-bottom: 1px solid ${colors.primary}; }
-  span.lbl { font-weight: bold; }
-  span.val {
-    max-width: 200px;
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-  }
-`;
-
-const DataRow = (props: { lbl: string, val: string }) => {
-  const { lbl, val } = props;
-  return (
-  <Row>
-    <span className="lbl">{lbl}</span>
-    <span className="val" title={val}>{val}</span>
-  </Row>
-  );
-};
-
 const HeadersCard = (props: { headers: any }): JSX.Element => {
   const headers = props.headers;
   return (
@@ -40,7 +17,7 @@ const HeadersCard = (props: { headers: any }): JSX.Element => {
       {
         Object.keys(headers).map((header: string, index: number) => {
           return (
-            <DataRow key={`header-${index}`} lbl={header} val={headers[header]} />
+            <Row key={`header-${index}`} lbl={header} val={headers[header]} />
           )
         })
       }      

src/components/Results/Lighthouse.tsx

@@ -4,6 +4,7 @@ import styled from 'styled-components';
 import colors from 'styles/colors';
 import Card from 'components/Form/Card';
 import Heading from 'components/Form/Heading';
+import { ExpandableRow } from 'components/Form/Row';
 
 const processScore = (percentile: number) => {
   return `${Math.round(percentile * 100)}%`;
@@ -78,23 +79,14 @@ const ScoreItem = (props: { scoreId: any, audits: Audit[] }) => {
   );
 };
 
-const ExpandableRow = (props: { lbl: string, val: string, list: string[], audits: Audit[] }) => {
+const makeValue = (audit: Audit) => {
-  const { lbl, val, list, audits } = props;
+  let score = audit.score;
-  return (
+  if (audit.displayValue) {
-  <Row>
+    score = audit.displayValue;
-    <details>
+  } else if (audit.scoreDisplayMode) {
-      <summary>
+    score = audit.score === 1 ? '✅ Pass' : '❌ Fail'; 
-        <span className="lbl">{lbl}</span>
+  }
-        <span className="val" title={val}>{val}</span>
+  return score;
-      </summary>
-      <ScoreList>
-        { list.map((li: string) => {
-          return <ScoreItem scoreId={li} audits={audits} />
-        }) }
-      </ScoreList>
-    </details>  
-  </Row>
-  );
 };
 
 const LighthouseCard = (props: { lighthouse: any }): JSX.Element => {
@@ -106,13 +98,15 @@ const LighthouseCard = (props: { lighthouse: any }): JSX.Element => {
       <Heading as="h3" size="small" align="left" color={colors.primary}>Performance</Heading>
       { Object.keys(categories).map((title: string, index: number) => {
         const scoreIds = categories[title].auditRefs.map((ref: { id: string }) => ref.id);
+        const scoreList = scoreIds.map((id: string) => {
+          return { lbl: audits[id].title, val: makeValue(audits[id]), title: audits[id].description, key: id }
+        })
         return (
           <ExpandableRow
             key={`lighthouse-${index}`}
             lbl={title}
             val={processScore(categories[title].score)}
-            list={scoreIds}
+            rowList={scoreList}
-            audits={audits}
           />
         );
       }) }

src/components/Results/RobotsTxt.tsx

@@ -0,0 +1,35 @@
+
+import styled from 'styled-components';
+import colors from 'styles/colors';
+import Card from 'components/Form/Card';
+import Heading from 'components/Form/Heading';
+import Row, { RowProps }  from 'components/Form/Row';
+
+const Outer = styled(Card)`
+  .content {
+    max-height: 20rem;
+    overflow-y: auto;
+  }
+`;
+
+const RobotsTxtCard = (props: { robotTxt: RowProps[] }): JSX.Element => {
+  return (
+    <Outer>
+      <Heading as="h3" size="small" align="left" color={colors.primary}>Crawl Rules</Heading>
+      <div className="content">
+      {
+        props.robotTxt.length === 0 && <p>No crawl rules found.</p>
+      }
+      {
+        props.robotTxt.map((row: RowProps, index: number) => {
+          return (
+            <Row key={row.key || `${row.lbl}-${index}`} lbl={row.lbl} val={row.val} />
+          )
+        })
+      }
+      </div>
+    </Outer>
+  );
+}
+
+export default RobotsTxtCard;

src/components/Results/Screenshot.tsx

@@ -6,7 +6,11 @@ import Heading from 'components/Form/Heading';
 
 const Outer = styled(Card)`
 overflow: auto;
-max-height: 20rem;
+max-height: 28rem;
+img {
+  border-radius: 6px;
+  width: 100%;
+}
 `;
 
 const ScreenshotCard = (props: { screenshot: string }): JSX.Element => {

src/components/Results/ServerLocation.tsx

@@ -7,27 +7,12 @@ import Heading from 'components/Form/Heading';
 import LocationMap from 'components/misc/LocationMap';
 import Flag from 'components/misc/Flag';
 import { TextSizes } from 'styles/typography';
+import Row, { StyledRow } from 'components/Form/Row';
 
 const Outer = styled(Card)`
   grid-row: span 2
 `;
 
-const Row = styled.div`
-  display: flex;
-  justify-content: space-between;
-  padding: 0.25rem;
-  &:not(:last-child) { border-bottom: 1px solid ${colors.primary}; }
-`;
-
-const RowLabel = styled.span`
-  font-weight: bold;
-`;
-
-const RowValue = styled.span`
-  display: flex;
-  img { margin-left: 0.5rem; }
-`;
-
 const SmallText = styled.span`
   opacity: 0.5;
   font-size: ${TextSizes.xSmall};
@@ -35,10 +20,16 @@ const SmallText = styled.span`
   display: block;
 `;
 
-const MapRow = styled(Row)`
+const MapRow = styled(StyledRow)`
+  padding-top: 1rem;
   flex-direction: column;
 `;
 
+const CountryValue = styled.span`
+  display: flex;
+  gap: 0.5rem;
+`;
+
 const ServerLocationCard = (location: ServerLocation): JSX.Element => {
 
   const {
@@ -50,28 +41,17 @@ const ServerLocationCard = (location: ServerLocation): JSX.Element => {
   return (
     <Outer>
       <Heading as="h3" size="small" align="left" color={colors.primary}>Location</Heading>
-      <Row>
+      <Row lbl="City" val={`${postCode}, ${city}, ${region}`} />
-        <RowLabel>City</RowLabel>
+      <Row lbl="" val="">
-        <RowValue>
+        <b>Country</b>
-          {postCode}, { city }, { region }
+        <CountryValue>
-        </RowValue>
-      </Row>
-      <Row>
-        <RowLabel>Country</RowLabel>
-        <RowValue>
           {country}
           { countryCode && <Flag countryCode={countryCode} width={28} /> }
-        </RowValue>
+        </CountryValue>
       </Row>
-      { timezone && <Row>
+      <Row lbl="Timezone" val={timezone} />
-        <RowLabel>Timezone</RowLabel><RowValue>{timezone}</RowValue>
+      <Row lbl="Languages" val={languages} />
-      </Row>}
+      <Row lbl="Currency" val={`${currency} (${currencyCode})`} />
-      { languages && <Row>
-        <RowLabel>Languages</RowLabel><RowValue>{languages}</RowValue>
-      </Row>}
-      { currency && <Row>
-        <RowLabel>Currency</RowLabel><RowValue>{currency} ({currencyCode})</RowValue>
-      </Row>}
       <MapRow>
         <LocationMap lat={coords.latitude} lon={coords.longitude} label={`Server (${isp})`} />
         <SmallText>Latitude: {coords.latitude}, Longitude: {coords.longitude} </SmallText>

src/pages/Home.tsx

@@ -81,7 +81,6 @@ const Home = (): JSX.Element => {
     fetch('https://ipapi.co/json/')
       .then(function(response) {
         response.json().then(jsonData => {
-          console.log(jsonData);
           setUserInput(jsonData.ip);
           setPlaceholder(defaultPlaceholder);
           setInputDisabled(true);

src/pages/Results.tsx

@@ -13,6 +13,9 @@ import BuiltWithCard from 'components/Results/BuiltWith';
 import LighthouseCard from 'components/Results/Lighthouse';
 import ScreenshotCard from 'components/Results/Screenshot';
 import SslCertCard from 'components/Results/SslCert';
+import HeadersCard from 'components/Results/Headers';
+import CookiesCard from 'components/Results/Cookies';
+import RobotsTxtCard from 'components/Results/RobotsTxt';
 import keys from 'utils/get-keys';
 import { determineAddressType, AddressType } from 'utils/address-type-checker';
 
@@ -21,6 +24,8 @@ import {
   getServerInfo, ServerInfo,
   getHostNames, HostNames,
   makeTechnologies, TechnologyGroup,
+  parseCookies, Cookie,
+  parseRobotsTxt,
   Whois,
 } from 'utils/result-processor';
 
@@ -49,23 +54,68 @@ const Header = styled(Card)`
   padding: 0.5rem 1rem;
 `;
 
-interface ResultsType {
-  serverLocation?: ServerLocation,
-  serverInfo?: ServerInfo,
-  hostNames?: HostNames | null,
-};
-
 const Results = (): JSX.Element => {
-  const [ results, setResults ] = useState<ResultsType>({});
+  const [ serverInfo, setServerInfo ] = useState<ServerInfo>();
+  const [ hostNames, setHostNames ] = useState<HostNames | null>();
   const [ locationResults, setLocationResults ] = useState<ServerLocation>();
   const [ whoIsResults, setWhoIsResults ] = useState<Whois>();
   const [ technologyResults, setTechnologyResults ] = useState<TechnologyGroup[]>();
   const [ lighthouseResults, setLighthouseResults ] = useState<any>();
   const [ sslResults, setSslResults ] = useState<any>();
+  const [ headersResults, setHeadersResults ] = useState<any>();
+  const [ robotsTxtResults, setRobotsTxtResults ] = useState<any>();
+  const [ cookieResults, setCookieResults ] = useState<Cookie[] | null>(null);
   const [ screenshotResult, setScreenshotResult ] = useState<string>();
   const [ ipAddress, setIpAddress ] = useState<undefined | string>(undefined);
   const [ addressType, setAddressType ] = useState<AddressType>('empt');
   const { address } = useParams();
+
+  type LoadingState = 'loading' | 'skipped' | 'success' | 'error';
+
+  interface LoadingJob {
+    name: string,
+    state: LoadingState,
+    error?: string,
+  }
+
+  const jobNames = [
+    'get-ip',
+    'ssl',
+    'cookies',
+    'robots-txt',
+    'headers',
+    'lighthouse',
+    'location',
+    'server-info',
+    'whois',
+  ] as const;
+
+  const initialJobs = jobNames.map((job: string) => {
+    return {
+      name: job,
+      state: 'loading' as LoadingState,
+    }
+  });
+
+  const [ loadingJobs, setLoadingJobs ] = useState<LoadingJob[]>(initialJobs);
+
+  const updateLoadingJobs = (job: string, newState: LoadingState, error?: string) => {
+    setLoadingJobs((prevJobs) => {
+      const newJobs = prevJobs.map((loadingJob: LoadingJob) => {
+        if (loadingJob.name === job) {
+          return { ...loadingJob, error, state: newState };
+        }
+        return loadingJob;
+      });
+  
+      if (newState === 'error') {
+        console.warn(`Error in ${job}: ${error}`);
+      }
+  
+      return newJobs;
+    });
+  };
+  
   
 
   useEffect(() => {
@@ -77,43 +127,102 @@ const Results = (): JSX.Element => {
 
   /* Get IP address from URL */
   useEffect(() => {
+    if (addressType !== 'url') return;
     const fetchIpAddress = () => {
       fetch(`/find-url-ip?address=${address}`)
       .then(function(response) {
         response.json().then(jsonData => {
           setIpAddress(jsonData.ip);
+          updateLoadingJobs('get-ip', 'success');
         });
       })
       .catch(function(error) {
-        console.log(error)
+        updateLoadingJobs('get-ip', 'error', error);
       });
     };
     if (!ipAddress) {
       fetchIpAddress();
     }
-  }, [address]);
+  }, [address, addressType]);
 
   /* Get SSL info */
   useEffect(() => {
+    if (addressType !== 'url') return;
     fetch(`/ssl-check?url=${address}`)
       .then(response => response.json())
       .then(response => {
-        console.log(response);
+        if (Object.keys(response).length > 0) {
-        setSslResults(response);
+          setSslResults(response);
+          updateLoadingJobs('ssl', 'success');
+        } else {
+          updateLoadingJobs('ssl', 'error', 'No SSL Cert found');
+        }
       })
-      .catch(err => console.error(err));
+      .catch(err => updateLoadingJobs('ssl', 'error', err));
-  }, [address])
+  }, [address, addressType])
+
+  /* Get Cookies */
+  useEffect(() => {
+    if (addressType !== 'url') return;
+    fetch(`/get-cookies?url=${address}`)
+      .then(response => response.json())
+      .then(response => {
+        setCookieResults(parseCookies(response.cookies));
+        updateLoadingJobs('cookies', 'success');
+      })
+      .catch(err => updateLoadingJobs('cookies', 'error', err));
+  }, [address, addressType])
+
+  /* Get Robots.txt */
+  useEffect(() => {
+    if (addressType !== 'url') return;
+    fetch(`/read-robots-txt?url=${address}`)
+      .then(response => response.text())
+      .then(response => {
+        setRobotsTxtResults(parseRobotsTxt(response));
+        updateLoadingJobs('robots-txt', 'success');
+      })
+      .catch(err => updateLoadingJobs('robots-txt', 'error', err));
+  }, [address, addressType])
+
+  /* Get Headers */
+  useEffect(() => {
+    if (addressType !== 'url') return;
+    fetch(`/get-headers?url=${address}`)
+      .then(response => response.json())
+      .then(response => {
+        setHeadersResults(response);
+        updateLoadingJobs('headers', 'success');
+      })
+      .catch(err => updateLoadingJobs('headers', 'error', err));
+  }, [address, addressType])
 
   /* Get Lighthouse report */
   useEffect(() => {
+    if (addressType !== 'url') return;
     fetch(`/lighthouse-report?url=${address}`)
       .then(response => response.json())
       .then(response => {
         setLighthouseResults(response.lighthouseResult);
         setScreenshotResult(response.lighthouseResult?.fullPageScreenshot?.screenshot?.data);
+        updateLoadingJobs('lighthouse', 'success');
       })
-      .catch(err => console.error(err));
+      .catch(err => {
-  }, [address])
+        // if (err.errorType === 'TimeoutError') {
+        // Netlify limits to 10 seconds, we can try again client-side...
+        const params = 'category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile';
+        const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${address}&${params}&key=${keys.googleCloud}`;
+        fetch(endpoint)
+          .then(response => response.json())
+          .then(response => {
+            setLighthouseResults(response.lightHouseResult);
+            setScreenshotResult(response?.lighthouseResult?.fullPageScreenshot?.screenshot?.data);
+            updateLoadingJobs('lighthouse', 'success');
+          })
+          .catch(err => updateLoadingJobs('lighthouse', 'error', err));
+      });
+  }, [address, addressType])
+
 
   /* Get IP address location info */
   useEffect(() => {
@@ -122,10 +231,11 @@ const Results = (): JSX.Element => {
       .then(function(response) {
         response.json().then(jsonData => {
           setLocationResults(getLocation(jsonData));
+          updateLoadingJobs('location', 'success');
         });
       })
       .catch(function(error) {
-        console.log(error)
+        updateLoadingJobs('location', 'error', error);
       });
     };
     if (ipAddress) {
@@ -136,18 +246,20 @@ const Results = (): JSX.Element => {
   /* Get hostnames and server info from Shodan */
   useEffect(() => {
     const applyShodanResults = (response: any) => {
-      const serverInfo = getServerInfo(response);
+      setServerInfo(getServerInfo(response));
-      const hostNames = getHostNames(response);
+      setHostNames(getHostNames(response));
-      setResults({...results, serverInfo, hostNames });
     }
     const fetchShodanData = () => {
       const apiKey = keys.shodan;
       fetch(`https://api.shodan.io/shodan/host/${ipAddress}?key=${apiKey}`)
         .then(response => response.json())
         .then(response => {
-          if (!response.error) applyShodanResults(response)
+          if (!response.error) {
+            applyShodanResults(response)
+            updateLoadingJobs('server-info', 'success');
+          }
         })
-        .catch(err => console.error(err));
+        .catch(err => updateLoadingJobs('server-info', 'error', err));
     };
     
 
@@ -158,17 +270,21 @@ const Results = (): JSX.Element => {
 
   /* Get BuiltWith tech stack */
   useEffect(() => {
+    if (addressType !== 'url') return;
     const apiKey = keys.builtWith;
     const endpoint = `https://api.builtwith.com/v21/api.json?KEY=${apiKey}&LOOKUP=${address}`;
     fetch(endpoint)
       .then(response => response.json())
       .then(response => {
         setTechnologyResults(makeTechnologies(response));
-      });
+        updateLoadingJobs('built-with', 'success');
-  }, [address]);
+      })
+      .catch(err => updateLoadingJobs('built-with', 'error', err));
+  }, [address, addressType]);
   
   /* Get WhoIs info for a given domain name */
   useEffect(() => {
+    if (addressType !== 'url') return;
     const applyWhoIsResults = (response: any) => {
       const whoIsResults: Whois = {
         created: response.date_created,
@@ -184,8 +300,9 @@ const Results = (): JSX.Element => {
         .then(response => response.json())
         .then(response => {
           if (!response.error) applyWhoIsResults(response)
+          updateLoadingJobs('whois', 'success');
         })
-        .catch(err => console.error(err));
+        .catch(err => updateLoadingJobs('whois', 'error', err));
     };
     
     if (addressType === 'url') {
@@ -193,24 +310,40 @@ const Results = (): JSX.Element => {
     }
   }, [addressType, address]);
 
+  const makeSiteName = (address: string): string => {
+    try {
+      return new URL(address).hostname.replace('www.', '');
+    } catch (error) {
+      return address;
+    }
+  }
+  
   return (
     <ResultsOuter>
       <Header as="header">
-        <Heading color={colors.primary} size="large">Results</Heading>
+        <Heading color={colors.primary} size="large">
-        <Heading color={colors.textColor} size="medium">
+          <a href="/">Web Check</a>
-          { address && <img width="32px" src={`https://icon.horse/icon/${new URL(address).hostname}`} alt="" /> }
-          {address}
         </Heading>
+        { address && 
+        <Heading color={colors.textColor} size="medium">
+          { addressType === 'url' && <img width="32px" src={`https://icon.horse/icon/${makeSiteName(address)}`} alt="" /> }
+          {makeSiteName(address)}
+        </Heading>
+        }
       </Header>
       <ResultsContent>
         { locationResults && <ServerLocationCard {...locationResults} />}
-        { results.serverInfo && <ServerInfoCard {...results.serverInfo} />}
+        { sslResults && <SslCertCard sslCert={sslResults} />}
-        { results?.hostNames && <HostNamesCard hosts={results?.hostNames} />}
+        { headersResults && <HeadersCard headers={headersResults} />}
+        { hostNames && <HostNamesCard hosts={hostNames} />}
         { whoIsResults && <WhoIsCard {...whoIsResults} />}
         { lighthouseResults && <LighthouseCard lighthouse={lighthouseResults} />}
+        { cookieResults && <CookiesCard cookies={cookieResults} />}
         { screenshotResult && <ScreenshotCard screenshot={screenshotResult} />}
         { technologyResults && <BuiltWithCard technologies={technologyResults} />}
-        { sslResults && <SslCertCard sslCert={sslResults} />}
+        { robotsTxtResults && <RobotsTxtCard robotTxt={robotsTxtResults} />}
+        { serverInfo && <ServerInfoCard {...serverInfo} />}
+        
       </ResultsContent>
     </ResultsOuter>
   );

src/utils/address-type-checker.ts

@@ -7,17 +7,29 @@ export type AddressType = 'ipV4' | 'ipV6' | 'url' | 'err' | 'empt';
 
 /* Checks if a given string looks like a URL */
 const isUrl = (value: string):boolean => {
-  const urlRegex= new RegExp(''
+  const urlPattern = new RegExp(
-    // + /(?:(?:(https?|ftp):)?\/\/)/.source
+    '^(https?:\\/\\/)?' + 
-    + /(?:([^:\n\r]+):([^@\n\r]+)@)?/.source
+    '(?!([0-9]{1,3}\\.){3}[0-9]{1,3})' + // Exclude IP addresses
-    + /(?:(?:www\.)?([^/\n\r]+))/.source
+    '(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*' + // Domain name or a subdomain
-    + /(\/[^?\n\r]+)?/.source
+    '([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\-]*[A-Za-z0-9])$', // Second level domain
-    + /(\?[^#\n\r]*)?/.source
+    'i' // Case-insensitive
-    + /(#?[^\n\r]*)?/.source
   );
-  return urlRegex.test(value);
+  return urlPattern.test(value);
 };
 
+// /* Checks if a given string looks like a URL */
+// const isUrl = (value: string):boolean => {
+//   const urlRegex= new RegExp(''
+//     // + /(?:(?:(https?|ftp):)?\/\/)/.source
+//     + /(?:([^:\n\r]+):([^@\n\r]+)@)?/.source
+//     + /(?:(?:www\.)?([^/\n\r]+))/.source
+//     + /(\/[^?\n\r]+)?/.source
+//     + /(\?[^#\n\r]*)?/.source
+//     + /(#?[^\n\r]*)?/.source
+//   );
+//   return urlRegex.test(value);
+// };
+
 /* Checks if a given string looks like an IP Version 4 Address */
 const isIpV4 = (value: string): boolean => {
   const ipPart = '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)';
@@ -51,7 +63,8 @@ const isShort = (value: string): boolean => {
 };
 
 /* Returns the address type for a given address */
-export const determineAddressType = (address: string): AddressType => {
+export const determineAddressType = (address: string | undefined): AddressType => {
+  if (!address) return 'err';
   if (isShort(address)) return 'empt';
   if (isUrl(address)) return 'url';
   if (isIpV4(address)) return 'ipV4';

src/utils/get-keys.ts

@@ -3,6 +3,7 @@ const keys = {
   shodan: process.env.REACT_APP_SHODAN_API_KEY,
   whoApi: process.env.REACT_APP_WHO_API_KEY,
   builtWith: process.env.REACT_APP_BUILT_WITH_API_KEY,
+  googleCloud: process.env.REACT_APP_GOOGLE_CLOUD_API_KEY,
 };
 
 export default keys;

src/utils/result-processor.ts

@@ -57,6 +57,8 @@ export interface ServerInfo {
   os?: string,
   ip?: string,
   ports?: string,
+  loc?: string,
+  type?: string,
 };
 
 export const getServerInfo = (response: any): ServerInfo => {
@@ -67,6 +69,8 @@ export const getServerInfo = (response: any): ServerInfo => {
     os: response.os,
     ip: response.ip_str,
     ports: response.ports.toString(),
+    loc: response.city ? `${response.city}, ${response.country_name}` : '',
+    type: response.tags ? response.tags.toString() : '',
   };
 };
 
@@ -115,3 +119,45 @@ export const makeTechnologies = (response: any): TechnologyGroup[] => {
   }, {});
   return technologies;
 };
+
+export type Cookie = {
+  name: string;
+  value: string;
+  attributes: Record<string, string>;
+};
+
+export const parseCookies = (cookiesHeader: string): Cookie[] => {
+  if (!cookiesHeader) return [];
+  return cookiesHeader.split(/,(?=\s[A-Za-z0-9]+=)/).map(cookieString => {
+    const [nameValuePair, ...attributePairs] = cookieString.split('; ').map(part => part.trim());
+    const [name, value] = nameValuePair.split('=');
+    const attributes: Record<string, string> = {};
+    attributePairs.forEach(pair => {
+      const [attributeName, attributeValue = ''] = pair.split('=');
+      attributes[attributeName] = attributeValue;
+    });
+    return { name, value, attributes };
+  });
+}
+
+type RobotsRule = {
+  lbl: string;
+  val: string;
+};
+
+export const parseRobotsTxt = (content: string): RobotsRule[] => {
+  const lines = content.split('\n');
+  const rules: RobotsRule[] = [];
+  lines.forEach(line => {
+    const match = line.match(/^(Allow|Disallow):\s*(\S*)$/);
+    if (match) {
+      const rule: RobotsRule = {
+        lbl: match[1],
+        val: match[2],
+      };
+      
+      rules.push(rule);
+    }
+  });
+  return rules;
+}