Implement middleware into newer lambdas

2025-05-13 10:44:36 +02:00 · 2023-08-10 14:26:36 +01:00 · 2023-08-10 14:26:36 +01:00 · f96c7ba25f
commit f96c7ba25f
parent 976ca7d47a 8624237760
21 changed files with 604 additions and 173 deletions
--- a/.github/README.md
+++ b/.github/README.md
@ -42,7 +42,7 @@ None of this is hard to find with a series of basic curl commands, or a combinat
 <details>
 <summary><b>IP Address</b></summary>

-<img width="300" src="undefined?" align="right" />
+<img width="300" src="./screenshots/wc_ip-adress.png" align="right" />

 ###### Description
 The IP Address task involves mapping the user provided URL to its corresponding IP address through a process known as Domain Name System (DNS) resolution. An IP address is a unique identifier given to every device on the Internet, and when paired with a domain name, it allows for accurate routing of online requests and responses.
--- a/.github/screenshots/wc_ip-adress.png
+++ b/.github/screenshots/wc_ip-adress.png
--- a/api/content-links.js
+++ b/api/content-links.js
@ -29,6 +29,19 @@ const handler = async (url) => {
  const internalLinks = [...internalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
  const externalLinks = [...externalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);

+  // If there were no links, then mark as skipped and show reasons
+  if (internalLinks.length === 0 && externalLinks.length === 0) {
+    return {
+      statusCode: 400,
+      body: JSON.stringify({
+        skipped: 'No internal or external links found. '
+          + 'This may be due to the website being dynamically rendered, using a client-side framework (like React), and without SSR enabled. '
+          + 'That would mean that the static HTML returned from the HTTP request doesn\'t contain any meaningful content for Web-Check to analyze. '
+          + 'You can rectify this by using a headless browser to render the page instead.',
+        }),
+    };
+  }
+
  return { internal: internalLinks, external: externalLinks };
 };

--- a/api/get-carbon.js
+++ b/api/get-carbon.js
@ -34,6 +34,13 @@ const handler = async (url) => {
      }).on('error', reject);
    });

+    if (!carbonData.statistics || (carbonData.statistics.adjustedBytes === 0 && carbonData.statistics.energy === 0)) {
+      return {
+        statusCode: 200,
+        body: JSON.stringify({ skipped: 'Not enough info to get carbon data' }),
+      };
+    }
+
    carbonData.scanUrl = url;
    return carbonData;
  } catch (error) {
--- a/api/mail-config.js
+++ b/api/mail-config.js
@ -0,0 +1,79 @@
+const dns = require('dns').promises;
+const URL = require('url-parse');
+
+exports.handler = async (event, context) => {
+  try {
+    let domain = event.queryStringParameters.url;
+    const parsedUrl = new URL(domain);
+    domain = parsedUrl.hostname || parsedUrl.pathname;
+
+    // Get MX records
+    const mxRecords = await dns.resolveMx(domain);
+
+    // Get TXT records
+    const txtRecords = await dns.resolveTxt(domain);
+
+    // Filter for only email related TXT records (SPF, DKIM, DMARC, and certain provider verifications)
+    const emailTxtRecords = txtRecords.filter(record => {
+      const recordString = record.join('');
+      return (
+        recordString.startsWith('v=spf1') ||
+        recordString.startsWith('v=DKIM1') ||
+        recordString.startsWith('v=DMARC1') ||
+        recordString.startsWith('protonmail-verification=') ||
+        recordString.startsWith('google-site-verification=') || // Google Workspace
+        recordString.startsWith('MS=') || // Microsoft 365
+        recordString.startsWith('zoho-verification=') || // Zoho
+        recordString.startsWith('titan-verification=') || // Titan
+        recordString.includes('bluehost.com') // BlueHost
+      );
+    });
+
+    // Identify specific mail services
+    const mailServices = emailTxtRecords.map(record => {
+      const recordString = record.join('');
+      if (recordString.startsWith('protonmail-verification=')) {
+        return { provider: 'ProtonMail', value: recordString.split('=')[1] };
+      } else if (recordString.startsWith('google-site-verification=')) {
+        return { provider: 'Google Workspace', value: recordString.split('=')[1] };
+      } else if (recordString.startsWith('MS=')) {
+        return { provider: 'Microsoft 365', value: recordString.split('=')[1] };
+      } else if (recordString.startsWith('zoho-verification=')) {
+        return { provider: 'Zoho', value: recordString.split('=')[1] };
+      } else if (recordString.startsWith('titan-verification=')) {
+        return { provider: 'Titan', value: recordString.split('=')[1] };
+      } else if (recordString.includes('bluehost.com')) {
+        return { provider: 'BlueHost', value: recordString };
+      } else {
+        return null;
+      }
+    }).filter(record => record !== null);
+
+    // Check MX records for Yahoo
+    const yahooMx = mxRecords.filter(record => record.exchange.includes('yahoodns.net'));
+    if (yahooMx.length > 0) {
+      mailServices.push({ provider: 'Yahoo', value: yahooMx[0].exchange });
+    }
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify({
+        mxRecords,
+        txtRecords: emailTxtRecords,
+        mailServices,
+      }),
+    };
+  } catch (error) {
+    if (error.code === 'ENOTFOUND' || error.code === 'ENODATA') {
+      return {
+        statusCode: 200,
+        body: JSON.stringify({ skipped: 'No mail server in use on this domain' }),
+      };
+    } else {
+      return {
+        statusCode: 500,
+        body: JSON.stringify({ error: error.message }),
+      };
+    }
+  }
+};
--- a/api/sitemap.js
+++ b/api/sitemap.js
@ -1,33 +1,64 @@
+const commonMiddleware = require('./_common/middleware');
+
 const axios = require('axios');
 const xml2js = require('xml2js');
-const middleware = require('./_common/middleware');

-const fetchSitemapHandler = async (url) => {
-  let sitemapUrl;
+const handler = async (url) => {
+  let sitemapUrl = `${url}/sitemap.xml`;

  try {
-    // Fetch robots.txt
-    const robotsRes = await axios.get(`${url}/robots.txt`);
+    // Try to fetch sitemap directly
+    let sitemapRes;
+    try {
+      sitemapRes = await axios.get(sitemapUrl, { timeout: 5000 });
+    } catch (error) {
+      if (error.response && error.response.status === 404) {
+        // If sitemap not found, try to fetch it from robots.txt
+        const robotsRes = await axios.get(`${url}/robots.txt`, { timeout: 5000 });
        const robotsTxt = robotsRes.data.split('\n');

        for (let line of robotsTxt) {
-      if (line.startsWith('Sitemap:')) {
-        sitemapUrl = line.split(' ')[1];
+          if (line.toLowerCase().startsWith('sitemap:')) {
+            sitemapUrl = line.split(' ')[1].trim();
+            break;
          }
        }

        if (!sitemapUrl) {
-      throw new Error('Sitemap not found in robots.txt');
+          return {
+            statusCode: 404,
+            body: JSON.stringify({ skipped: 'No sitemap found' }),
+          };
        }

-    // Fetch sitemap
-    const sitemapRes = await axios.get(sitemapUrl);
-    const sitemap = await xml2js.parseStringPromise(sitemapRes.data);
+        sitemapRes = await axios.get(sitemapUrl, { timeout: 5000 });
+      } else {
+        throw error; // If other error, throw it
+      }
+    }

-    return sitemap;
+    const parser = new xml2js.Parser();
+    const sitemap = await parser.parseStringPromise(sitemapRes.data);
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify(sitemap),
+    };
  } catch (error) {
-    throw new Error(error.message);
+    // If error occurs
+    console.log(error.message);
+    if (error.code === 'ECONNABORTED') {
+      return {
+        statusCode: 500,
+        body: JSON.stringify({ error: 'Request timed out' }),
+      };
+    } else {
+      return {
+        statusCode: 500,
+        body: JSON.stringify({ error: error.message }),
+      };
+    }
  }
 };

-exports.handler = middleware(fetchSitemapHandler);
+exports.handler = commonMiddleware(handler);
--- a/api/social-tags.js
+++ b/api/social-tags.js
@ -0,0 +1,68 @@
+const axios = require('axios');
+const cheerio = require('cheerio');
+
+exports.handler = async (event, context) => {
+  let url = event.queryStringParameters.url;
+  
+  // Check if url includes protocol
+  if (!url.startsWith('http://') && !url.startsWith('https://')) {
+    url = 'http://' + url;
+  }
+  
+  try {
+    const response = await axios.get(url);
+    const html = response.data;
+    const $ = cheerio.load(html);
+    
+    const metadata = {
+      // Basic meta tags
+      title: $('head title').text(),
+      description: $('meta[name="description"]').attr('content'),
+      keywords: $('meta[name="keywords"]').attr('content'),
+      canonicalUrl: $('link[rel="canonical"]').attr('href'),
+
+      // OpenGraph Protocol
+      ogTitle: $('meta[property="og:title"]').attr('content'),
+      ogType: $('meta[property="og:type"]').attr('content'),
+      ogImage: $('meta[property="og:image"]').attr('content'),
+      ogUrl: $('meta[property="og:url"]').attr('content'),
+      ogDescription: $('meta[property="og:description"]').attr('content'),
+      ogSiteName: $('meta[property="og:site_name"]').attr('content'),
+      
+      // Twitter Cards
+      twitterCard: $('meta[name="twitter:card"]').attr('content'),
+      twitterSite: $('meta[name="twitter:site"]').attr('content'),
+      twitterCreator: $('meta[name="twitter:creator"]').attr('content'),
+      twitterTitle: $('meta[name="twitter:title"]').attr('content'),
+      twitterDescription: $('meta[name="twitter:description"]').attr('content'),
+      twitterImage: $('meta[name="twitter:image"]').attr('content'),
+
+      // Misc
+      themeColor: $('meta[name="theme-color"]').attr('content'),
+      robots: $('meta[name="robots"]').attr('content'),
+      googlebot: $('meta[name="googlebot"]').attr('content'),
+      generator: $('meta[name="generator"]').attr('content'),
+      viewport: $('meta[name="viewport"]').attr('content'),
+      author: $('meta[name="author"]').attr('content'),
+      publisher: $('link[rel="publisher"]').attr('href'),
+      favicon: $('link[rel="icon"]').attr('href')
+    };
+
+    if (Object.keys(metadata).length === 0) {
+      return {
+        statusCode: 200,
+        body: JSON.stringify({ skipped: 'No metadata found' }),
+      };
+    }
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify(metadata),
+    };
+  } catch (error) {
+    return {
+      statusCode: 500,
+      body: JSON.stringify({ error: 'Failed fetching data' }),
+    };
+  }
+};
--- a/src/components/Form/Card.tsx
+++ b/src/components/Form/Card.tsx
@ -30,7 +30,7 @@ export const Card = (props: CardProps): JSX.Element => {
    <ErrorBoundary title={heading}>
      <StyledCard styles={styles}>
        { actionButtons && actionButtons }
-        { heading && <Heading as="h3" align="left" color={colors.primary}>{heading}</Heading> }
+        { heading && <Heading className="inner-heading" as="h3" align="left" color={colors.primary}>{heading}</Heading> }
        {children}
      </StyledCard>
    </ErrorBoundary>
--- a/src/components/Form/Heading.tsx
+++ b/src/components/Form/Heading.tsx
@ -10,6 +10,7 @@ interface HeadingProps {
  inline?: boolean;
  children: React.ReactNode;
  id?: string;
+  className?: string;
 };

 const StyledHeading = styled.h1<HeadingProps>`
@ -47,10 +48,14 @@ const StyledHeading = styled.h1<HeadingProps>`
  ${props => props.inline ? 'display: inline;' : '' }
 `;

+const makeAnchor = (title: string): string => {
+  return title.toLowerCase().replace(/[^\w\s]|_/g, "").replace(/\s+/g, "-");
+};
+
 const Heading = (props: HeadingProps): JSX.Element => {
-  const { children, as, size, align, color, inline, id } = props;
+  const { children, as, size, align, color, inline, id, className } = props;
  return (
-    <StyledHeading as={as} size={size} align={align} color={color} inline={inline} id={id}>
+    <StyledHeading as={as} size={size} align={align} color={color} inline={inline} className={className} id={id || makeAnchor((children || '')?.toString())}>
      {children}
    </StyledHeading>
  );
--- a/src/components/Results/ContentLinks.tsx
+++ b/src/components/Results/ContentLinks.tsx
@ -43,9 +43,8 @@ const getPathName = (link: string) => {
 };

 const ContentLinksCard = (props: { data: any, title: string, actionButtons: any }): JSX.Element => {
-  const { internal, external} = props.data;
-  console.log('Internal Links', internal);
-  console.log('External Links', external);
+  const internal = props.data.internal || [];
+  const external = props.data.external || [];
  return (
    <Card heading={props.title} actionButtons={props.actionButtons} styles={cardStyles}>
      <Heading as="h3" size="small" color={colors.primary}>Summary</Heading>
@ -71,17 +70,6 @@ const ContentLinksCard = (props: { data: any, title: string, actionButtons: any
          ))}
        </details>
      )}
-      {/* {portData.openPorts.map((port: any) => (
-          <Row key={port} lbl="" val="">
-            <span>{port}</span>
-          </Row>
-        )
-      )}
-      <br />
-      <small>
-        Unable to establish connections to:<br />
-        {portData.failedPorts.join(', ')}
-      </small> */}
    </Card>
  );
 }
--- a/src/components/Results/MailConfig.tsx
+++ b/src/components/Results/MailConfig.tsx
@ -0,0 +1,45 @@
+
+import { Card } from 'components/Form/Card';
+import Row from 'components/Form/Row';
+import Heading from 'components/Form/Heading';
+import colors from 'styles/colors';
+
+const cardStyles = ``;
+
+const MailConfigCard = (props: {data: any, title: string, actionButtons: any }): JSX.Element => {
+  const mailServer = props.data;
+  const txtRecords = (mailServer.txtRecords || []).join('').toLowerCase() || '';
+  return (
+    <Card heading={props.title} actionButtons={props.actionButtons} styles={cardStyles}>
+      <Heading as="h3" color={colors.primary} size="small">Mail Security Checklist</Heading>
+      <Row lbl="SPF" val={txtRecords.includes('spf')} />
+      <Row lbl="DKIM" val={txtRecords.includes('dkim')} />
+      <Row lbl="DMARC" val={txtRecords.includes('dmarc')} />
+      <Row lbl="BIMI" val={txtRecords.includes('bimi')} />
+
+      { mailServer.mxRecords && <Heading as="h3" color={colors.primary} size="small">MX Records</Heading>}
+      { mailServer.mxRecords && mailServer.mxRecords.map((record: any) => (
+          <Row lbl="" val="">
+            <span>{record.exchange}</span>
+            <span>{record.priority ? `Priority: ${record.priority}` : ''}</span>
+          </Row>
+        ))
+      }
+      { mailServer.mailServices.length > 0 && <Heading as="h3" color={colors.primary} size="small">External Mail Services</Heading>}
+      { mailServer.mailServices && mailServer.mailServices.map((service: any) => (
+        <Row lbl={service.provider} val={service.value} />
+        ))
+      }
+
+      { mailServer.txtRecords && <Heading as="h3" color={colors.primary} size="small">Mail-related TXT Records</Heading>}
+      { mailServer.txtRecords && mailServer.txtRecords.map((record: any) => (
+          <Row lbl="" val="">
+            <span>{record}</span>
+          </Row>
+        ))
+      }
+    </Card>
+  );
+}
+
+export default MailConfigCard;
--- a/src/components/Results/SocialTags.tsx
+++ b/src/components/Results/SocialTags.tsx
@ -0,0 +1,44 @@
+
+import { Card } from 'components/Form/Card';
+import Row from 'components/Form/Row';
+import colors from 'styles/colors';
+
+const cardStyles = `
+  .banner-image img {
+    width: 100%;
+    border-radius: 4px;
+    margin: 0.5rem 0;
+  }
+  .color-field {
+    border-radius: 4px;
+    &:hover {
+      color: ${colors.primary};
+    }
+  }
+`;
+
+const SocialTagsCard = (props: {data: any, title: string, actionButtons: any }): JSX.Element => {
+  const tags = props.data;
+  return (
+    <Card heading={props.title} actionButtons={props.actionButtons} styles={cardStyles}>
+      { tags.title && <Row lbl="Title" val={tags.title} /> }
+      { tags.description && <Row lbl="Description" val={tags.description} /> }
+      { tags.keywords && <Row lbl="Keywords" val={tags.keywords} /> }
+      { tags.canonicalUrl && <Row lbl="Canonical URL" val={tags.canonicalUrl} /> }
+      { tags.themeColor && <Row lbl="" val="">
+        <span className="lbl">Theme Color</span>
+        <span className="val color-field" style={{background: tags.themeColor}}>{tags.themeColor}</span>
+      </Row> }
+      { tags.twitterSite && <Row lbl="" val="">
+        <span className="lbl">Twitter Site</span>
+        <span className="val"><a href={`https://x.com/${tags.twitterSite}`}>{tags.twitterSite}</a></span>
+      </Row> }
+      { tags.author && <Row lbl="Author" val={tags.author} />}
+      { tags.publisher && <Row lbl="Publisher" val={tags.publisher} />}
+      { tags.generator && <Row lbl="Generator" val={tags.generator} />}
+      { tags.ogImage && <div className="banner-image"><img src={tags.ogImage} alt="Banner" /></div> }
+    </Card>
+  );
+}
+
+export default SocialTagsCard;
--- a/src/components/misc/AdditionalResources.tsx
+++ b/src/components/misc/AdditionalResources.tsx
@ -8,12 +8,13 @@ margin: 0;
 padding: 1rem;
 display: grid;
 gap: 0.5rem;
-grid-template-columns: repeat(auto-fit, minmax(20rem, 1fr));
+grid-template-columns: repeat(auto-fit, minmax(19rem, 1fr));
 li a.resource-wrap {
  display: flex;
+  flex-direction: column;
  align-items: start;
  gap: 0.25rem;
-  padding: 0.25rem;
+  padding: 0.25rem 0.5rem;
  background: ${colors.background};
  border-radius: 8px;
  text-decoration: none;
@ -37,27 +38,32 @@ li a.resource-wrap {
  }
 }
 img {
-  width: 4rem;
+  width: 2.5rem;
  border-radius: 4px;
  margin: 0.25rem 0.1rem 0.1rem 0.1rem;
 }
+p, a {
+  margin: 0;
+}
+a.resource-link {
+  color: ${colors.primary};
+  opacity: 0.75;
+  font-size: 0.9rem;
+  transition: all 0.2s ease-in-out;
+}
+.resource-title {
+  font-weight: bold;
+}
+.resource-lower {
+  display: flex;
+  align-items: center;
+  gap: 0.5rem;
+}
 .resource-details {
  max-width: 20rem;
  display: flex;
  flex-direction: column;
  gap: 0.1rem;
-  p, a {
-    margin: 0;
-  }
-  a.resource-link {
-    color: ${colors.primary};
-    opacity: 0.75;
-    font-size: 0.9rem;
-    transition: all 0.2s ease-in-out;
-  }
-  .resource-title {
-    font-weight: bold;
-  }
  .resource-description {
    color: ${colors.textColorSecondary};
    font-size: 0.9rem;
@ -155,6 +161,13 @@ const resources = [
    description: 'Checks the performance, accessibility and SEO of a page on mobile + desktop.',
    searchLink: 'https://developers.google.com/speed/pagespeed/insights/?url={URL}',
  },
+  {
+    title: 'Built With',
+    link: 'https://builtwith.com/',
+    icon: 'https://i.ibb.co/5LXBDfD/Built-with.png',
+    description: 'View the tech stack of a website',
+    searchLink: 'https://builtwith.com/{URL}',
+  },
  {
    title: 'DNS Dumpster',
    link: 'https://dnsdumpster.com/',
@ -192,7 +205,7 @@ const resources = [
 ];

 const makeLink = (resource: any, scanUrl: string | undefined): string => {
-  return (scanUrl && resource.searchLink) ? resource.searchLink.replaceAll('{URL}', scanUrl.replace('https://', '')) : '#';
+  return (scanUrl && resource.searchLink) ? resource.searchLink.replaceAll('{URL}', scanUrl.replace('https://', '')) : resource.link;
 };

 const AdditionalResources = (props: { url?: string }): JSX.Element => {
@ -203,12 +216,14 @@ const AdditionalResources = (props: { url?: string }): JSX.Element => {
          return (
            <li>
              <a className="resource-wrap" href={makeLink(resource, props.url)}>
-              <img src={resource.icon} alt="" />
-              <div className="resource-details">
                <p className="resource-title">{resource.title}</p>
                <a className="resource-link" href={resource.link} target="_blank" rel="noreferrer">{new URL(resource.link).hostname}</a>
+                <div className="resource-lower">
+                  <img src={resource.icon} alt="" />
+                  <div className="resource-details">
                    <p className="resource-description">{resource.description}</p>
                  </div>
+                </div>
              </a>
            </li>
          );
--- a/src/components/misc/Footer.tsx
+++ b/src/components/misc/Footer.tsx
@ -41,7 +41,7 @@ const Link = styled.a`
 `;

 const Footer = (props: { isFixed?: boolean }): JSX.Element => {
-  const licenseUrl = 'https://github.com/lissy93/web-check/blob/main/LICENSE';
+  const licenseUrl = 'https://github.com/lissy93/web-check/blob/master/LICENSE';
  const authorUrl = 'https://aliciasykes.com';
  const githubUrl = 'https://github.com/lissy93/web-check';
  return (
--- a/src/components/misc/ProgressBar.tsx
+++ b/src/components/misc/ProgressBar.tsx
@ -167,6 +167,9 @@ p {
 }
 pre {
  color: ${colors.danger};
+  &.info {
+    color: ${colors.warning};
+  }
 }
 `;

@ -191,6 +194,7 @@ const jobNames = [
  'hosts',
  'quality',
  'cookies',
+  'ssl',
  // 'server-info',
  'redirects',
  'robots-txt',
@ -202,7 +206,9 @@ const jobNames = [
  'sitemap',
  'hsts',
  'security-txt',
+  'social-tags',
  'linked-pages',
+  'mail-config',
  // 'whois',
  'features',
  'carbon',
@ -360,7 +366,7 @@ const ProgressLoader = (props: { loadStatus: LoadingJob[], showModal: (err: Reac
    }
  };

-  const showErrorModal = (name: string, state: LoadingState, timeTaken: number | undefined, error: string) => {
+  const showErrorModal = (name: string, state: LoadingState, timeTaken: number | undefined, error: string, isInfo?: boolean) => {
    const errorContent = (
      <ErrorModalContent>
        <Heading as="h3">Error Details for {name}</Heading>
@ -368,7 +374,8 @@ const ProgressLoader = (props: { loadStatus: LoadingJob[], showModal: (err: Reac
          The {name} job failed with an {state} state after {timeTaken} ms.
          The server responded with the following error:
        </p>
-        <pre>{error}</pre>
+        { /* If isInfo == true, then add .info className to pre */}
+        <pre className={isInfo ? 'info' : 'error'}>{error}</pre>
      </ErrorModalContent>
    );
    props.showModal(errorContent);
@ -409,6 +416,7 @@ const ProgressLoader = (props: { loadStatus: LoadingJob[], showModal: (err: Reac
                <i>{(timeTaken && state !== 'loading') ? ` Took ${timeTaken} ms` : '' }</i>
                { (retry && state !== 'success' && state !== 'loading') && <FailedJobActionButton onClick={retry}>↻ Retry</FailedJobActionButton> }
                { (error && state === 'error') && <FailedJobActionButton onClick={() => showErrorModal(name, state, timeTaken, error)}>■ Show Error</FailedJobActionButton> }
+                { (error && state === 'skipped') && <FailedJobActionButton onClick={() => showErrorModal(name, state, timeTaken, error, true)}>■ Show Reason</FailedJobActionButton> }
              </li>
            );
          })
--- a/src/hooks/motherOfAllHooks.ts
+++ b/src/hooks/motherOfAllHooks.ts
@ -38,21 +38,20 @@ const useMotherOfAllHooks = <ResultType = any>(params: UseIpAddressProps<ResultT
  const doTheFetch = () => {
    return fetchRequest()
    .then((res: any) => {
-      if (!res) {
+      if (!res) { // No response :(
+        updateLoadingJobs(jobId, 'error', res.error || 'No response', reset);
+      } else if (res.error) { // Response returned an error message
        updateLoadingJobs(jobId, 'error', res.error, reset);
-        throw new Error('No response');
-      }
-      if (res.error) {
-        updateLoadingJobs(jobId, 'error', res.error, reset);
-        throw new Error(res.error);
-      }
-      // All went to plan, set results and mark as done
+      } else if (res.skipped) { // Response returned a skipped message
+        updateLoadingJobs(jobId, 'skipped', res.skipped, reset);
+      }  else { // Yay, everything went to plan :)
        setResult(res);
        updateLoadingJobs(jobId, 'success', '', undefined, res);
+      }
    })
    .catch((err) => {
-      // Something fucked up, log the error
-      updateLoadingJobs(jobId, 'error', err.error || err.message, reset);
+      // Something fucked up
+      updateLoadingJobs(jobId, 'error', err.error || err.message || 'Unknown error', reset);
      throw err;
    })
  }
@ -69,6 +68,7 @@ const useMotherOfAllHooks = <ResultType = any>(params: UseIpAddressProps<ResultT
        pending: `Updating Data (${jobId})`,
        success: `Completed (${jobId})`,
        error: `Failed to update (${jobId})`,
+        skipped: `Skipped job (${jobId}), as no valid results for host`,
      };
      // Initiate fetch, and show progress toast
      toast.promise(fetchyFetch, toastOptions).catch(() => {});
--- a/src/index.css
+++ b/src/index.css
@ -5,6 +5,10 @@
  font-style: normal;
 }

+html {
+  scroll-behavior: smooth;
+}
+
 body {
  margin: 0;
  font-family: 'PTMono', -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
--- a/src/pages/About.tsx
+++ b/src/pages/About.tsx
@ -5,9 +5,9 @@ import Heading from 'components/Form/Heading';
 import Footer from 'components/misc/Footer';
 import Nav from 'components/Form/Nav';
 import Button from 'components/Form/Button';
+import AdditionalResources from 'components/misc/AdditionalResources';
 import { StyledCard } from 'components/Form/Card';
-import docs, { about, license, fairUse } from 'utils/docs';
-
+import docs, { about, license, fairUse, supportUs } from 'utils/docs';

 const AboutContainer = styled.div`
 width: 95vw;
@ -16,6 +16,11 @@ margin: 2rem auto;
 padding-bottom: 1rem;
 header {
  margin 1rem 0;
+  width: auto;
+}
+section {
+  width: auto;
+  .inner-heading { display: none; }
 }
 `;

@ -32,6 +37,9 @@ const Section = styled(StyledCard)`
  margin-bottom: 2rem;
  overflow: clip;
  max-height: 100%;
+  section {
+    clear: both;
+  }
  h3 {
    font-size: 1.5rem;
  }
@ -71,21 +79,32 @@ const Section = styled(StyledCard)`
      }
    }
  }
-  .screenshot {
+  .example-screenshot {
+    float: right; 
+    display: inline-flex;
+    flex-direction: column;
+    clear: both;
+    max-width: 300px;
+    img {
      float: right;
      break-inside: avoid;
      max-width: 300px;
-    max-height: 28rem;
+      // max-height: 30rem;
      border-radius: 6px;
      clear: both;
    }
+    figcaption {
+      font-size: 0.8rem;
+      text-align: center;
+      opacity: 0.7;
+    }
+  }
 `;

 const makeAnchor = (title: string): string => {
  return title.toLowerCase().replace(/[^\w\s]|_/g, "").replace(/\s+/g, "-");
 };

-
 const About = (): JSX.Element => {
  return (
    <div>
@ -118,9 +137,13 @@ const About = (): JSX.Element => {
        </div>
        {docs.map((section, sectionIndex: number) => (
          <section key={section.title}>
+            { sectionIndex > 0 && <hr /> }
            <Heading as="h3" size="small" id={makeAnchor(section.title)} color={colors.primary}>{section.title}</Heading>
            {section.screenshot &&
+              <figure className="example-screenshot">
                <img className="screenshot" src={section.screenshot} alt={`Example Screenshot ${section.title}`} />
+                <figcaption>Fig.{sectionIndex + 1} - Example of {section.title}</figcaption>
+              </figure> 
            }
            {section.description && <>
              <Heading as="h4" size="small">Description</Heading>
@ -142,11 +165,24 @@ const About = (): JSX.Element => {
                ))}
              </ul>
            </>}
-            { sectionIndex < docs.length - 1 && <hr /> }
+            {/* { sectionIndex < docs.length - 1 && <hr /> } */}
          </section>
        ))}
      </Section>

+      <Heading as="h2" size="medium" color={colors.primary}>API Documentation</Heading>
+      <Section>
+        <p>// Coming soon...</p>
+      </Section>
+
+      <Heading as="h2" size="medium" color={colors.primary}>Additional Resources</Heading>
+      <AdditionalResources />
+
+      <Heading as="h2" size="medium" color={colors.primary}>Support Us</Heading>
+      <Section>
+        {supportUs.map((para, index: number) => (<p dangerouslySetInnerHTML={{__html: para}} />))}
+      </Section>
+
      <Heading as="h2" size="medium" color={colors.primary}>Terms & Info</Heading>
      <Section>
      <Heading as="h3" size="small" color={colors.primary}>License</Heading>
@ -166,6 +202,11 @@ const About = (): JSX.Element => {
        <Heading as="h3" size="small" color={colors.primary}>Privacy</Heading>
        <p>
        Analytics are used on the demo instance (via a self-hosted Plausible instance), this only records the URL you visited but no personal data.
+        There's also some basic error logging (via a self-hosted GlitchTip instance), this is only used to help me fix bugs.
+        <br />
+        <br />
+        Neither your IP address, browser/OS/hardware info, nor any other data will ever be collected or logged.
+        (You may verify this yourself, either by inspecting the source code or the using developer tools)
        </p>
        <hr />
        <Heading as="h3" size="small" color={colors.primary}>Support</Heading>
--- a/src/pages/Home.tsx
+++ b/src/pages/Home.tsx
@ -179,8 +179,15 @@ const Home = (): JSX.Element => {
          </ul>
        </div>
        <div className="links">
-          <a href="https://app.netlify.com/start/deploy?repository=https://github.com/lissy93/web-check"><Button>Deploy your own Instance</Button></a>
-          <a href="https://github.com/lissy93/web-check"><Button>View on GitHub</Button></a>
+          <a href="https://github.com/lissy93/web-check" title="Check out the source code and documentation on GitHub, and get support or contribute">
+            <Button>View on GitHub</Button>
+          </a>
+          <a href="https://app.netlify.com/start/deploy?repository=https://github.com/lissy93/web-check" title="Deploy your own private or public instance of Web-Check to Netlify">
+            <Button>Deploy your own</Button>
+          </a>
+          <a href="/about#api-documentation" title="View the API documentation, to use Web-Check programmatically">
+            <Button>API Docs</Button>
+          </a>
        </div>
      </SiteFeaturesWrapper>
      <Footer isFixed={true} />
--- a/src/pages/Results.tsx
+++ b/src/pages/Results.tsx
@ -46,6 +46,8 @@ import DnsServerCard from 'components/Results/DnsServer';
 import TechStackCard from 'components/Results/TechStack';
 import SecurityTxtCard from 'components/Results/SecurityTxt';
 import ContentLinksCard from 'components/Results/ContentLinks';
+import SocialTagsCard from 'components/Results/SocialTags';
+import MailConfigCard from 'components/Results/MailConfig';

 import keys from 'utils/get-keys';
 import { determineAddressType, AddressType } from 'utils/address-type-checker';
@ -341,6 +343,14 @@ const Results = (): JSX.Element => {
    fetchRequest: () => fetch(`${api}/security-txt?url=${address}`).then(res => parseJson(res)),
  });

+  // Get social media previews, from a sites social meta tags
+  const [socialTagResults, updateSocialTagResults] = useMotherHook({
+    jobId: 'social-tags',
+    updateLoadingJobs,
+    addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
+    fetchRequest: () => fetch(`${api}/social-tags?url=${address}`).then(res => parseJson(res)),
+  });
+
  // Get site features from BuiltWith
  const [siteFeaturesResults, updateSiteFeaturesResults] = useMotherHook({
    jobId: 'features',
@ -388,6 +398,14 @@ const Results = (): JSX.Element => {
    fetchRequest: () => fetch(`${api}/content-links?url=${address}`).then(res => parseJson(res)),
  });

+  // Get mail config for server, based on DNS records
+  const [mailConfigResults, updateMailConfigResults] = useMotherHook({
+    jobId: 'mail-config',
+    updateLoadingJobs,
+    addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
+    fetchRequest: () => fetch(`${api}/mail-config?url=${address}`).then(res => parseJson(res)),
+  });
+
  /* Cancel remaining jobs after  10 second timeout */
  useEffect(() => {
    const checkJobs = () => {
@ -434,7 +452,9 @@ const Results = (): JSX.Element => {
    { id: 'txt-records', title: 'TXT Records', result: txtRecordResults, Component: TxtRecordCard, refresh: updateTxtRecordResults },
    { id: 'hsts', title: 'HSTS Check', result: hstsResults, Component: HstsCard, refresh: updateHstsResults },
    { id: 'whois', title: 'Domain Info', result: whoIsResults, Component: WhoIsCard, refresh: updateWhoIsResults },
+    { id: 'mail-config', title: 'Email Configuration', result: mailConfigResults, Component: MailConfigCard, refresh: updateMailConfigResults },
    { id: 'dns-server', title: 'DNS Server', result: dnsServerResults, Component: DnsServerCard, refresh: updateDnsServerResults },
+    { id: 'social-tags', title: 'Social Tags', result: socialTagResults, Component: SocialTagsCard, refresh: updateSocialTagResults },
    { id: 'linked-pages', title: 'Linked Pages', result: linkedPagesResults, Component: ContentLinksCard, refresh: updateLinkedPagesResults },
    { id: 'features', title: 'Site Features', result: siteFeaturesResults, Component: SiteFeaturesCard, refresh: updateSiteFeaturesResults },
    { id: 'sitemap', title: 'Pages', result: sitemapResults, Component: SitemapCard, refresh: updateSitemapResults },
--- a/src/utils/docs.ts
+++ b/src/utils/docs.ts
@ -12,26 +12,26 @@ const docs: Doc[] = [
    id: "get-ip",
    title: "IP Info",
    description:
-      "The IP Address task involves mapping the user provided URL to its corresponding IP address through a process known as Domain Name System (DNS) resolution. An IP address is a unique identifier given to every device on the Internet, and when paired with a domain name, it allows for accurate routing of online requests and responses.",
-    use: "Identifying the IP address of a domain can be incredibly valuable for OSINT purposes. This information can aid in creating a detailed map of a target's network infrastructure, pinpointing the physical location of a server, identifying the hosting service, and even discovering other domains that are hosted on the same IP address. In cybersecurity, it's also useful for tracking the sources of attacks or malicious activities.",
+      "An IP address (Internet Protocol address) is a numerical label assigned to each device connected to a network / the internet. The IP associated with a given domain can be found by querying the Domain Name System (DNS) for the domain's A (address) record.",
+    use: "Finding the IP of a given server is the first step to conducting further investigations, as it allows us to probe the server for additional info. Including creating a detailed map of a target's network infrastructure, pinpointing the physical location of a server, identifying the hosting service, and even discovering other domains that are hosted on the same IP address.",
    resources: [
-      "https://en.wikipedia.org/wiki/IP_address",
-      "https://tools.ietf.org/html/rfc791",
-      "https://www.cloudflare.com/learning/dns/what-is-dns/",
-      "https://www.whois.com/whois-lookup",
+      { title: 'Understanding IP Addresses', link: 'https://www.digitalocean.com/community/tutorials/understanding-ip-addresses-subnets-and-cidr-notation-for-networking'},
+      { title: 'IP Addresses - Wiki', link: 'https://en.wikipedia.org/wiki/IP_address'},
+      { title: 'RFC-791 Internet Protocol', link: 'https://tools.ietf.org/html/rfc791'},
+      { title: 'whatismyipaddress.com', link: 'https://whatismyipaddress.com/'},
    ],
  },
  {
    id: "ssl",
    title: "SSL Chain",
    description:
-      "The SSL task involves checking if the site has a valid Secure Sockets Layer (SSL) certificate. SSL is a protocol for establishing authenticated and encrypted links between networked computers. It's commonly used for securing communications over the internet, such as web browsing sessions, email transmissions, and more. In this task, we reach out to the server and initiate a SSL handshake. If successful, we gather details about the SSL certificate presented by the server.",
+    "SSL certificates are digital certificates that authenticate the identity of a website or server, enable secure encrypted communication (HTTPS), and establish trust between clients and servers. A valid SSL certificate is required for a website to be able to use the HTTPS protocol, and encrypt user + site data in transit. SSL certificates are issued by Certificate Authorities (CAs), which are trusted third parties that verify the identity and legitimacy of the certificate holder.",  
    use: "SSL certificates not only provide the assurance that data transmission to and from the website is secure, but they also provide valuable OSINT data. Information from an SSL certificate can include the issuing authority, the domain name, its validity period, and sometimes even organization details. This can be useful for verifying the authenticity of a website, understanding its security setup, or even for discovering associated subdomains or other services.",
    resources: [
-      "https://en.wikipedia.org/wiki/Transport_Layer_Security",
-      "https://tools.ietf.org/html/rfc8446",
-      "https://letsencrypt.org/docs/",
-      "https://www.sslshopper.com/ssl-checker.html",
+      { title: 'TLS - Wiki', link: 'https://en.wikipedia.org/wiki/Transport_Layer_Security'},
+      { title: 'What is SSL (via Cloudflare learning)', link: 'https://www.cloudflare.com/learning/ssl/what-is-ssl/'},
+      { title: 'RFC-8446 - TLS', link: 'https://tools.ietf.org/html/rfc8446'},
+      { title: 'SSL Checker', link: 'https://www.sslshopper.com/ssl-checker.html'},
    ],
    screenshot: 'https://i.ibb.co/kB7LsV1/wc-ssl.png',
  },
@ -39,13 +39,13 @@ const docs: Doc[] = [
    id: "dns",
    title: "DNS Records",
    description:
-      "The DNS Records task involves querying the Domain Name System (DNS) for records associated with the target domain. DNS is a system that translates human-readable domain names into IP addresses that computers use to communicate. Various types of DNS records exist, including A (address), MX (mail exchange), NS (name server), CNAME (canonical name), and TXT (text), among others.",
+      "This task involves looking up the DNS records associated with a specific domain. DNS is a system that translates human-readable domain names into IP addresses that computers use to communicate. Various types of DNS records exist, including A (address), MX (mail exchange), NS (name server), CNAME (canonical name), and TXT (text), among others.",
    use: "Extracting DNS records can provide a wealth of information in an OSINT investigation. For example, A and AAAA records can disclose IP addresses associated with a domain, potentially revealing the location of servers. MX records can give clues about a domain's email provider. TXT records are often used for various administrative purposes and can sometimes inadvertently leak internal information. Understanding a domain's DNS setup can also be useful in understanding how its online infrastructure is built and managed.",
    resources: [
-      "https://en.wikipedia.org/wiki/List_of_DNS_record_types",
-      "https://tools.ietf.org/html/rfc1035",
-      "https://mxtoolbox.com/DNSLookup.aspx",
-      "https://www.dnswatch.info/",
+      { title: 'What are DNS records? (via Cloudflare learning)', link: 'https://www.cloudflare.com/learning/dns/dns-records/'},
+      { title: 'DNS Record Types', link: 'https://en.wikipedia.org/wiki/List_of_DNS_record_types'},
+      { title: 'RFC-1035 - DNS', link: 'https://tools.ietf.org/html/rfc1035'},
+      { title: 'DNS Lookup (via MxToolbox)', link: 'https://mxtoolbox.com/DNSLookup.aspx'},
    ],
    screenshot: 'https://i.ibb.co/7Q1kMwM/wc-dns.png',
  },
@ -54,12 +54,12 @@ const docs: Doc[] = [
    title: "Cookies",
    description:
      "The Cookies task involves examining the HTTP cookies set by the target website. Cookies are small pieces of data stored on the user's computer by the web browser while browsing a website. They hold a modest amount of data specific to a particular client and website, such as site preferences, the state of the user's session, or tracking information.",
-    use: "Cookies provide a wealth of information in an OSINT investigation. They can disclose information about how the website tracks and interacts with its users. For instance, session cookies can reveal how user sessions are managed, and tracking cookies can hint at what kind of tracking or analytics frameworks are being used. Additionally, examining cookie policies and practices can offer insights into the site's security settings and compliance with privacy regulations.",
+    use: "Cookies can disclose information about how the website tracks and interacts with its users. For instance, session cookies can reveal how user sessions are managed, and tracking cookies can hint at what kind of tracking or analytics frameworks are being used. Additionally, examining cookie policies and practices can offer insights into the site's security settings and compliance with privacy regulations.",
    resources: [
-      "https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies",
-      "https://www.cookiepro.com/knowledge/what-is-a-cookie/",
-      "https://owasp.org/www-community/controls/SecureFlag",
-      "https://tools.ietf.org/html/rfc6265",
+      { title: 'HTTP Cookie Docs (Mozilla)', link: 'https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies' },
+      { title: 'What are Cookies (via Cloudflare Learning)', link: 'https://www.cloudflare.com/learning/privacy/what-are-cookies/' },
+      { title: 'Testing for Cookie Attributes (OWASP)', link: 'https://owasp.org/www-project-web-security-testing-guide/v42/4-Web_Application_Security_Testing/06-Session_Management_Testing/02-Testing_for_Cookies_Attributes' },
+      { title: 'RFC-6265 - Coolies', link: 'https://tools.ietf.org/html/rfc6265' },
    ],
    screenshot: 'https://i.ibb.co/TTQ6DtP/wc-cookies.png',
  },
@ -67,13 +67,13 @@ const docs: Doc[] = [
    id: "robots-txt",
    title: "Crawl Rules",
    description:
-      "The Crawl Rules task is focused on retrieving and interpreting the 'robots.txt' file from the target website. This text file is part of the Robots Exclusion Protocol (REP), a group of web standards that regulate how robots crawl the web, access and index content, and serve that content up to users. The file indicates which parts of the site the website owner doesn't want to be accessed by web crawler bots.",
-    use: "The 'robots.txt' file can provide valuable information for an OSINT investigation. It often discloses the directories and pages that the site owner doesn't want to be indexed, potentially because they contain sensitive information. Moreover, it might reveal the existence of otherwise hidden or unlinked directories. Additionally, understanding crawl rules may offer insights into a website's SEO strategies.",
+      "Robots.txt is a file found (usually) at the root of a domain, and is used to implement the Robots Exclusion Protocol (REP) to indicate which pages should be ignored by which crawlers and bots. It's good practice to avoid search engine crawlers from over-loading your site, but should not be used to keep pages out of search results (use the noindex meta tag or header instead).",
+    use: "It's often useful to check the robots.txt file during an investigation, as it can sometimes disclose the directories and pages that the site owner doesn't want to be indexed, potentially because they contain sensitive information, or reveal the existence of otherwise hidden or unlinked directories. Additionally, understanding crawl rules may offer insights into a website's SEO strategies.",
    resources: [
-      "https://developers.google.com/search/docs/advanced/robots/intro",
-      "https://www.robotstxt.org/robotstxt.html",
-      "https://moz.com/learn/seo/robotstxt",
-      "https://en.wikipedia.org/wiki/Robots_exclusion_standard",
+      { title: 'Google Search Docs - Robots.txt', link: 'https://developers.google.com/search/docs/advanced/robots/intro' },
+      { title: 'Learn about robots.txt (via Moz.com)', link: 'https://moz.com/learn/seo/robotstxt' },
+      { title: 'RFC-9309 -  Robots Exclusion Protocol', link: 'https://datatracker.ietf.org/doc/rfc9309/' },
+      { title: 'Robots.txt - wiki', link: 'https://en.wikipedia.org/wiki/Robots_exclusion_standard' },
    ],
    screenshot: 'https://i.ibb.co/KwQCjPf/wc-robots.png',
  },
@ -84,10 +84,10 @@ const docs: Doc[] = [
      "The Headers task involves extracting and interpreting the HTTP headers sent by the target website during the request-response cycle. HTTP headers are key-value pairs sent at the start of an HTTP response, or before the actual data. Headers contain important directives for how to handle the data being transferred, including cache policies, content types, encoding, server information, security policies, and more.",
    use: "Analyzing HTTP headers can provide significant insights in an OSINT investigation. Headers can reveal specific server configurations, chosen technologies, caching directives, and various security settings. This information can help to determine a website's underlying technology stack, server-side security measures, potential vulnerabilities, and general operational practices.",
    resources: [
-      "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers",
-      "https://tools.ietf.org/html/rfc7231#section-3.2",
-      "https://www.w3schools.com/tags/ref_httpheaders.asp",
-      "https://owasp.org/www-project-secure-headers/",
+      { title: 'HTTP Headers - Docs', link: 'https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers' },
+      { title: 'RFC-7231 Section 7 - Headers', link: 'https://datatracker.ietf.org/doc/html/rfc7231#section-7' },
+      { title: 'List of header response fields', link: 'https://en.wikipedia.org/wiki/List_of_HTTP_header_fields' },
+      { title: 'OWASP Secure Headers Project', link: 'https://owasp.org/www-project-secure-headers/' },
    ],
    screenshot: 'https://i.ibb.co/t3xcwP1/wc-headers.png',
  },
@ -95,13 +95,15 @@ const docs: Doc[] = [
    id: "quality",
    title: "Quality Metrics",
    description:
-      "The Headers task involves extracting and interpreting the HTTP headers sent by the target website during the request-response cycle. HTTP headers are key-value pairs sent at the start of an HTTP response, or before the actual data. Headers contain important directives for how to handle the data being transferred, including cache policies, content types, encoding, server information, security policies, and more.",
-    use: "Analyzing HTTP headers can provide significant insights in an OSINT investigation. Headers can reveal specific server configurations, chosen technologies, caching directives, and various security settings. This information can help to determine a website's underlying technology stack, server-side security measures, potential vulnerabilities, and general operational practices.",
+      "Using Lighthouse, the Quality Metrics task measures the performance, accessibility, best practices, and SEO of the target website. This returns a simple checklist of 100 core metrics, along with a score for each category, to gauge the overall quality of a given site.",
+    use: "Useful for assessing a site's technical health, SEO issues, identify vulnerabilities, and ensure compliance with standards.",
    resources: [
-      "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers",
-      "https://tools.ietf.org/html/rfc7231#section-3.2",
-      "https://www.w3schools.com/tags/ref_httpheaders.asp",
-      "https://owasp.org/www-project-secure-headers/",
+      { title: 'Lighthouse Docs', link: 'https://developer.chrome.com/docs/lighthouse/' },
+      { title: 'Google Page Speed Tools', link: 'https://developers.google.com/speed' },
+      { title: 'W3 Accessibility Tools', link: 'https://www.w3.org/WAI/test-evaluate/' },
+      { title: 'Google Search Console', link: 'https://search.google.com/search-console' },
+      { title: 'SEO Checker', link: 'https://www.seobility.net/en/seocheck/' },
+      { title: 'PWA Builder', link: 'https://www.pwabuilder.com/' },
    ],
    screenshot: 'https://i.ibb.co/Kqg8rx7/wc-quality.png',
  },
@ -109,13 +111,11 @@ const docs: Doc[] = [
    id: "location",
    title: "Server Location",
    description:
-      "The Server Location task determines the physical location of a server hosting a website based on its IP address. The geolocation data typically includes the country, region, and often city where the server is located. The task also provides additional contextual information such as the official language, currency, and flag of the server's location country.",
-    use: "In the realm of OSINT, server location information can be very valuable. It can give an indication of the possible jurisdiction that laws the data on the server falls under, which can be important in legal or investigative contexts. The server location can also hint at the target audience of a website and reveal inconsistencies that could suggest the use of hosting or proxy services to disguise the actual location.",
+      "The Server Location task determines the physical location of the server hosting a given website based on its IP address. This is done by looking up the IP in a location database, which maps the IP to a lat + long of known data centers and ISPs. From the latitude and longitude, it's then possible to show additional contextual info, like a pin on the map, along with address, flag, time zone, currency, etc.",
+    use: "Knowing the server location is a good first step in better understanding a website. For site owners this aids in optimizing content delivery, ensuring compliance with data residency requirements, and identifying potential latency issues that may impact user experience in specific geographical regions. And for security researcher, assess the risk posed by specific regions or jurisdictions regarding cyber threats and regulations.",
    resources: [
-      "https://en.wikipedia.org/wiki/Geolocation_software",
-      "https://www.iplocation.net/",
-      "https://www.cloudflare.com/learning/cdn/glossary/geolocation/",
-      "https://developers.google.com/maps/documentation/geolocation/intro",
+      { title: 'IP Locator', link: 'https://geobytes.com/iplocator/' },
+      { title: 'Internet Geolocation - Wiki', link: 'https://en.wikipedia.org/wiki/Internet_geolocation' },
    ],
    screenshot: 'https://i.ibb.co/cXH2hfR/wc-location.png',
  },
@ -123,13 +123,13 @@ const docs: Doc[] = [
    id: "hosts",
    title: "Associated Hosts",
    description:
-      "This task involves identifying and listing all domains and subdomains (hostnames) that are associated with the website's primary domain. This process often involves DNS enumeration to discover any linked domains and hostnames.",
-    use: "In OSINT investigations, understanding the full scope of a target's web presence is critical. Associated domains could lead to uncovering related projects, backup sites, development/test sites, or services linked to the main site. These can sometimes provide additional information or potential security vulnerabilities. A comprehensive list of associated domains and hostnames can also give an overview of the organization's structure and online footprint.",
+      "This task involves identifying and listing all domains and subdomains (hostnames) that are associated with the website's primary domain. This process often involves DNS enumeration to discover any linked domains and hostnames, as well as looking at known DNS records.",
+    use: "During an investigation, understanding the full scope of a target's web presence is critical. Associated domains could lead to uncovering related projects, backup sites, development/test sites, or services linked to the main site. These can sometimes provide additional information or potential security vulnerabilities. A comprehensive list of associated domains and hostnames can also give an overview of the organization's structure and online footprint.",
    resources: [
-      "https://en.wikipedia.org/wiki/Domain_Name_System",
-      "https://resources.infosecinstitute.com/topic/dns-enumeration-pentest/",
-      "https://subdomainfinder.c99.nl/",
-      "https://securitytrails.com/blog/top-dns-enumeration-tools",
+      { title: 'DNS Enumeration - Wiki', link: 'https://en.wikipedia.org/wiki/DNS_enumeration' },
+      { title: 'OWASP - Enumerate Applications on Webserver', link: 'https://owasp.org/www-project-web-security-testing-guide/latest/4-Web_Application_Security_Testing/01-Information_Gathering/04-Enumerate_Applications_on_Webserver' },
+      { title: 'DNS Enumeration - DNS Dumpster', link: 'https://dnsdumpster.com/' },
+      { title: 'Subdomain Finder', link: 'https://subdomainfinder.c99.nl/' },
    ],
    screenshot: 'https://i.ibb.co/25j1sT7/wc-hosts.png',
  },
@ -138,12 +138,11 @@ const docs: Doc[] = [
    title: "Redirect Chain",
    description:
      "This task traces the sequence of HTTP redirects that occur from the original URL to the final destination URL. An HTTP redirect is a response with a status code that advises the client to go to another URL. Redirects can occur for several reasons, such as URL normalization (directing to the www version of the site), enforcing HTTPS, URL shorteners, or forwarding users to a new site location.",
-    use: "Understanding the redirect chain can be crucial for several reasons. From a security perspective, long or complicated redirect chains can be a sign of potential security risks, such as unencrypted redirects in the chain. Additionally, redirects can impact website performance and SEO, as each redirect introduces additional round-trip-time (RTT). For OSINT, understanding the redirect chain can help identify relationships between different domains or reveal the use of certain technologies or hosting providers.",
+    use: "Understanding the redirect chain can be useful for several reasons. From a security perspective, long or complicated redirect chains can be a sign of potential security risks, such as unencrypted redirects in the chain. Additionally, redirects can impact website performance and SEO, as each redirect introduces additional round-trip-time (RTT). For OSINT, understanding the redirect chain can help identify relationships between different domains or reveal the use of certain technologies or hosting providers.",
    resources: [
-      "https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections",
-      "https://en.wikipedia.org/wiki/URL_redirection",
-      "https://www.screamingfrog.co.uk/server-response-codes/",
-      "https://ahrefs.com/blog/301-redirects/",
+      { title: 'HTTP Redirects - MDN', link: 'https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections' },
+      { title: 'URL Redirection - Wiki', link: 'https://en.wikipedia.org/wiki/URL_redirection' },
+      { title: '301 Redirects explained', link: 'https://ahrefs.com/blog/301-redirects/' },
    ],
    screenshot: 'https://i.ibb.co/hVVrmwh/wc-redirects.png',
  },
@ -151,27 +150,22 @@ const docs: Doc[] = [
    id: "txt-records",
    title: "TXT Records",
    description:
-      "TXT records are a type of Domain Name Service (DNS) record that provides text information to sources outside your domain. They can be used for a variety of purposes, such as verifying domain ownership, ensuring email security, and even preventing unauthorized changes to your website.",
-    use: "In the context of OSINT, TXT records can be a valuable source of information. They may reveal details about the domain's email configuration, the use of specific services like Google Workspace or Microsoft 365, or security measures in place such as SPF and DKIM. Understanding these details can give an insight into the technologies used by the organization, their email security practices, and potential vulnerabilities.",
+      "TXT records are a type of DNS record that provides text information to sources outside your domain. They can be used for a variety of purposes, such as verifying domain ownership, ensuring email security, and even preventing unauthorized changes to your website.",
+    use: "The TXT records often reveal which external services and technologies are being used with a given domain. They may reveal details about the domain's email configuration, the use of specific services like Google Workspace or Microsoft 365, or security measures in place such as SPF and DKIM. Understanding these details can give an insight into the technologies used by the organization, their email security practices, and potential vulnerabilities.",
    resources: [
-      "https://www.cloudflare.com/learning/dns/dns-records/dns-txt-record/",
-      "https://en.wikipedia.org/wiki/TXT_record",
-      "https://tools.ietf.org/html/rfc7208",
-      "https://dmarc.org/wiki/FAQ",
+      { title: 'TXT Records (via Cloudflare Learning)', link: 'https://www.cloudflare.com/learning/dns/dns-records/dns-txt-record/' },
+      { title: 'TXT Records - Wiki', link: 'https://en.wikipedia.org/wiki/TXT_record' },
+      { title: 'RFC-1464 - TXT Records', link: 'https://datatracker.ietf.org/doc/html/rfc1464' },
+      { title: 'TXT Record Lookup (via MxToolbox)', link: 'https://mxtoolbox.com/TXTLookup.aspx' },
    ],
    screenshot: 'https://i.ibb.co/wyt21QN/wc-txt-records.png',
  },
  {
    id: "status",
    title: "Server Status",
-    description:
-      "TXT records are a type of Domain Name Service (DNS) record that provides text information to sources outside your domain. They can be used for a variety of purposes, such as verifying domain ownership, ensuring email security, and even preventing unauthorized changes to your website.",
-    use: "In the context of OSINT, TXT records can be a valuable source of information. They may reveal details about the domain's email configuration, the use of specific services like Google Workspace or Microsoft 365, or security measures in place such as SPF and DKIM. Understanding these details can give an insight into the technologies used by the organization, their email security practices, and potential vulnerabilities.",
+    description: "Checks if a server is online and responding to requests.",
+    use: "",
    resources: [
-      "https://www.cloudflare.com/learning/dns/dns-records/dns-txt-record/",
-      "https://en.wikipedia.org/wiki/TXT_record",
-      "https://tools.ietf.org/html/rfc7208",
-      "https://dmarc.org/wiki/FAQ",
    ],
    screenshot: 'https://i.ibb.co/V9CNLBK/wc-status.png',
  },
@ -180,12 +174,10 @@ const docs: Doc[] = [
    title: "Open Ports",
    description:
      "Open ports on a server are endpoints of communication which are available for establishing connections with clients. Each port corresponds to a specific service or protocol, such as HTTP (port 80), HTTPS (port 443), FTP (port 21), etc. The open ports on a server can be determined using techniques such as port scanning.",
-    use: "In the context of OSINT, knowing which ports are open on a server can provide valuable information about the services running on that server. This information can be useful for understanding the potential vulnerabilities of the system, or for understanding the nature of the services the server is providing. For example, a server with port 22 open (SSH) might be used for remote administration, while a server with port 443 open is serving HTTPS traffic.",
+    use: "Knowing which ports are open on a server can provide information about the services running on that server, useful for understanding the potential vulnerabilities of the system, or for understanding the nature of the services the server is providing.",
    resources: [
-      "https://www.netwrix.com/port_scanning.html",
-      "https://nmap.org/book/man-port-scanning-basics.html",
-      "https://www.cloudflare.com/learning/ddos/glossary/open-port/",
-      "https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers",
+      { title: 'List of TCP & UDP Port Numbers', link: 'https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers' },
+      { title: 'NMAP - Port Scanning Basics', link: 'https://nmap.org/book/man-port-scanning-basics.html' },
    ],
    screenshot: 'https://i.ibb.co/F8D1hmf/wc-ports.png',
  },
@ -210,12 +202,13 @@ const docs: Doc[] = [
      "This task calculates the estimated carbon footprint of a website. It's based on the amount of data being transferred and processed, and the energy usage of the servers that host and deliver the website. The larger the website and the more complex its features, the higher its carbon footprint is likely to be.",
    use: "From an OSINT perspective, understanding a website's carbon footprint doesn't directly provide insights into its internal workings or the organization behind it. However, it can still be valuable data in broader analyses, especially in contexts where environmental impact is a consideration. For example, it can be useful for activists, researchers, or ethical hackers who are interested in the sustainability of digital infrastructure, and who want to hold organizations accountable for their environmental impact.",
    resources: [
-      "https://www.websitecarbon.com/",
-      "https://www.thegreenwebfoundation.org/",
-      "https://www.nature.com/articles/s41598-020-76164-y",
-      "https://www.sciencedirect.com/science/article/pii/S0959652620307817",
+      { title: 'WebsiteCarbon - Carbon Calculator', link: 'https://www.websitecarbon.com/' },
+      { title: 'The Green Web Foundation', link: 'https://www.thegreenwebfoundation.org/' },
+      { title: 'The Eco Friendly Web Alliance', link: 'https://ecofriendlyweb.org/' },
+      { title: 'Reset.org', link: 'https://en.reset.org/' },
+      { title: 'Your website is killing the planet - via Wired', link: 'https://www.wired.co.uk/article/internet-carbon-footprint' },
    ],
-    screenshot: 'https://i.ibb.co/dmbFxjN/wc-carbon.png',
+    screenshot: 'https://i.ibb.co/5v6fSyw/Screenshot-from-2023-07-29-19-07-50.png',
  },
  {
    id: "server-info",
@ -261,7 +254,7 @@ const docs: Doc[] = [
    id: "dnssec",
    title: "DNS Security Extensions",
    description:
-      "Without DNSSEC, it\'s possible for MITM attackers to spoof records and lead users to phishing sites. This is because the DNS system includes no built-in methods to verify that the response to the request was not forged, or that any other part of the process wasn’t interrupted by an attacker. The DNS Security Extensions (DNSSEC) secures DNS lookups by signing your DNS records using public keys, so browsers can detect if the response has been tampered with. Another solution to this issue is DoH (DNS over HTTPS) and DoT (DNS over TLD).",
+      "Without DNSSEC, it's possible for MITM attackers to spoof records and lead users to phishing sites. This is because the DNS system includes no built-in methods to verify that the response to the request was not forged, or that any other part of the process wasn’t interrupted by an attacker. The DNS Security Extensions (DNSSEC) secures DNS lookups by signing your DNS records using public keys, so browsers can detect if the response has been tampered with. Another solution to this issue is DoH (DNS over HTTPS) and DoT (DNS over TLD).",
    use: "DNSSEC information provides insight into an organization's level of cybersecurity maturity and potential vulnerabilities, particularly around DNS spoofing and cache poisoning. If no DNS secururity (DNSSEC, DoH, DoT, etc) is implemented, this may provide an entry point for an attacker.",
    resources: [
      "https://dnssec-analyzer.verisignlabs.com/",
@ -288,7 +281,7 @@ const docs: Doc[] = [
    +'mechanism that helps protect websites against protocol downgrade attacks and '
    + 'cookie hijacking. A website can be included in the HSTS preload list by '
    + 'conforming to a set of requirements and then submitting itself to the list.',
-    use: `There are several reasons why it\'s important for a site to be HSTS enabled:
+    use: `There are several reasons why it's important for a site to be HSTS enabled:
  1. User bookmarks or manually types http://example.com and is subject to a man-in-the-middle attacker
    HSTS automatically redirects HTTP requests to HTTPS for the target domain
  2. Web application that is intended to be purely HTTPS inadvertently contains HTTP links or serves content over HTTP
@ -309,6 +302,7 @@ const docs: Doc[] = [
    description: 'This check takes a screenshot of webpage that the requested URL / IP resolves to, and displays it.',
    use: 'This may be useful to see what a given website looks like, free of the constraints of your browser, IP, or location.',
    resources: [],
+    screenshot: 'https://i.ibb.co/2F0x8kP/Screenshot-from-2023-07-29-18-34-48.png',
  },
  {
    id: 'dns-server',
@ -320,14 +314,14 @@ const docs: Doc[] = [
  {
    id: 'tech-stack',
    title: 'Tech Stack',
-    description: 'Checks what technologies a site is built with.'
+    description: 'Checks what technologies a site is built with. '
    + 'This is done by fetching and parsing the site, then comparing it against a bit list of RegEx maintained by Wappalyzer to identify the unique fingerprints that different technologies leave.',
    use: 'Identifying a website\'s tech stack aids in evaluating its security by exposing potential vulnerabilities, '
    + 'informs competitive analyses and development decisions, and can guide tailored marketing strategies. '
    + 'Ethical application of this knowledge is crucial to avoid harmful activities like data theft or unauthorized intrusion.',
    resources: [
-      'https://builtwith.com/',
-      'https://github.com/wappalyzer/wappalyzer/tree/master/src/technologies'
+      { title: 'Wappalyzer fingerprints', link: 'https://github.com/wappalyzer/wappalyzer/tree/master/src/technologies'},
+      { title: 'BuiltWith - Check what tech a site is using', link: 'https://builtwith.com/'},
    ],
  },
  {
@ -336,9 +330,9 @@ const docs: Doc[] = [
    description: 'This job finds and parses a site\'s listed sitemap. This file lists public sub-pages on the site, which the author wishes to be crawled by search engines. Sitemaps help with SEO, but are also useful for seeing all a sites public content at a glance.',
    use: 'Understand the structure of a site\'s public-facing content, and for site-owners, check that you\'re site\'s sitemap is accessible, parsable and contains everything you wish it to.',
    resources: [
-      'https://developers.google.com/search/docs/crawling-indexing/sitemaps/overview',
-      'https://www.sitemaps.org/protocol.html',
-      'https://www.conductor.com/academy/xml-sitemap/',
+      { title: 'Learn about Sitemaps', link: 'https://developers.google.com/search/docs/crawling-indexing/sitemaps/overview'},
+      { title: 'Sitemap XML spec', link: 'https://www.sitemaps.org/protocol.html'},
+      { title: 'Sitemap tutorial', link: 'https://www.conductor.com/academy/xml-sitemap/'},
    ],
    screenshot: 'https://i.ibb.co/GtrCQYq/Screenshot-from-2023-07-21-12-28-38.png',
  },
@ -362,6 +356,60 @@ const docs: Doc[] = [
    ],
    screenshot: 'https://i.ibb.co/tq1FT5r/Screenshot-from-2023-07-24-20-31-21.png',
  },
+  {
+    id: 'linked-pages',
+    title: 'Linked Pages',
+    description: 'Displays all internal and external links found on a site, identified by the href attributes attached to anchor elements.',
+    use: "For site owners, this is useful for diagnosing SEO issues, improving the site structure, understanding how content is inter-connected. External links can show partnerships, dependencies, and potential reputation risks. " +
+    "From a security standpoint, the outbound links can help identify any potential malicious or compromised sites the website is unknowingly linking to. Analyzing internal links can aid in understanding the site's structure and potentially uncover hidden or vulnerable pages which are not intended to be public. " +
+    "And for an OSINT investigator, it can aid in building a comprehensive understanding of the target, uncovering related entities, resources, or even potential hidden parts of the site.",
+    resources: [
+      { title: 'W3C Link Checker', link: 'https://validator.w3.org/checklink'},
+    ],
+    screenshot: 'https://i.ibb.co/LtK14XR/Screenshot-from-2023-07-29-11-16-44.png',
+  },
+  {
+    id: 'social-tags',
+    title: 'Social Tags',
+    description: 'Websites can include certain meta tags, that tell search engines and social media platforms what info to display. This usually includes a title, description, thumbnail, keywords, author, social accounts, etc.',
+    use: 'Adding this data to your site will boost SEO, and as an OSINT researcher it can be useful to understand how a given web app describes itself',
+    resources: [
+      { title: 'SocialSharePreview.com', link: 'https://socialsharepreview.com/'},
+      { title: 'The guide to social meta tags', link: 'https://css-tricks.com/essential-meta-tags-social-media/'},
+      { title: 'Web.dev metadata tags', link: 'https://web.dev/learn/html/metadata/'},
+      { title: 'Open Graph Protocol', link: 'https://ogp.me/'},
+      { title: 'Twitter Cards', link: 'https://developer.twitter.com/en/docs/twitter-for-websites/cards/overview/abouts-cards'},
+      { title: 'Facebook Open Graph', link: 'https://developers.facebook.com/docs/sharing/webmasters'},
+    ],
+    screenshot: 'https://i.ibb.co/4srTT1w/Screenshot-from-2023-07-29-11-15-27.png',
+  },
+  {
+    id: 'mail-config',
+    title: 'Email Configuration',
+    description: "DMARC (Domain-based Message Authentication, Reporting & Conformance): DMARC is an email authentication protocol that works with SPF and DKIM to prevent email spoofing and phishing. It allows domain owners to specify how to handle unauthenticated mail via a published policy in DNS, and provides a way for receiving mail servers to send feedback about emails' compliance to the sender. " +
+    "BIMI (Brand Indicators for Message Identification): BIMI is an emerging email standard that enables organizations to display a logo in their customers' email clients automatically. BIMI ties the logo to the domain's DMARC record, providing another level of visual assurance to recipients that the email is legitimate. " +
+    "DKIM (DomainKeys Identified Mail): DKIM is an email security standard designed to make sure that messages were not altered in transit between the sending and recipient servers. It uses digital signatures linked to the domain of the sender to verify the sender and ensure message integrity. " +
+    "SPF (Sender Policy Framework): SPF is an email authentication method designed to prevent email spoofing. It specifies which mail servers are authorized to send email on behalf of a domain by creating a DNS record. This helps protect against spam by providing a way for receiving mail servers to check that incoming mail from a domain comes from a host authorized by that domain's administrators.",
+    use: "This information is helpful for researchers as it helps assess a domain's email security posture, uncover potential vulnerabilities, and verify the legitimacy of emails for phishing detection. These details can also provide insight into the hosting environment, potential service providers, and the configuration patterns of a target organization, assisting in investigative efforts.",
+    resources: [
+      { title: 'Intro to DMARC, DKIM, and SPF (via Cloudflare)', link: 'https://www.cloudflare.com/learning/email-security/dmarc-dkim-spf/' },
+      { title: 'EasyDMARC Domain Scanner', link: 'https://easydmarc.com/tools/domain-scanner' },
+      { title: 'MX Toolbox', link: 'https://mxtoolbox.com/' },
+      { title: 'RFC-7208 - SPF', link: 'https://datatracker.ietf.org/doc/html/rfc7208' },
+      { title: 'RFC-6376 - DKIM', link: 'https://datatracker.ietf.org/doc/html/rfc6376' },
+      { title: 'RFC-7489 - DMARC', link: 'https://datatracker.ietf.org/doc/html/rfc7489' },
+      { title: 'BIMI Group', link: 'https://bimigroup.org/' },
+    ],
+    screenshot: 'https://i.ibb.co/yqhwx5G/Screenshot-from-2023-07-29-18-22-20.png',
+  },
+  // {
+  //   id: '',
+  //   title: '',
+  //   description: '',
+  //   use: '',
+  //   resources: [],
+  //   screenshot: '',
+  // },
 ];

 export const about = [
@ -409,10 +457,18 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 `;

+export const supportUs = [
+  "The hosted app is free to use without restriction. All the code is open source, so you're also free to deploy your own instance, or make any modifications.",
+  "Running web-check does cost me a small amount of money each month, so if you're finding the app useful, consider <a href='https://github.com/sponsors/Lissy93'>sponsoring me on GitHub</a> if you're able to. Even just $1 or $2/month would be a huge help in supporting the ongoing project running costs.",
+  "Otherwise, there are other ways you can help out, like submitting or reviewing a pull request to the <a href='https://github.com/Lissy93/web-check'>GitHub repo</a>, upvoting us on <a href='https://www.producthunt.com/posts/web-check'>Product Hunt</a>, or by sharing with your network.",
+  "But don't feel obliged to do anything, as this app (and all my other projects) will always remain 100% free and open source, and I will do my best to ensure the managed instances remain up and available for as long as possible :)",
+];
+
 export const fairUse = [
-  'Please use this tool responsibly. Do not use it for hosts you do not have permission to scan. Do not use it to attack or disrupt services.',
-  'Requests are rate-limited to prevent abuse. If you need to make more bandwidth, please deploy your own instance.',
-  'The hosted instance is only for demo use, as excessive use will quickly deplete my lambda function credits, making it unavailable for others and/or costing me money.',
+  'Please use this tool responsibly. Do not use it for hosts you do not have permission to scan. Do not use it as part of a scheme to attack or disrupt services.',
+  'Requests may be rate-limited to prevent abuse. If you need to make more bandwidth, please deploy your own instance.',
+  'There is no guarantee of uptime or availability. If you need to make sure the service is available, please deploy your own instance.',
+  'Please use fairly, as excessive use will quickly deplete the lambda function credits, making the service unavailable for others (and/or empty my bank account!).',
 ];

 export default docs;