Merge pull request #37 from Lissy93/FEAT/malware-checking

Feat/malware checking

api/threats.js

@@ -0,0 +1,95 @@
+const axios = require('axios');
+const xml2js = require('xml2js');
+const middleware = require('./_common/middleware');
+
+const getGoogleSafeBrowsingResult = async (url) => {
+  try {
+    const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
+    const apiEndpoint = `https://safebrowsing.googleapis.com/v4/threatMatches:find?key=${apiKey}`;
+
+    const requestBody = {
+      threatInfo: {
+        threatTypes: [
+          'MALWARE', 'SOCIAL_ENGINEERING', 'UNWANTED_SOFTWARE', 'POTENTIALLY_HARMFUL_APPLICATION', 'API_ABUSE'
+        ],
+        platformTypes: ["ANY_PLATFORM"],
+        threatEntryTypes: ["URL"],
+        threatEntries: [{ url }]
+      }
+    };
+
+    const response = await axios.post(apiEndpoint, requestBody);
+    if (response.data && response.data.matches) {
+      return {
+        unsafe: true,
+        details: response.data.matches
+      };
+    } else {
+      return { unsafe: false };
+    }
+  } catch (error) {
+    return { error: `Request failed: ${error.message}` };
+  }
+};
+
+const getUrlHausResult = async (url) => {
+  let domain = new URL(url).hostname;
+  return await axios({
+    method: 'post',
+    url: 'https://urlhaus-api.abuse.ch/v1/host/',
+    headers: {
+      'Content-Type': 'application/x-www-form-urlencoded'
+    },
+    data: `host=${domain}`
+  })
+  .then((x) => x.data)
+  .catch((e) => ({ error: `Request to URLHaus failed, ${e.message}`}));
+};
+
+
+const getPhishTankResult = async (url) => {
+  try {
+    const encodedUrl = Buffer.from(url).toString('base64');
+    const endpoint = `https://checkurl.phishtank.com/checkurl/?url=${encodedUrl}`;
+    const headers = {
+      'User-Agent': 'phishtank/web-check',
+    };
+    const response = await axios.post(endpoint, null, { headers, timeout: 3000 });
+    const parsed = await xml2js.parseStringPromise(response.data, { explicitArray: false });
+    return parsed.response.results;
+  } catch (error) {
+    return { error: `Request to PhishTank failed: ${error.message}` };
+  }
+}
+
+const getCloudmersiveResult = async (url) => {
+  try {
+    const endpoint = 'https://api.cloudmersive.com/virus/scan/website';
+    const headers = {
+      'Content-Type': 'application/x-www-form-urlencoded',
+      'Apikey': process.env.CLOUDMERSIVE_API_KEY,
+    };
+    const data = `Url=${encodeURIComponent(url)}`;
+    const response = await axios.post(endpoint, data, { headers });
+    return response.data;
+  } catch (error) {
+    return { error: `Request to Cloudmersive failed: ${error.message}` };
+  }
+};
+
+const handler = async (url) => {
+  try {
+    const urlHaus = await getUrlHausResult(url);
+    const phishTank = await getPhishTankResult(url);
+    const cloudmersive = await getCloudmersiveResult(url);
+    const safeBrowsing = await getGoogleSafeBrowsingResult(url);
+    if (urlHaus.error && phishTank.error && cloudmersive.error && safeBrowsing.error) {
+      throw new Error(`All requests failed - ${urlHaus.error} ${phishTank.error} ${cloudmersive.error} ${safeBrowsing.error}`);
+    }
+    return JSON.stringify({ urlHaus, phishTank, cloudmersive, safeBrowsing });
+  } catch (error) {
+    throw new Error(error.message);
+  }
+};
+
+exports.handler = middleware(handler);

src/components/Results/Threats.tsx

@@ -0,0 +1,88 @@
+
+import styled from 'styled-components';
+import colors from 'styles/colors';
+import { Card } from 'components/Form/Card';
+import Row, { ExpandableRow } from 'components/Form/Row';
+
+const Expandable = styled.details`
+margin-top: 0.5rem;
+cursor: pointer;
+summary::marker {
+  color: ${colors.primary};
+}
+`;
+
+const getExpandableTitle = (urlObj: any) => {
+  let pathName = '';
+  try {
+    pathName = new URL(urlObj.url).pathname;
+  } catch(e) {}
+  return `${pathName} (${urlObj.id})`;
+}
+
+const convertToDate = (dateString: string): string => {
+  const [date, time] = dateString.split(' ');
+  const [year, month, day] = date.split('-').map(Number);
+  const [hour, minute, second] = time.split(':').map(Number);
+  const dateObject = new Date(year, month - 1, day, hour, minute, second);
+  if (isNaN(dateObject.getTime())) {
+    return dateString;
+  }
+  return dateObject.toString();
+}
+
+const MalwareCard = (props: {data: any, title: string, actionButtons: any }): JSX.Element => {
+  const urlHaus = props.data.urlHaus || {};
+  const phishTank = props.data.phishTank || {};
+  const cloudmersive = props.data.cloudmersive || {};
+  const safeBrowsing = props.data.safeBrowsing || {};
+  return (
+    <Card heading={props.title} actionButtons={props.actionButtons}>
+      { safeBrowsing && !safeBrowsing.error && (
+        <Row lbl="Google Safe Browsing" val={safeBrowsing.unsafe ? '❌ Unsafe' : '✅ Safe'} />
+      )}
+      { ((cloudmersive && !cloudmersive.error) || safeBrowsing?.details) && (
+        <Row lbl="Threat Type" val={safeBrowsing?.details?.threatType || cloudmersive.WebsiteThreatType || 'None :)'} />
+      )}
+      { phishTank && !phishTank.error && (
+        <Row lbl="Phishing Status" val={phishTank.url0 ? '❌ Phishing Identified' : '✅ No Phishing Identified!'} />
+      )}
+      { phishTank.url0 && phishTank.url0.phish_detail_page && (
+        <Row lbl="" val="">
+          <span className="lbl">Phish Info</span>
+          <span className="val"><a href={phishTank.url0.phish_detail_page}>{phishTank.url0.phish_id}</a></span>  
+        </Row>
+      )}
+      { urlHaus.query_status === 'no_results' && <Row lbl="Status" val="✅ Nothing Found!" />}
+      { urlHaus.query_status === 'ok' && (
+        <>
+        <Row lbl="Status" val="❌ Malware Found" />
+        <Row lbl="First Seen" val={convertToDate(urlHaus.firstseen)} />
+        <Row lbl="Bad URLs Count" val={urlHaus.url_count} />
+        </>
+      )}
+      {urlHaus.urls && (
+        <Expandable>
+          <summary>Expand Results</summary>
+          { urlHaus.urls.map((urlResult: any, index: number) => {
+          const rows = [
+            { lbl: 'ID', val: urlResult.id },
+            { lbl: 'Status', val: urlResult.url_status },
+            { lbl: 'Date Added', val: convertToDate(urlResult.date_added) },
+            { lbl: 'Threat Type', val: urlResult.threat },
+            { lbl: 'Reported By', val: urlResult.reporter },
+            { lbl: 'Takedown Time', val: urlResult.takedown_time_seconds },
+            { lbl: 'Larted', val: urlResult.larted },
+            { lbl: 'Tags', val: (urlResult.tags || []).join(', ') },
+            { lbl: 'Reference', val: urlResult.urlhaus_reference },      
+            { lbl: 'File Path', val: urlResult.url },      
+          ];
+          return (<ExpandableRow lbl={getExpandableTitle(urlResult)} val="" rowList={rows} />)
+        })}
+        </Expandable>
+      )}
+    </Card>
+  );
+}
+
+export default MalwareCard;

src/components/misc/ProgressBar.tsx

@@ -218,6 +218,7 @@ const jobNames = [
   'rank',
   'archives',
   'block-lists',
+  'threats',
 ] as const;
 
 export const initialJobs = jobNames.map((job: string) => {

src/pages/Results.tsx

@@ -53,6 +53,7 @@ import FirewallCard from 'components/Results/Firewall';
 import ArchivesCard from 'components/Results/Archives';
 import RankCard from 'components/Results/Rank';
 import BlockListsCard from 'components/Results/BlockLists';
+import ThreatsCard from 'components/Results/Threats';
 
 import keys from 'utils/get-keys';
 import { determineAddressType, AddressType } from 'utils/address-type-checker';
@@ -448,6 +449,14 @@ const Results = (): JSX.Element => {
     fetchRequest: () => fetch(`${api}/block-lists?url=${address}`).then(res => parseJson(res)),
   });
 
+  // Check if a host is present on the URLHaus malware list
+  const [threatResults, updateThreatResults] = useMotherHook({
+    jobId: 'threats',
+    updateLoadingJobs,
+    addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
+    fetchRequest: () => fetch(`${api}/threats?url=${address}`).then(res => parseJson(res)),
+  });
+
   /* Cancel remaining jobs after  10 second timeout */
   useEffect(() => {
     const checkJobs = () => {
@@ -503,6 +512,7 @@ const Results = (): JSX.Element => {
     { id: 'linked-pages', title: 'Linked Pages', result: linkedPagesResults, Component: ContentLinksCard, refresh: updateLinkedPagesResults },
     { id: 'txt-records', title: 'TXT Records', result: txtRecordResults, Component: TxtRecordCard, refresh: updateTxtRecordResults },    
     { id: 'block-lists', title: 'Block Lists', result: blockListsResults, Component: BlockListsCard, refresh: updateBlockListsResults },    
+    { id: 'threats', title: 'Threats', result: threatResults, Component: ThreatsCard, refresh: updateThreatResults },    
     { id: 'features', title: 'Site Features', result: siteFeaturesResults, Component: SiteFeaturesCard, refresh: updateSiteFeaturesResults },
     { id: 'sitemap', title: 'Pages', result: sitemapResults, Component: SitemapCard, refresh: updateSitemapResults },
     { id: 'carbon', title: 'Carbon Footprint', result: carbonResults, Component: CarbonFootprintCard, refresh: updateCarbonResults },

src/utils/docs.ts

@@ -470,6 +470,17 @@ const docs: Doc[] = [
     resources: [],
     screenshot: '',
   },
+  {
+    id: 'malware',
+    title: 'Malware & Phishing Detection',
+    description: '',
+    use: '',
+    resources: [
+      { title: 'URLHaus', link: 'https://urlhaus-api.abuse.ch/'},
+      { title: 'PhishTank', link: 'https://www.phishtank.com/'},
+    ],
+    screenshot: '',
+  },
   // {
   //   id: '',
   //   title: '',