From 93ed8d6c4423b9b7e32c31dc9cf129a5cee8f887 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Fri, 18 Aug 2023 19:33:49 +0100 Subject: [PATCH 1/5] Working on lambda function for malware, phishing and viruses check --- api/malware.js | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 api/malware.js diff --git a/api/malware.js b/api/malware.js new file mode 100644 index 0000000..3b4e226 --- /dev/null +++ b/api/malware.js @@ -0,0 +1,61 @@ +const axios = require('axios'); +const xml2js = require('xml2js'); +const middleware = require('./_common/middleware'); + +const getUrlHausResult = async (url) => { + let domain = new URL(url).hostname; + return await axios({ + method: 'post', + url: 'https://urlhaus-api.abuse.ch/v1/host/', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded' + }, + data: `host=${domain}` + }) + .then((x) => x.data) + .catch((e) => ({ error: `Request to URLHaus failed, ${e.message}`})); +}; + + +const getPhishTankResult = async (url) => { + try { + const encodedUrl = Buffer.from(url).toString('base64'); + const endpoint = `https://checkurl.phishtank.com/checkurl/?url=${encodedUrl}`; + const headers = { + 'User-Agent': 'phishtank/web-check', + }; + const response = await axios.post(endpoint, null, { headers, timeout: 3000 }); + const parsed = await xml2js.parseStringPromise(response.data, { explicitArray: false }); + return parsed.response.results; + } catch (error) { + return { error: `Request to PhishTank failed: ${error.message}` }; + } +} + +const getCloudmersiveResult = async (url) => { + try { + const endpoint = 'https://api.cloudmersive.com/virus/scan/website'; + const headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Apikey': process.env.CLOUDMERSIVE_API_KEY, + }; + const data = `Url=${encodeURIComponent(url)}`; + const response = await axios.post(endpoint, data, { headers }); + return response.data; + } catch (error) { + return { error: `Request to Cloudmersive failed: ${error.message}` }; + } +}; + +const handler = async (url) => { + try { + const urlHaus = await getUrlHausResult(url); + const phishTank = await getPhishTankResult(url); + const cloudmersive = await getCloudmersiveResult(url); + return JSON.stringify({ urlHaus, phishTank, cloudmersive }); + } catch (error) { + throw new Error(error.message); + } +}; + +exports.handler = middleware(handler); From 83c8d311b30660696046a6aac5547d061f3521b5 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Fri, 18 Aug 2023 19:34:06 +0100 Subject: [PATCH 2/5] UI code for malware checking --- src/components/Results/Malware.tsx | 70 +++++++++++++++++++++++++++++ src/components/misc/ProgressBar.tsx | 1 + src/pages/Results.tsx | 10 +++++ src/utils/docs.ts | 11 +++++ 4 files changed, 92 insertions(+) create mode 100644 src/components/Results/Malware.tsx diff --git a/src/components/Results/Malware.tsx b/src/components/Results/Malware.tsx new file mode 100644 index 0000000..1b681a8 --- /dev/null +++ b/src/components/Results/Malware.tsx @@ -0,0 +1,70 @@ + +import styled from 'styled-components'; +import colors from 'styles/colors'; +import { Card } from 'components/Form/Card'; +import Row, { ExpandableRow } from 'components/Form/Row'; + +const Expandable = styled.details` +margin-top: 0.5rem; +cursor: pointer; +summary::marker { + color: ${colors.primary}; +} +`; + +const getExpandableTitle = (urlObj: any) => { + let pathName = ''; + try { + pathName = new URL(urlObj.url).pathname; + } catch(e) {} + return `${pathName} (${urlObj.id})`; +} + +const convertToDate = (dateString: string): string => { + const [date, time] = dateString.split(' '); + const [year, month, day] = date.split('-').map(Number); + const [hour, minute, second] = time.split(':').map(Number); + const dateObject = new Date(year, month - 1, day, hour, minute, second); + if (isNaN(dateObject.getTime())) { + return dateString; + } + return dateObject.toString(); +} + +const MalwareCard = (props: {data: any, title: string, actionButtons: any }): JSX.Element => { + const urlHaus = props.data.urlHaus; + return ( + + { urlHaus.query_status === 'no_results' && } + { urlHaus.query_status === 'ok' && ( + <> + + + + + )} + {urlHaus.urls && ( + + Expand Results + { urlHaus.urls.map((urlResult: any, index: number) => { + const rows = [ + { lbl: 'ID', val: urlResult.id }, + { lbl: 'Status', val: urlResult.url_status }, + { lbl: 'Date Added', val: convertToDate(urlResult.date_added) }, + { lbl: 'Threat Type', val: urlResult.threat }, + { lbl: 'Reported By', val: urlResult.reporter }, + { lbl: 'Takedown Time', val: urlResult.takedown_time_seconds }, + { lbl: 'Larted', val: urlResult.larted }, + { lbl: 'Tags', val: (urlResult.tags || []).join(', ') }, + { lbl: 'Reference', val: urlResult.urlhaus_reference }, + { lbl: 'File Path', val: urlResult.url }, + ]; + return () + })} + + )} + + ); +} + +export default MalwareCard; diff --git a/src/components/misc/ProgressBar.tsx b/src/components/misc/ProgressBar.tsx index 210d342..0164dfc 100644 --- a/src/components/misc/ProgressBar.tsx +++ b/src/components/misc/ProgressBar.tsx @@ -218,6 +218,7 @@ const jobNames = [ 'rank', 'archives', 'block-lists', + 'malware', ] as const; export const initialJobs = jobNames.map((job: string) => { diff --git a/src/pages/Results.tsx b/src/pages/Results.tsx index f0d5b72..a8027a4 100644 --- a/src/pages/Results.tsx +++ b/src/pages/Results.tsx @@ -53,6 +53,7 @@ import FirewallCard from 'components/Results/Firewall'; import ArchivesCard from 'components/Results/Archives'; import RankCard from 'components/Results/Rank'; import BlockListsCard from 'components/Results/BlockLists'; +import MalwareCard from 'components/Results/Malware'; import keys from 'utils/get-keys'; import { determineAddressType, AddressType } from 'utils/address-type-checker'; @@ -448,6 +449,14 @@ const Results = (): JSX.Element => { fetchRequest: () => fetch(`${api}/block-lists?url=${address}`).then(res => parseJson(res)), }); + // Check if a host is present on the URLHaus malware list + const [malwareResults, updateMalwareResults] = useMotherHook({ + jobId: 'malware', + updateLoadingJobs, + addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, + fetchRequest: () => fetch(`${api}/malware?url=${address}`).then(res => parseJson(res)), + }); + /* Cancel remaining jobs after 10 second timeout */ useEffect(() => { const checkJobs = () => { @@ -503,6 +512,7 @@ const Results = (): JSX.Element => { { id: 'linked-pages', title: 'Linked Pages', result: linkedPagesResults, Component: ContentLinksCard, refresh: updateLinkedPagesResults }, { id: 'txt-records', title: 'TXT Records', result: txtRecordResults, Component: TxtRecordCard, refresh: updateTxtRecordResults }, { id: 'block-lists', title: 'Block Lists', result: blockListsResults, Component: BlockListsCard, refresh: updateBlockListsResults }, + { id: 'malware', title: 'Malware', result: malwareResults, Component: MalwareCard, refresh: updateMalwareResults }, { id: 'features', title: 'Site Features', result: siteFeaturesResults, Component: SiteFeaturesCard, refresh: updateSiteFeaturesResults }, { id: 'sitemap', title: 'Pages', result: sitemapResults, Component: SitemapCard, refresh: updateSitemapResults }, { id: 'carbon', title: 'Carbon Footprint', result: carbonResults, Component: CarbonFootprintCard, refresh: updateCarbonResults }, diff --git a/src/utils/docs.ts b/src/utils/docs.ts index 9db788c..ea4cbd8 100644 --- a/src/utils/docs.ts +++ b/src/utils/docs.ts @@ -470,6 +470,17 @@ const docs: Doc[] = [ resources: [], screenshot: '', }, + { + id: 'malware', + title: 'Malware & Phishing Detection', + description: '', + use: '', + resources: [ + { title: 'URLHaus', link: 'https://urlhaus-api.abuse.ch/'}, + { title: 'PhishTank', link: 'https://www.phishtank.com/'}, + ], + screenshot: '', + }, // { // id: '', // title: '', From 759bb603df0aec3d1777ffa196e05aee5a0a2992 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sat, 19 Aug 2023 22:00:15 +0100 Subject: [PATCH 3/5] Adds functionality for threats, malware, phishing, viruses --- api/{malware.js => threats.js} | 3 +++ .../Results/{Malware.tsx => Threats.tsx} | 14 ++++++++++++++ src/components/misc/ProgressBar.tsx | 2 +- src/pages/Results.tsx | 10 +++++----- 4 files changed, 23 insertions(+), 6 deletions(-) rename api/{malware.js => threats.js} (91%) rename src/components/Results/{Malware.tsx => Threats.tsx} (78%) diff --git a/api/malware.js b/api/threats.js similarity index 91% rename from api/malware.js rename to api/threats.js index 3b4e226..edc987c 100644 --- a/api/malware.js +++ b/api/threats.js @@ -52,6 +52,9 @@ const handler = async (url) => { const urlHaus = await getUrlHausResult(url); const phishTank = await getPhishTankResult(url); const cloudmersive = await getCloudmersiveResult(url); + if (urlHaus.error && phishTank.error && cloudmersive.error) { + throw new Error(`All requests failed - ${urlHaus.error} ${phishTank.error} ${cloudmersive.error}`); + } return JSON.stringify({ urlHaus, phishTank, cloudmersive }); } catch (error) { throw new Error(error.message); diff --git a/src/components/Results/Malware.tsx b/src/components/Results/Threats.tsx similarity index 78% rename from src/components/Results/Malware.tsx rename to src/components/Results/Threats.tsx index 1b681a8..a892eaf 100644 --- a/src/components/Results/Malware.tsx +++ b/src/components/Results/Threats.tsx @@ -33,8 +33,22 @@ const convertToDate = (dateString: string): string => { const MalwareCard = (props: {data: any, title: string, actionButtons: any }): JSX.Element => { const urlHaus = props.data.urlHaus; + const phishTank = props.data.phishTank; + const cloudmersive = props.data.cloudmersive; return ( + { cloudmersive && !cloudmersive.error && ( + + )} + { phishTank && !phishTank.error && ( + + )} + { phishTank.url0 && phishTank.url0.phish_detail_page && ( + + Phish Info + {phishTank.url0.phish_id} + + )} { urlHaus.query_status === 'no_results' && } { urlHaus.query_status === 'ok' && ( <> diff --git a/src/components/misc/ProgressBar.tsx b/src/components/misc/ProgressBar.tsx index 0164dfc..81eb98e 100644 --- a/src/components/misc/ProgressBar.tsx +++ b/src/components/misc/ProgressBar.tsx @@ -218,7 +218,7 @@ const jobNames = [ 'rank', 'archives', 'block-lists', - 'malware', + 'threats', ] as const; export const initialJobs = jobNames.map((job: string) => { diff --git a/src/pages/Results.tsx b/src/pages/Results.tsx index a8027a4..69cb682 100644 --- a/src/pages/Results.tsx +++ b/src/pages/Results.tsx @@ -53,7 +53,7 @@ import FirewallCard from 'components/Results/Firewall'; import ArchivesCard from 'components/Results/Archives'; import RankCard from 'components/Results/Rank'; import BlockListsCard from 'components/Results/BlockLists'; -import MalwareCard from 'components/Results/Malware'; +import ThreatsCard from 'components/Results/Threats'; import keys from 'utils/get-keys'; import { determineAddressType, AddressType } from 'utils/address-type-checker'; @@ -450,11 +450,11 @@ const Results = (): JSX.Element => { }); // Check if a host is present on the URLHaus malware list - const [malwareResults, updateMalwareResults] = useMotherHook({ - jobId: 'malware', + const [threatResults, updateThreatResults] = useMotherHook({ + jobId: 'threats', updateLoadingJobs, addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, - fetchRequest: () => fetch(`${api}/malware?url=${address}`).then(res => parseJson(res)), + fetchRequest: () => fetch(`${api}/threats?url=${address}`).then(res => parseJson(res)), }); /* Cancel remaining jobs after 10 second timeout */ @@ -512,7 +512,7 @@ const Results = (): JSX.Element => { { id: 'linked-pages', title: 'Linked Pages', result: linkedPagesResults, Component: ContentLinksCard, refresh: updateLinkedPagesResults }, { id: 'txt-records', title: 'TXT Records', result: txtRecordResults, Component: TxtRecordCard, refresh: updateTxtRecordResults }, { id: 'block-lists', title: 'Block Lists', result: blockListsResults, Component: BlockListsCard, refresh: updateBlockListsResults }, - { id: 'malware', title: 'Malware', result: malwareResults, Component: MalwareCard, refresh: updateMalwareResults }, + { id: 'threats', title: 'Threats', result: threatResults, Component: ThreatsCard, refresh: updateThreatResults }, { id: 'features', title: 'Site Features', result: siteFeaturesResults, Component: SiteFeaturesCard, refresh: updateSiteFeaturesResults }, { id: 'sitemap', title: 'Pages', result: sitemapResults, Component: SitemapCard, refresh: updateSitemapResults }, { id: 'carbon', title: 'Carbon Footprint', result: carbonResults, Component: CarbonFootprintCard, refresh: updateCarbonResults }, From 8ca747c02f990e4081e6d2722ca7bbf04da389f9 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sat, 19 Aug 2023 23:36:47 +0100 Subject: [PATCH 4/5] Adds support for Google Safe Browsing in threats API endpoint --- api/threats.js | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/api/threats.js b/api/threats.js index edc987c..7d87ce1 100644 --- a/api/threats.js +++ b/api/threats.js @@ -2,6 +2,36 @@ const axios = require('axios'); const xml2js = require('xml2js'); const middleware = require('./_common/middleware'); +const getGoogleSafeBrowsingResult = async (url) => { + try { + const apiKey = process.env.GOOGLE_CLOUD_API_KEY; + const apiEndpoint = `https://safebrowsing.googleapis.com/v4/threatMatches:find?key=${apiKey}`; + + const requestBody = { + threatInfo: { + threatTypes: [ + 'MALWARE', 'SOCIAL_ENGINEERING', 'UNWANTED_SOFTWARE', 'POTENTIALLY_HARMFUL_APPLICATION', 'API_ABUSE' + ], + platformTypes: ["ANY_PLATFORM"], + threatEntryTypes: ["URL"], + threatEntries: [{ url }] + } + }; + + const response = await axios.post(apiEndpoint, requestBody); + if (response.data && response.data.matches) { + return { + unsafe: true, + details: response.data.matches + }; + } else { + return { unsafe: false }; + } + } catch (error) { + return { error: `Request failed: ${error.message}` }; + } +}; + const getUrlHausResult = async (url) => { let domain = new URL(url).hostname; return await axios({ @@ -52,10 +82,11 @@ const handler = async (url) => { const urlHaus = await getUrlHausResult(url); const phishTank = await getPhishTankResult(url); const cloudmersive = await getCloudmersiveResult(url); - if (urlHaus.error && phishTank.error && cloudmersive.error) { - throw new Error(`All requests failed - ${urlHaus.error} ${phishTank.error} ${cloudmersive.error}`); + const safeBrowsing = await getGoogleSafeBrowsingResult(url); + if (urlHaus.error && phishTank.error && cloudmersive.error && safeBrowsing.error) { + throw new Error(`All requests failed - ${urlHaus.error} ${phishTank.error} ${cloudmersive.error} ${safeBrowsing.error}`); } - return JSON.stringify({ urlHaus, phishTank, cloudmersive }); + return JSON.stringify({ urlHaus, phishTank, cloudmersive, safeBrowsing }); } catch (error) { throw new Error(error.message); } From 737639ae846ab84d774bcffcac0cebe5d6f475bb Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Sat, 19 Aug 2023 23:38:22 +0100 Subject: [PATCH 5/5] Updates the UI for threats from Google Safe Browsing --- src/components/Results/Threats.tsx | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/components/Results/Threats.tsx b/src/components/Results/Threats.tsx index a892eaf..1a77123 100644 --- a/src/components/Results/Threats.tsx +++ b/src/components/Results/Threats.tsx @@ -32,13 +32,17 @@ const convertToDate = (dateString: string): string => { } const MalwareCard = (props: {data: any, title: string, actionButtons: any }): JSX.Element => { - const urlHaus = props.data.urlHaus; - const phishTank = props.data.phishTank; - const cloudmersive = props.data.cloudmersive; + const urlHaus = props.data.urlHaus || {}; + const phishTank = props.data.phishTank || {}; + const cloudmersive = props.data.cloudmersive || {}; + const safeBrowsing = props.data.safeBrowsing || {}; return ( - { cloudmersive && !cloudmersive.error && ( - + { safeBrowsing && !safeBrowsing.error && ( + + )} + { ((cloudmersive && !cloudmersive.error) || safeBrowsing?.details) && ( + )} { phishTank && !phishTank.error && (