Implements global page rank widget

api/legacy-rank.js

@@ -0,0 +1,70 @@
+const axios = require('axios');
+const unzipper = require('unzipper');
+const csv = require('csv-parser');
+const fs = require('fs');
+const middleware = require('./_common/middleware');
+
+// Should also work with the following sources:
+// https://www.domcop.com/files/top/top10milliondomains.csv.zip
+// https://tranco-list.eu/top-1m.csv.zip
+// https://www.domcop.com/files/top/top10milliondomains.csv.zip
+// https://radar.cloudflare.com/charts/LargerTopDomainsTable/attachment?id=525&top=1000000
+// https://statvoo.com/dl/top-1million-sites.csv.zip
+
+const FILE_URL = 'https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip';
+const TEMP_FILE_PATH = '/tmp/top-1m.csv';
+
+const handler = async (url) => {
+  let domain = null;
+
+  try {
+    domain = new URL(url).hostname;
+  } catch (e) {
+    throw new Error('Invalid URL');
+  }
+
+// Download and unzip the file if not in cache
+if (!fs.existsSync(TEMP_FILE_PATH)) {
+  const response = await axios({
+    method: 'GET',
+    url: FILE_URL,
+    responseType: 'stream'
+  });
+
+  await new Promise((resolve, reject) => {
+    response.data
+      .pipe(unzipper.Extract({ path: '/tmp' }))
+      .on('close', resolve)
+      .on('error', reject);
+  });
+}
+
+// Parse the CSV and find the rank
+return new Promise((resolve, reject) => {
+  const csvStream = fs.createReadStream(TEMP_FILE_PATH)
+    .pipe(csv({
+      headers: ['rank', 'domain'],
+    }))
+    .on('data', (row) => {
+      if (row.domain === domain) {
+        csvStream.destroy();
+        resolve({
+          domain: domain,
+          rank: row.rank,
+          isFound: true,
+        });
+      }
+    })
+    .on('end', () => {
+      resolve({
+        skipped: `Skipping, as ${domain} is not present in the Umbrella top 1M list.`,
+        domain: domain,
+        isFound: false,
+      });
+    })
+    .on('error', reject);
+});
+};
+
+exports.handler = middleware(handler);
+

api/rank.js

@@ -0,0 +1,24 @@
+const axios = require('axios');
+const middleware = require('./_common/middleware');
+
+const handler = async (url) => { 
+  const domain = url ? new URL(url).hostname : null;
+  if (!domain) throw new Error('Invalid URL');
+
+  try {
+    const auth = process.env.TRANCO_API_KEY ? // Auth is optional.
+      { auth: { username: process.env.TRANCO_USERNAME, password: process.env.TRANCO_API_KEY } }
+      : {};
+    const response = await axios.get(
+      `https://tranco-list.eu/api/ranks/domain/${domain}`, { timeout: 2000 }, auth,
+      );
+    if (!response.data || !response.data.ranks || response.data.ranks.length === 0) {
+      return { skipped: `Skipping, as ${domain} isn't ranked in the top 100 million sites yet.`};
+    }
+    return response.data;
+  } catch (error) {
+    return { error: `Unable to fetch rank, ${error.message}` };
+  }
+};
+
+exports.handler = middleware(handler);

src/components/Results/Rank.tsx

@@ -0,0 +1,90 @@
+
+import { AreaChart, Area, Tooltip, CartesianGrid, ResponsiveContainer } from 'recharts';
+import colors from 'styles/colors';
+import { Card } from 'components/Form/Card';
+import Row from 'components/Form/Row';
+
+const cardStyles = `
+span.val {
+  &.up { color: ${colors.success}; }
+  &.down { color: ${colors.danger}; }
+}
+.rank-average {
+  text-align: center;
+  font-size: 1.8rem;
+  font-weight: bold;
+}
+.chart-container {
+  margin-top: 1rem;
+}
+`;
+
+const makeRankStats = (data: {date: string, rank: number }[]) => {
+  const average = Math.round(data.reduce((acc, cur) => acc + cur.rank, 0) / data.length);
+  const today = data[0].rank;
+  const yesterday = data[1].rank;
+  const percentageChange = ((today - yesterday) / yesterday) * 100;
+  return {
+    average,
+    percentageChange
+  };
+};
+
+const makeChartData = (data: {date: string, rank: number }[]) => {
+  return data.map((d) => {
+    return {
+      date: d.date,
+      uv: d.rank
+    };
+  });
+};
+
+const RankCard = (props: { data: any, title: string, actionButtons: any }): JSX.Element => {
+  const data = props.data.ranks || [];
+  const { average, percentageChange } = makeRankStats(data);
+  const chartData = makeChartData(data);
+  return (
+    <Card heading={props.title} actionButtons={props.actionButtons} styles={cardStyles}>
+      <div className="rank-average">{data[0].rank.toLocaleString()}</div>
+      <Row lbl="Change since Yesterday" val={`${percentageChange > 0 ? '+':''} ${percentageChange.toFixed(2)}%`} />
+      <Row lbl="Historical Average Rank" val={average.toLocaleString()} />
+      <div className="chart-container">
+        <ResponsiveContainer width="100%" height={100}>
+          <AreaChart width={400} height={100} data={chartData}>
+            <defs>
+              <linearGradient id="colorUv" x1="0" y1="0" x2="0" y2="1">
+                <stop offset="20%" stopColor="#0f1620" stopOpacity={0.8}/>
+                <stop offset="80%" stopColor="#0f1620" stopOpacity={0}/>
+              </linearGradient>
+            </defs>
+            <CartesianGrid strokeDasharray="4" strokeWidth={0.25} verticalPoints={[50,100,150,200,250,300,350]} horizontalPoints={[25,50,75]} />
+            <Tooltip contentStyle={{background: colors.background, color: colors.textColor, borderRadius: 4 }}
+                labelFormatter={(value) => ['Date : ', data[value].date]}/>
+            <Area type="monotone" dataKey="uv" stroke="#9fef00" fillOpacity={1} name="Rank" fill={`${colors.backgroundDarker}a1`} />
+          </AreaChart>
+        </ResponsiveContainer>
+      </div>
+
+      {/* { domain.Domain_Name && <Row lbl="Registered Domain" val={domain.Domain_Name} /> }
+      { domain.Creation_Date && <Row lbl="Creation Date" val={domain.Creation_Date} /> }
+      { domain.Updated_Date && <Row lbl="Updated Date" val={domain.Updated_Date} /> }
+      { domain.Registry_Expiry_Date && <Row lbl="Registry Expiry Date" val={domain.Registry_Expiry_Date} /> }
+      { domain.Registry_Domain_ID && <Row lbl="Registry Domain ID" val={domain.Registry_Domain_ID} /> }
+      { domain.Registrar_WHOIS_Server && <Row lbl="Registrar WHOIS Server" val={domain.Registrar_WHOIS_Server} /> }
+      { domain.Registrar && <Row lbl="" val="">
+        <span className="lbl">Registrar</span>
+        <span className="val"><a href={domain.Registrar_URL || '#'}>{domain.Registrar}</a></span>
+      </Row> }
+      { domain.Registrar_IANA_ID && <Row lbl="Registrar IANA ID" val={domain.Registrar_IANA_ID} /> } */}
+
+      {/* <Row lbl="" val="">
+        <span className="lbl">Is Up?</span>
+        { serverStatus.isUp ? <span className="val up">✅ Online</span> : <span className="val down">❌ Offline</span>}
+      </Row>
+      <Row lbl="Status Code" val={serverStatus.responseCode} />
+      { serverStatus.responseTime && <Row lbl="Response Time" val={`${Math.round(serverStatus.responseTime)}ms`} /> } */}
+    </Card>
+  );
+}
+
+export default RankCard;

src/pages/Results.tsx

@@ -51,6 +51,7 @@ import MailConfigCard from 'components/Results/MailConfig';
 import HttpSecurityCard from 'components/Results/HttpSecurity';
 import FirewallCard from 'components/Results/Firewall';
 import ArchivesCard from 'components/Results/Archives';
+import RankCard from 'components/Results/Rank';
 
 import keys from 'utils/get-keys';
 import { determineAddressType, AddressType } from 'utils/address-type-checker';
@@ -430,6 +431,14 @@ const Results = (): JSX.Element => {
     fetchRequest: () => fetch(`${api}/archives?url=${address}`).then(res => parseJson(res)),
   });
 
+  // Get website's global ranking, from Tranco
+  const [rankResults, updateRankResults] = useMotherHook({
+    jobId: 'rank',
+    updateLoadingJobs,
+    addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
+    fetchRequest: () => fetch(`${api}/rank?url=${address}`).then(res => parseJson(res)),
+  });
+
   /* Cancel remaining jobs after  10 second timeout */
   useEffect(() => {
     const checkJobs = () => {
@@ -475,6 +484,7 @@ const Results = (): JSX.Element => {
     { id: 'status', title: 'Server Status', result: serverStatusResults, Component: ServerStatusCard, refresh: updateServerStatusResults },
     { id: 'ports', title: 'Open Ports', result: portsResults, Component: OpenPortsCard, refresh: updatePortsResults },
     { id: 'security-txt', title: 'Security.Txt', result: securityTxtResults, Component: SecurityTxtCard, refresh: updateSecurityTxtResults },
+    { id: 'rank', title: 'Global Ranking', result: rankResults, Component: RankCard, refresh: updateRankResults },
     { id: 'screenshot', title: 'Screenshot', result: screenshotResult || lighthouseResults?.fullPageScreenshot?.screenshot, Component: ScreenshotCard, refresh: updateScreenshotResult },
     { id: 'mail-config', title: 'Email Configuration', result: mailConfigResults, Component: MailConfigCard, refresh: updateMailConfigResults },
     { id: 'hsts', title: 'HSTS Check', result: hstsResults, Component: HstsCard, refresh: updateHstsResults },