extern/web-check
1
0
Fork 0
mirror of https://github.com/Lissy93/web-check.git synced 2024-11-25 09:43:13 +01:00
Code Issues Packages Projects Releases Wiki Activity

Convert all API endpoints into ESM modules

Browse Source
This commit is contained in:
Alicia Sykes 2024-05-06 21:51:32 +01:00
parent e255c358cb
commit c9e57400fd
34 changed files with 191 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/_common/middleware.js
View File

@ -124,4 +124,4 @@ const commonMiddleware = (handler) => {
return nativeMode ? vercelHandler : netlifyHandler;
};
module.exports = commonMiddleware;
export default commonMiddleware;

10
api/archives.js
View File

@ -1,5 +1,5 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
import axios from 'axios';
import middleware from './_common/middleware.js';
const convertTimestampToDate = (timestamp) => {
const [year, month, day, hour, minute, second] = [
@ -46,7 +46,7 @@ const getScanFrequency = (firstScan, lastScan, totalScans, changeCount) => {
};
};
const getWaybackData = async (url) => {
const wayBackHandler = async (url) => {
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${url}&output=json&fl=timestamp,statuscode,digest,length,offset`;
try {
@ -80,5 +80,5 @@ const getWaybackData = async (url) => {
}
};
module.exports = middleware(getWaybackData);
module.exports.handler = middleware(getWaybackData);
export const handler = middleware(wayBackHandler);
export default handler;

12
api/block-lists.js
View File

@ -1,6 +1,6 @@
const dns = require('dns');
const { URL } = require('url');
const middleware = require('./_common/middleware');
import dns from 'dns';
import { URL } from 'url';
import middleware from './_common/middleware.js';
const DNS_SERVERS = [
{ name: 'AdGuard', ip: '176.103.130.130' },
@ -94,12 +94,12 @@ const checkDomainAgainstDnsServers = async (domain) => {
return results;
};
const handler = async (url) => {
export const blockListHandler = async (url) => {
const domain = new URL(url).hostname;
const results = await checkDomainAgainstDnsServers(domain);
return { blocklists: results };
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(blockListHandler);
export default handler;

10
api/carbon.js
View File

@ -1,7 +1,7 @@
const https = require('https');
const middleware = require('./_common/middleware');
import https from 'https';
import middleware from './_common/middleware.js';
const handler = async (url) => {
const carbonHandler = async (url) => {
// First, get the size of the website's HTML
const getHtmlSize = (url) => new Promise((resolve, reject) => {
@ -48,5 +48,5 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(carbonHandler);
export default handler;

12
api/cookies.js
View File

@ -1,6 +1,6 @@
const axios = require('axios');
const puppeteer = require('puppeteer');
const middleware = require('./_common/middleware');
import axios from 'axios';
import puppeteer from 'puppeteer';
import middleware from './_common/middleware.js';
const getPuppeteerCookies = async (url) => {
const browser = await puppeteer.launch({
@ -21,7 +21,7 @@ const getPuppeteerCookies = async (url) => {
}
};
const handler = async (url) => {
const cookieHandler = async (url) => {
let headerCookies = null;
let clientCookies = null;
@ -54,5 +54,5 @@ const handler = async (url) => {
return { headerCookies, clientCookies };
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(cookieHandler);
export default handler;

15
api/dns-server.js
View File

@ -1,9 +1,8 @@
const dns = require('dns');
const dnsPromises = dns.promises;
const axios = require('axios');
const middleware = require('./_common/middleware');
import { promises as dnsPromises, lookup } from 'dns';
import axios from 'axios';
import middleware from './_common/middleware.js';
const handler = async (url) => {
const dnsHandler = async (url) => {
try {
const domain = url.replace(/^(?:https?:\/\/)?/i, "");
const addresses = await dnsPromises.resolve4(domain);
@ -41,5 +40,7 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(dnsHandler);
export default handler;

12
api/dns.js
View File

@ -1,8 +1,8 @@
const dns = require('dns');
const util = require('util');
const middleware = require('./_common/middleware');
import dns from 'dns';
import util from 'util';
import middleware from './_common/middleware.js';
const handler = async (url) => {
const dnsHandler = async (url) => {
let hostname = url;
// Handle URLs by extracting hostname
@ -51,5 +51,5 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(dnsHandler);
export default handler;

10
api/dnssec.js
View File

@ -1,7 +1,7 @@
const https = require('https');
const middleware = require('./_common/middleware'); // Make sure this path is correct
import https from 'https';
import middleware from './_common/middleware.js';
const handler = async (domain) => {
const dnsSecHandler = async (domain) => {
const dnsTypes = ['DNSKEY', 'DS', 'RRSIG'];
const records = {};
@ -53,5 +53,5 @@ const handler = async (domain) => {
return records;
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(dnsSecHandler);
export default handler;

10
api/features.js
View File

@ -1,7 +1,7 @@
const https = require('https');
const middleware = require('./_common/middleware');
import https from 'https';
import middleware from './_common/middleware.js';
const handler = async (url) => {
const featuresHandler = async (url) => {
const apiKey = process.env.BUILT_WITH_API_KEY;
if (!url) {
@ -45,5 +45,5 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(featuresHandler);
export default handler;

10
api/firewall.js
View File

@ -1,5 +1,5 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
import axios from 'axios';
import middleware from './_common/middleware.js';
const hasWaf = (waf) => {
return {
@ -7,7 +7,7 @@ const hasWaf = (waf) => {
}
};
const handler = async (url) => {
const firewallHandler = async (url) => {
const fullUrl = url.startsWith('http') ? url : `http://${url}`;
try {
@ -102,5 +102,5 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(firewallHandler);
export default handler;

10
api/get-ip.js
View File

@ -1,5 +1,5 @@
const dns = require('dns');
const middleware = require('./_common/middleware');
import dns from 'dns';
import middleware from './_common/middleware.js';
const lookupAsync = (address) => {
return new Promise((resolve, reject) => {
@ -13,11 +13,11 @@ const lookupAsync = (address) => {
});
};
const handler = async (url) => {
const ipHandler = async (url) => {
const address = url.replaceAll('https://', '').replaceAll('http://', '');
return await lookupAsync(address);
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(ipHandler);
export default handler;

10
api/headers.js
View File

@ -1,7 +1,7 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
import axios from 'axios';
import middleware from './_common/middleware.js';
const handler = async (url, event, context) => {
const headersHandler = async (url, event, context) => {
try {
const response = await axios.get(url, {
validateStatus: function (status) {
@ -15,5 +15,5 @@ const handler = async (url, event, context) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(headersHandler);
export default handler;

11
api/hsts.js
View File

@ -1,7 +1,7 @@
const https = require('https');
const middleware = require('./_common/middleware');
import https from 'https';
import middleware from './_common/middleware.js';
const handler = async (url, event, context) => {
const hstsHandler = async (url, event, context) => {
const errorResponse = (message, statusCode = 500) => {
return {
statusCode: statusCode,
@ -45,6 +45,5 @@ const handler = async (url, event, context) => {
});
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(hstsHandler);
export default handler;

10
api/http-security.js
View File

@ -1,7 +1,7 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
import axios from 'axios';
import middleware from './_common/middleware.js';
const handler = async (url) => {
const httpsSecHandler = async (url) => {
const fullUrl = url.startsWith('http') ? url : `http://${url}`;
try {
@ -22,5 +22,5 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(httpsSecHandler);
export default handler;

17
api/legacy-rank.js
View File

@ -1,8 +1,8 @@
const axios = require('axios');
const unzipper = require('unzipper');
const csv = require('csv-parser');
const fs = require('fs');
const middleware = require('./_common/middleware');
import axios from 'axios';
import unzipper from 'unzipper';
import csv from 'csv-parser';
import fs from 'fs';
import middleware from './_common/middleware.js';
// Should also work with the following sources:
// https://www.domcop.com/files/top/top10milliondomains.csv.zip
@ -14,7 +14,7 @@ const middleware = require('./_common/middleware');
const FILE_URL = 'https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip';
const TEMP_FILE_PATH = '/tmp/top-1m.csv';
const handler = async (url) => {
const rankHandler = async (url) => {
let domain = null;
try {
@ -66,6 +66,5 @@ return new Promise((resolve, reject) => {
});
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(rankHandler);
export default handler;

14
api/linked-pages.js
View File

@ -1,9 +1,9 @@
const axios = require('axios');
const cheerio = require('cheerio');
const urlLib = require('url');
const middleware = require('./_common/middleware');
import axios from 'axios';
import cheerio from 'cheerio';
import urlLib from 'url';
import middleware from './_common/middleware.js';
const handler = async (url) => {
const linkedPagesHandler = async (url) => {
const response = await axios.get(url);
const html = response.data;
const $ = cheerio.load(html);
@ -45,5 +45,5 @@ const handler = async (url) => {
return { internal: internalLinks, external: externalLinks };
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(linkedPagesHandler);
export default handler;

13
api/mail-config.js
View File

@ -1,9 +1,8 @@
const middleware = require('./_common/middleware');
import dns from 'dns';
import URL from 'url-parse';
import middleware from './_common/middleware.js';
const dns = require('dns').promises;
const URL = require('url-parse');
const handler = async (url, event, context) => {
const mailConfigHandler = async (url, event, context) => {
try {
const domain = new URL(url).hostname || new URL(url).pathname;
@ -77,5 +76,5 @@ const handler = async (url, event, context) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(mailConfigHandler);
export default handler;

10
api/ports.js
View File

@ -1,5 +1,5 @@
const net = require('net');
const middleware = require('./_common/middleware');
import net from 'net';
import middleware from './_common/middleware.js';
// A list of commonly used ports.
const PORTS = [
@ -34,7 +34,7 @@ async function checkPort(port, domain) {
});
}
const handler = async (url, event, context) => {
const portsHandler = async (url, event, context) => {
const domain = url.replace(/(^\w+:|^)\/\//, '');
const delay = ms => new Promise(res => setTimeout(res, ms));
@ -80,5 +80,5 @@ const errorResponse = (message, statusCode = 444) => {
return { error: message };
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(portsHandler);
export default handler;

10
api/quality.js
View File

@ -1,7 +1,7 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
import axios from 'axios';
import middleware from './_common/middleware.js';
const handler = async (url, event, context) => {
const qualityHandler = async (url, event, context) => {
const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
if (!apiKey) {
@ -18,5 +18,5 @@ const handler = async (url, event, context) => {
return (await axios.get(endpoint)).data;
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(qualityHandler);
export default handler;

10
api/rank.js
View File

@ -1,7 +1,7 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
import axios from 'axios';
import middleware from './_common/middleware.js';
const handler = async (url) => {
const rankHandler = async (url) => {
const domain = url ? new URL(url).hostname : null;
if (!domain) throw new Error('Invalid URL');
@ -21,6 +21,6 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(rankHandler);
export default handler;

15
api/redirects.js
View File

@ -1,9 +1,10 @@
const handler = async (url) => {
const redirects = [url];
const got = await import('got');
import got from 'got';
import middleware from './_common/middleware.js';
const redirectsHandler = async (url) => {
const redirects = [url];
try {
await got.default(url, {
await got(url, {
followRedirect: true,
maxRedirects: 12,
hooks: {
@ -23,7 +24,5 @@ const handler = async (url) => {
}
};
const middleware = require('./_common/middleware');
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(redirectsHandler);
export default handler;

10
api/robots-txt.js
View File

@ -1,5 +1,5 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
import axios from 'axios';
import middleware from './_common/middleware.js';
const parseRobotsTxt = (content) => {
const lines = content.split('\n');
@ -31,7 +31,7 @@ const parseRobotsTxt = (content) => {
return { robots: rules };
}
const handler = async function(url) {
const robotsHandler = async function(url) {
let parsedURL;
try {
parsedURL = new URL(url);
@ -67,5 +67,5 @@ const handler = async function(url) {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(robotsHandler);
export default handler;

12
api/screenshot.js
View File

@ -1,8 +1,8 @@
const puppeteer = require('puppeteer-core');
const chromium = require('chrome-aws-lambda');
const middleware = require('./_common/middleware');
import puppeteer from 'puppeteer-core';
import chromium from 'chrome-aws-lambda';
import middleware from './_common/middleware.js';
const handler = async (targetUrl) => {
const screenshotHandler = async (targetUrl) => {
if (!targetUrl) {
throw new Error('URL is missing from queryStringParameters');
@ -58,5 +58,5 @@ const handler = async (targetUrl) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(screenshotHandler);
export default handler;

14
api/security-txt.js
View File

@ -1,6 +1,8 @@
const { https } = require('follow-redirects');
const { URL } = require('url');
const middleware = require('./_common/middleware');
import { URL } from 'url';
import followRedirects from 'follow-redirects';
import middleware from './_common/middleware.js';
const { https } = followRedirects;
const SECURITY_TXT_PATHS = [
'/security.txt',
@ -38,7 +40,7 @@ const isPgpSigned = (result) => {
return false;
};
const handler = async (urlParam) => {
const securityTxtHandler = async (urlParam) => {
let url;
try {
@ -90,5 +92,5 @@ async function fetchSecurityTxt(baseURL, path) {
});
}
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(securityTxtHandler);
export default handler;

13
api/sitemap.js
View File

@ -1,9 +1,8 @@
const middleware = require('./_common/middleware');
import axios from 'axios';
import xml2js from 'xml2js';
import middleware from './_common/middleware.js';
const axios = require('axios');
const xml2js = require('xml2js');
const handler = async (url) => {
const sitemapHandler = async (url) => {
let sitemapUrl = `${url}/sitemap.xml`;
const hardTimeOut = 5000;
@ -49,6 +48,6 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(sitemapHandler);
export default handler;

13
api/social-tags.js
View File

@ -1,9 +1,8 @@
const middleware = require('./_common/middleware');
import axios from 'axios';
import cheerio from 'cheerio';
import middleware from './_common/middleware.js';
const axios = require('axios');
const cheerio = require('cheerio');
const handler = async (url) => {
const socialTagsHandler = async (url) => {
// Check if url includes protocol
if (!url.startsWith('http://') && !url.startsWith('https://')) {
@ -61,5 +60,5 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(socialTagsHandler);
export default handler;

10
api/ssl.js
View File

@ -1,7 +1,7 @@
const tls = require('tls');
const middleware = require('./_common/middleware');
import tls from 'tls';
import middleware from './_common/middleware.js';
const handler = async (urlString) => {
const sslHandler = async (urlString) => {
try {
const parsedUrl = new URL(urlString);
const options = {
@ -40,5 +40,5 @@ const handler = async (urlString) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(sslHandler);
export default handler;

12
api/status.js
View File

@ -1,8 +1,8 @@
const https = require('https');
const { performance, PerformanceObserver } = require('perf_hooks');
const middleware = require('./_common/middleware');
import https from 'https';
import { performance, PerformanceObserver } from 'perf_hooks';
import middleware from './_common/middleware.js';
const handler = async (url) => {
const statusHandler = async (url) => {
if (!url) {
throw new Error('You must provide a URL query parameter!');
}
@ -55,5 +55,5 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(statusHandler);
export default handler;

10
api/tech-stack.js
View File

@ -1,7 +1,7 @@
const Wappalyzer = require('wappalyzer');
const middleware = require('./_common/middleware');
import Wappalyzer from 'wappalyzer';
import middleware from './_common/middleware.js';
const handler = async (url) => {
const techStackHandler = async (url) => {
const options = {};
const wappalyzer = new Wappalyzer(options);
@ -27,5 +27,5 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(techStackHandler);
export default handler;

12
api/threats.js
View File

@ -1,6 +1,6 @@
const axios = require('axios');
const xml2js = require('xml2js');
const middleware = require('./_common/middleware');
import axios from 'axios';
import xml2js from 'xml2js';
import middleware from './_common/middleware.js';
const getGoogleSafeBrowsingResult = async (url) => {
try {
@ -84,7 +84,7 @@ const getCloudmersiveResult = async (url) => {
}
};
const handler = async (url) => {
const threatsHandler = async (url) => {
try {
const urlHaus = await getUrlHausResult(url);
const phishTank = await getPhishTankResult(url);
@ -99,5 +99,5 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(threatsHandler);
export default handler;

10
api/tls.js
View File

@ -1,9 +1,9 @@
const axios = require('axios');
const middleware = require('./_common/middleware');
import axios from 'axios';
import middleware from './_common/middleware.js';
const MOZILLA_TLS_OBSERVATORY_API = 'https://tls-observatory.services.mozilla.com/api/v1';
const handler = async (url) => {
const tlsHandler = async (url) => {
try {
const domain = new URL(url).hostname;
const scanResponse = await axios.post(`${MOZILLA_TLS_OBSERVATORY_API}/scan?target=${domain}`);
@ -25,5 +25,5 @@ const handler = async (url) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(tlsHandler);
export default handler;

12
api/trace-route.js
View File

@ -1,8 +1,8 @@
const traceroute = require('traceroute');
const url = require('url');
const middleware = require('./_common/middleware');
import url from 'url';
import traceroute from 'traceroute';
import middleware from './_common/middleware.js';
const handler = async (urlString, context) => {
const traceRouteHandler = async (urlString, context) => {
// Parse the URL and get the hostname
const urlObject = url.parse(urlString);
const host = urlObject.hostname;
@ -28,5 +28,5 @@ const handler = async (urlString, context) => {
};
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(traceRouteHandler);
export default handler;

10
api/txt-records.js
View File

@ -1,7 +1,7 @@
const dns = require('dns').promises;
const middleware = require('./_common/middleware');
import dns from 'dns/promises';
import middleware from './_common/middleware.js';
const handler = async (url, event, context) => {
const txtRecordHandler = async (url, event, context) => {
try {
const parsedUrl = new URL(url);
@ -29,5 +29,5 @@ const handler = async (url, event, context) => {
}
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(txtRecordHandler);
export default handler;

14
api/whois.js
View File

@ -1,7 +1,7 @@
const net = require('net');
const psl = require('psl');
const axios = require('axios');
const middleware = require('./_common/middleware');
import net from 'net';
import psl from 'psl';
import axios from 'axios';
import middleware from './_common/middleware.js';
const getBaseDomain = (url) => {
let protocol = '';
@ -83,7 +83,7 @@ const fetchFromMyAPI = async (hostname) => {
}
};
const handler = async (url) => {
const whoisHandler = async (url) => {
if (!url.startsWith('http://') && !url.startsWith('https://')) {
url = 'http://' + url;
}
@ -106,6 +106,6 @@ const handler = async (url) => {
};
};
module.exports = middleware(handler);
module.exports.handler = middleware(handler);
export const handler = middleware(whoisHandler);
export default handler;