1
0
mirror of https://github.com/Lissy93/web-check.git synced 2025-07-23 06:40:37 +02:00
Files
.github
api
_common
archives.js
block-lists.js
carbon.js
cookies.js
dns-server.js
dns.js
dnssec.js
features.js
firewall.js
get-ip.js
headers.js
hsts.js
http-security.js
legacy-rank.js
linked-pages.js
mail-config.js
ports.js
quality.js
rank.js
redirects.js
robots-txt.js
screenshot.js
security-txt.js
sitemap.js
social-tags.js
ssl.js
status.js
tech-stack.js
threats.js
tls.js
trace-route.js
txt-records.js
whois.js
public
src
.env
.gitignore
Dockerfile
LICENSE
astro.config.mjs
docker-compose.yml
fly.toml
netlify.toml
package.json
server.js
svelte.config.js
tsconfig.json
vercel.json
vite.config.js
yarn.lock
web-check/api/linked-pages.js
2024-05-18 15:01:52 +01:00

50 lines
2.0 KiB
JavaScript

import axios from 'axios';
import cheerio from 'cheerio';
import urlLib from 'url';
import middleware from './_common/middleware.js';
const linkedPagesHandler = async (url) => {
const response = await axios.get(url);
const html = response.data;
const $ = cheerio.load(html);
const internalLinksMap = new Map();
const externalLinksMap = new Map();
// Get all links on the page
$('a[href]').each((i, link) => {
const href = $(link).attr('href');
const absoluteUrl = urlLib.resolve(url, href);
// Check if absolute / relative, append to appropriate map or increment occurrence count
if (absoluteUrl.startsWith(url)) {
const count = internalLinksMap.get(absoluteUrl) || 0;
internalLinksMap.set(absoluteUrl, count + 1);
} else if (href.startsWith('http://') || href.startsWith('https://')) {
const count = externalLinksMap.get(absoluteUrl) || 0;
externalLinksMap.set(absoluteUrl, count + 1);
}
});
// Sort by most occurrences, remove supplicates, and convert to array
const internalLinks = [...internalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
const externalLinks = [...externalLinksMap.entries()].sort((a, b) => b[1] - a[1]).map(entry => entry[0]);
// If there were no links, then mark as skipped and show reasons
if (internalLinks.length === 0 && externalLinks.length === 0) {
return {
statusCode: 400,
body: {
skipped: 'No internal or external links found. '
+ 'This may be due to the website being dynamically rendered, using a client-side framework (like React), and without SSR enabled. '
+ 'That would mean that the static HTML returned from the HTTP request doesn\'t contain any meaningful content for Web-Check to analyze. '
+ 'You can rectify this by using a headless browser to render the page instead.',
},
};
}
return { internal: internalLinks, external: externalLinks };
};
export const handler = middleware(linkedPagesHandler);
export default handler;