diff --git a/src/App.vue b/src/App.vue index f5615f7..1218b51 100644 --- a/src/App.vue +++ b/src/App.vue @@ -6,8 +6,8 @@ import ConfigModal from './components/ConfigModal.vue'; import { loadConfig } from './config'; import InfoBar from './components/InfoBar.vue'; import { gitVersion } from '@/defaults' -import { regexEscape } from '@/utils' -import {type Config, type Post} from '@/types'; +import { type Config, type Post } from '@/types'; +import { fetchPosts } from '@/sources' const config = ref(); @@ -17,7 +17,6 @@ const hidden = ref>([]) const banned = ref>([]) const updateInProgress = ref(false) -const accountToLocalId: Record = {} var updateIntervalHandle: number; var lastUpdate = 0; @@ -61,233 +60,8 @@ watch(visibilityState, () => { restartUpdates() }) -/** - * Fetch a json resources from a given URL. - * Automaticaly detect mastodon rate limits and wait and retry up to 3 times. - */ -async function fetchJson(url: string) { - var rs = await fetch(url) - // Auto-retry rate limit errors - var errCount = 0 - while (!rs.ok) { - if (errCount++ > 3) - break // Do not retry anymore - if (rs.headers.get("X-RateLimit-Remaining") === "0") { - const resetTime = new Date(rs.headers.get("X-RateLimit-Reset") || (new Date().getTime() + 10000)).getTime(); - const referenceTime = new Date(rs.headers.get("Date") || new Date()).getTime(); - const sleep = Math.max(0, resetTime - referenceTime) + 1000 // 1 second leeway - await new Promise(resolve => setTimeout(resolve, sleep)); - } else { - break // Do not retry - } - - // Retry - rs = await fetch(url) - } - - const json = await rs.json() - if (json.error) { - console.warn(`Fetch error: ${rs.status} ${JSON.stringify(json)}`) - const err = new Error(json.error); - (err as any).status = rs.status; - throw err; - } - return json -} - -/** - * Returns the instance-local account ID for a given user name. - * Results are cached. Returns null if not found, or undefined on errors. - */ -async function getLocalUser(user: string, domain: string): Promise { - const key = `${user}@${domain}` - - if (!accountToLocalId.hasOwnProperty(key)) { - try { - accountToLocalId[key] = (await fetchJson(`https://${domain}/api/v1/accounts/lookup?acct=${encodeURIComponent(user)}`)) - } catch (e) { - if ((e as any).status === 404) - accountToLocalId[key] = null; - } - } - return accountToLocalId[key] -} - -/** - * Check if a mastodon status document should be accepted - */ -const filterStatus = (status: any) => { - const cfg = config.value - if (!cfg) return false; - - // Filter reblogs? - if (cfg.hideBoosts && status.reblog) return false; - - // Unwrap boosts here so the other filters are checked against the status that - // is going to be displayed, not just the boost-status. - if (status.reblog) - status = status.reblog - - // Filter by language - if (cfg.languages.length > 0 - && !cfg.languages.includes(status.language || "en")) return false; - // Filter sensitive content? - if (cfg.hideSensitive && status.sensitive) return false; - // Filter replies? - if (cfg.hideReplies && status.in_reply_to_id) return false; - // Filter bots? - if (cfg.hideBots && status.account?.bot) return false; - // Filter bad hashtags or words - if (cfg.badWords.length) { - const pattern = new RegExp(`\\b(${cfg.badWords.map(regexEscape).join("|")})\\b`, 'i'); - if (status.tags?.find((tag: any) => cfg.badWords.includes(tag.name))) - return false; - if (status.content.match(pattern)) - return false; - } - - // Filter non-public content - if (status.visibility !== "public") return false; - // Filter limited or suspended accounts - if (status.account?.suspended) return false; - if (status.account?.limted) return false; - - // Accept anything else - return true; -} - -/** - * Convert a mastdon status object to a Post. - */ -const statusToWallPost = (status: any): Post => { - var date = status.created_at - - if (status.reblog) - status = status.reblog - - var media; - const image = status.media_attachments?.find((m: any) => m.type == "image") - if (image) - media = image.url - - return { - id: status.id, - url: status.url, - author: { - name: status.account.display_name || status.account.username, - url: status.account.url, - avatar: status.account.avatar, - }, - content: status.content, - date, - media, - } -} - -/** - * Fetch Posts from all sources. - */ -async function fetchAllPosts() { - const cfg = config.value - if (!cfg) return [] - - type Task = () => Promise; - - // Group tasks by domain (see below) - const domainTasks: Record> = {} - const addTask = (domain: string, task: Task) => { - (domainTasks[domain] ??= []).push(task) - } - - // Load tags from all servers - for (const domain of cfg.servers) { - for (const tag of cfg.tags) { - addTask(domain, () => { - return fetchJson(`https://${domain}/api/v1/timelines/tag/${encodeURIComponent(tag)}?limit=${cfg.limit}`) - }) - } - } - - // Load account timelines from the home server of the account, or all servers - // if the account is not fully qualified (missing domain part). - for (const account of cfg.accounts) { - const [user, domain] = account.split('@', 2) - const domains = domain ? [domain] : [...cfg.servers] - for (const domain of domains) { - addTask(domain, async () => { - const localUser = await getLocalUser(user, domain) - if (!localUser || !localUser.id) return []; - if (localUser.bot && cfg.hideBots && cfg.hideBoosts) return []; - - let url = `https://${domain}/api/v1/accounts/${encodeURIComponent(localUser.id)}/statuses?limit=${cfg.limit}` - if (cfg.hideReplies) url += "&exclude_replies=True" - if (cfg.hideBoosts) url += "&exclude_reblogs=True" - return await fetchJson(url) - }) - } - } - - // Load trends from all servers - if (cfg.loadTrends) { - for (const domain of cfg.servers) { - addTask(domain, async () => { - return await fetchJson(`https://${domain}/api/v1/trends/statuses?limit=${cfg.limit}`) - }) - } - } - - // Load public timeline from all servers, optionally limited to just local - // or just federated posts. - if (cfg.loadPublic || cfg.loadFederated) { - for (const domain of cfg.servers) { - let url = `https://${domain}/api/v1/timelines/public` - if (!cfg.loadPublic) - url += "?remote=True" - if (!cfg.loadFederated) - url += "?local=True" - addTask(domain, async () => { - return await fetchJson(url) - }) - } - } - - // Collect results - const posts: Post[] = [] - const addOrRepaceStatus = (status: any) => { - if (!status || !filterStatus(status)) return; - const post = statusToWallPost(status) - const i = posts.findIndex(p => p.url === post.url) - if (i >= 0) - posts[i] = post - else - posts.unshift(post) - } - - // Be nice and not overwhelm servers with parallel requests. - // Run tasks for the same domain in sequence instead. - const groupedTasks = Object.entries(domainTasks) - .map(([domain, tasks]) => { - return async () => { - for (const task of tasks) { - try { - (await task()).forEach(addOrRepaceStatus) - } catch (err) { - console.warn(`Update task failed for domain ${domain}`, err) - } - } - } - }) - - // Start all the domain-grouped tasks in parallel, so reach server can be - // processed as fast as its rate-limit allows. - // TODO: Add a timeout - await Promise.allSettled(groupedTasks.map(task => task())) - - // Done. Return collected posts - return posts -} /** * Starts or restarts the update interval timer. @@ -336,7 +110,7 @@ async function updateWall() { updateInProgress.value = true try { - allPosts.value = await fetchAllPosts() + allPosts.value = await fetchPosts(cfg) console.debug("Update completed") } catch (e) { console.warn("Update failed", e) @@ -347,8 +121,12 @@ async function updateWall() { } +/** + * Filter and order posts based on real-time criteria (e.g. pinned or hidden posts). + * Most of filtering already happened earlier. + */ const filteredPosts = computed(() => { - // copy to make sure those are detected as a reactive dependency + // Copy to make sure those are detected as a reactive dependencies var posts: Array = JSON.parse(JSON.stringify(allPosts.value)) const pinnedLocal = [...pinned.value] const hiddenLocal = [...hidden.value] diff --git a/src/config.ts b/src/config.ts index 276bf2e..2aa33ab 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,7 +1,7 @@ import { arrayUnique, deepClone, isString } from "./utils"; import { fallbackConfig, siteConfigUrl } from "@/defaults"; -import {type Config} from '@/types'; +import { type Config } from '@/types'; diff --git a/src/defaults.ts b/src/defaults.ts index 80e6276..9949d34 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -1,5 +1,5 @@ -import {type Config} from '@/types'; +import { type Config } from '@/types'; // Fallback configuration in case the site config fails to load or is missing required fields. // TODO: Maybe just fail in that case and not hard-code mastodon.social? diff --git a/src/main.ts b/src/main.ts index 893773c..947a2fd 100644 --- a/src/main.ts +++ b/src/main.ts @@ -23,7 +23,7 @@ import 'bootstrap/dist/js/bootstrap.bundle.js' import { createApp } from 'vue' import App from '@/App.vue' -import {VueMasonryPlugin} from 'vue-masonry'; +import { VueMasonryPlugin } from 'vue-masonry'; import VueDOMPurifyHTML from 'vue-dompurify-html'; const app = createApp(App) diff --git a/src/sources.ts b/src/sources.ts new file mode 100644 index 0000000..672ec68 --- /dev/null +++ b/src/sources.ts @@ -0,0 +1,231 @@ +import type { Config, MastodonAccount, MastodonStatus, Post } from "./types"; +import { regexEscape } from "./utils"; + + +/** + * Fetch unique posts from all sources (curently only Mastodon is implemented) + */ +export async function fetchPosts(cfg: Config): Promise { + type Task = () => Promise; + + // Group tasks by domain (see below) + const domainTasks: Record> = {} + const addTask = (domain: string, task: Task) => { + (domainTasks[domain] ??= []).push(task) + } + + // Load tags from all servers + for (const domain of cfg.servers) { + for (const tag of cfg.tags) { + addTask(domain, async () => { + return await fetchJson(domain, `api/v1/timelines/tag/${encodeURIComponent(tag)}`, { limit: cfg.limit.toString() }) + }) + } + } + + // Load account timelines from the home server of the account, or all servers + // if the account is not fully qualified (missing domain part). + for (const account of cfg.accounts) { + const [user, domain] = account.split('@', 2) + const domains = domain ? [domain] : [...cfg.servers] + for (const domain of domains) { + addTask(domain, async () => { + const localUser = await getLocalUser(user, domain) + if (!localUser || !localUser.id) return []; + if (localUser.bot && cfg.hideBots && cfg.hideBoosts) return []; + + const query: Record = { limit: cfg.limit } + if (cfg.hideReplies) query.exclude_replies = "True" + if (cfg.hideBoosts) query.exclude_reblogs = "True" + return await fetchJson(domain, `api/v1/accounts/${encodeURIComponent(localUser.id)}/statuses`, query) + }) + } + } + + // Load trends from all servers + if (cfg.loadTrends) { + for (const domain of cfg.servers) { + addTask(domain, async () => { + return await fetchJson(domain, "api/v1/trends/statuses", { limit: cfg.limit }) + }) + } + } + + // Load public timeline from all servers, optionally limited to just local + // or just federated posts. + if (cfg.loadPublic || cfg.loadFederated) { + for (const domain of cfg.servers) { + const query: Record = { limit: cfg.limit } + if (!cfg.loadPublic) query.remote = "True" + if (!cfg.loadFederated) query.local = "True" + addTask(domain, async () => { + return await fetchJson(domain, "api/v1/timelines/public", query) + }) + } + } + + // Collect results + const posts: Post[] = [] + const addOrRepacePost = (post: Post) => { + const i = posts.findIndex(p => p.id === post.id) + if (i >= 0) + posts[i] = post + else + posts.unshift(post) + } + + // Be nice and not overwhelm servers with parallel requests. + // Run tasks for the same domain in sequence instead. + const groupedTasks = Object.entries(domainTasks) + .map(([domain, tasks]) => { + return async () => { + for (const task of tasks) { + try { + (await task()) + .filter(status => filterStatus(cfg, status)) + .map(statusToWallPost) + .forEach(addOrRepacePost) + } catch (err) { + console.warn(`Update task failed for domain ${domain}`, err) + } + } + } + }) + + // Start all the domain-grouped tasks in parallel, so reach server can be + // processed as fast as its rate-limit allows. + // TODO: Add a timeout + await Promise.allSettled(groupedTasks.map(task => task())) + + // Done. Return collected posts + return posts +} + +/** + * Returns the instance-local account for a given user name. + * Results are cached. Returns null if not found, or undefined on errors. + */ +const accountCache: Record = {} +async function getLocalUser(user: string, domain: string): Promise { + const key = `${user}@${domain}` + + if (!accountCache.hasOwnProperty(key)) { + try { + accountCache[key] = (await fetchJson(domain, "v1/accounts/lookup", { acct: user })) as MastodonAccount + } catch (e) { + if ((e as any).status === 404) + accountCache[key] = null; + } + } + return accountCache[key] +} + + +/** + * Fetch a json resources from a given URL. + * Automaticaly detect mastodon rate limits and wait and retry up to 3 times. + */ +async function fetchJson(domain: string, path: string, query?: Record) { + let url = `https://${domain}/${path}` + if (query && Object.keys(query).length) { + const pairs = Object.entries(query).map(([key, value]) => [key, value.toString()]) + url += "?" + new URLSearchParams(pairs).toString() + } + var rs = await fetch(url) + + // Auto-retry rate limit errors + var errCount = 0 + while (!rs.ok) { + if (errCount++ > 3) + break // Do not retry anymore + + if (rs.headers.get("X-RateLimit-Remaining") === "0") { + const resetTime = new Date(rs.headers.get("X-RateLimit-Reset") || (new Date().getTime() + 10000)).getTime(); + const referenceTime = new Date(rs.headers.get("Date") || new Date()).getTime(); + const sleep = Math.max(0, resetTime - referenceTime) + 1000 // 1 second leeway + await new Promise(resolve => setTimeout(resolve, sleep)); + } else { + break // Do not retry + } + + // Retry + rs = await fetch(url) + } + + const json = await rs.json() + if (json.error) { + console.warn(`Fetch error: ${rs.status} ${JSON.stringify(json)}`) + const err = new Error(json.error); + (err as any).status = rs.status; + throw err; + } + return json +} + +/** + * Check if a mastodon status document should be accepted + */ +const filterStatus = (cfg: Config, status: MastodonStatus) => { + // Filter reblogs? + if (cfg.hideBoosts && status.reblog) return false; + + // Unwrap boosts here so the other filters are checked against the status that + // is going to be displayed, not just the boost-status. + if (status.reblog) + status = status.reblog + + // Filter by language + if (cfg.languages.length > 0 + && !cfg.languages.includes(status.language || "en")) return false; + // Filter sensitive content? + if (cfg.hideSensitive && status.sensitive) return false; + // Filter replies? + if (cfg.hideReplies && status.in_reply_to_id) return false; + // Filter bots? + if (cfg.hideBots && status.account?.bot) return false; + // Filter bad hashtags or words + if (cfg.badWords.length) { + const pattern = new RegExp(`\\b(${cfg.badWords.map(regexEscape).join("|")})\\b`, 'i'); + if (status.tags?.find((tag: any) => cfg.badWords.includes(tag.name))) + return false; + if (status.content.match(pattern)) + return false; + } + + // Filter non-public content + if (status.visibility !== "public") return false; + // Filter limited or suspended accounts + if (status.account?.suspended) return false; + if (status.account?.limited) return false; + + // Accept anything else + return true; +} + +/** + * Convert a mastdon status object to a Post. + */ +const statusToWallPost = (status: MastodonStatus): Post => { + let date = status.created_at + + if (status.reblog) + status = status.reblog + + var media; + const image = status.media_attachments?.find((m: any) => m.type == "image") + if (image) + media = image.url + + return { + id: status.uri, + url: status.url || status.uri, + author: { + name: status.account.display_name || status.account.username, + url: status.account.url, + avatar: status.account.avatar, + }, + content: status.content, + date, + media, + } +} diff --git a/src/types.ts b/src/types.ts index cbe0c0b..c84e8a1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,19 +1,3 @@ -export type Post = { - id: string; - url: string; - content: string; - date: string; - - author?: { - name: string; - avatar?: string; - url?: string; - }; - - media?: string; - pinned?: boolean; -}; - export type Config = { servers: Array, tags: Array, @@ -40,4 +24,83 @@ export type Config = { showText: boolean, showMedia: boolean, playVideos: boolean, -} \ No newline at end of file +} + +export type Post = { + id: string; + url: string; + content: string; + date: string; + + author?: { + name: string; + avatar?: string; + url?: string; + }; + + media?: string; + pinned?: boolean; +}; + + +/** + * Mastodon types. We only model what is important for us + */ + +export type MastodonStatus = { + id: string; + account: MastodonAccount; + created_at: string; + content: string; + edited_at?: string; + emojis: Array; + in_reply_to_account_id?: string | null; + in_reply_to_id?: string | null; + language?: string | null; + media_attachments: Array; + reblog?: MastodonStatus | null; + sensitive: boolean; + tags: Array; + uri: string; + url?: string | null; + visibility: 'direct' | 'private' | 'public' | 'unlisted'; +} + +export type MastodonAccount = { + id: string; + username: string; + acct: string; + avatar: string; + avatar_static: string; + bot: boolean; + created_at: string; + discoverable?: boolean; + display_name: string; + emojis: Array; + limited?: boolean; + locked: boolean; + moved?: MastodonAccount; + noindex?: boolean; + suspended?: boolean; + url: string; +} + +export type MastodonEmoji = { + shortcode: string; + static_url: string; + url: string; +} + +export type MastodonTag = { + name: string; + url: string; +} + +export type MastodonMediaAttachment = { + id: string; + type: 'audio' | 'video' | 'gifv' | 'unknown'; + blurhash?: string | null; + description?: string | null; + preview_url: string; + url: string; +} diff --git a/src/utils.ts b/src/utils.ts index 6d42e88..dda1c45 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -10,7 +10,7 @@ export function isString(test: any) { } export function arrayUnique(array: T[]) { - return array.filter((v,i,a) => a.indexOf(v) === i) + return array.filter((v, i, a) => a.indexOf(v) === i) } export function regexEscape(str: string) { @@ -18,7 +18,7 @@ export function regexEscape(str: string) { } export function deepClone(obj: any) { - if(window.structuredClone) + if (window.structuredClone) return window.structuredClone(obj) return JSON.parse(JSON.stringify(obj)) }