Added support for multiple servers

Hashtags and domain-less accounts are searched on all configured servers. Accounts with a domain are now always fetched from their home server.

closes #2
This commit is contained in:
Marcel Hellkamp 2023-07-17 14:33:29 +02:00
parent 6c72830545
commit d93a7f32da
7 changed files with 297 additions and 141 deletions

View File

@ -1,5 +1,5 @@
{
"server": "mastodon.social",
"servers": ["mastodon.social"],
"tags": ["foss", "cats", "dogs"],
"accounts": [],
"limit": 20,

View File

@ -1,9 +1,10 @@
<script setup lang="ts">
import { computed, inject, onBeforeUnmount, onMounted, onUpdated, ref, watch, watchEffect } from 'vue';
import { computed, inject, onBeforeUnmount, onMounted, onUpdated, ref, watch } from 'vue';
import Card, { type Post } from './components/Card.vue';
import { useWindowSize, watchDebounced } from '@vueuse/core'
import ConfigModal from './components/ConfigModal.vue';
import { loadConfig, type Config } from './config';
import InfoBar from './components/InfoBar.vue';
const config = ref<Config>();
@ -12,10 +13,9 @@ const pinned = ref<Array<string>>([])
const hidden = ref<Array<string>>([])
const banned = ref<Array<string>>([])
const updateInProgress = ref(false)
const lastError = ref<string>()
var updateIntervalHandle: number;
const userToId: Record<string, string> = {}
const accountToLocalId: Record<string, string | null> = {}
onMounted(async () => {
config.value = await loadConfig()
@ -54,6 +54,56 @@ watch(() => config.value?.interval, () => {
}, interval * 1000)
})
// Souces grouped by server
type SourceConfig = {
domain: string,
tags: string[],
accounts: string[],
}
// Source configurations grouped by server domain
const groupedSources = computed<Array<SourceConfig>>(() => {
const cfg = config.value
if (!cfg) return [];
const sources: Record<string, SourceConfig> = {}
const forServer = (domain: string) => {
if (!sources.hasOwnProperty(domain))
sources[domain] = { domain, tags: [], accounts: [] }
return sources[domain]
}
// Tags are searched on all servers
cfg.servers.forEach(domain => {
const source = forServer(domain)
source.tags = [...cfg.tags]
})
// Accounts are searched on the server they belong to.
// Non-qualified accounts are searched on all servers.
cfg.accounts.forEach(account => {
var [user, domain] = account.split('@', 2)
if (domain) {
forServer(domain).accounts.push(user)
} else {
cfg.servers.forEach(domain => {
forServer(domain).accounts.push(user)
})
}
})
return Object.values(sources).map(src => {
src.accounts = src.accounts.sort().filter((v, i, a) => a.indexOf(v) == i)
src.tags = src.tags.sort().filter((v, i, a) => a.indexOf(v) == i)
return src
});
})
/**
* Fetch a json resources from a given URL.
* Automaticaly detect mastodon rate limits and wait and retry up to 3 times.
*/
async function fetchJson(url: string) {
var rs = await fetch(url)
@ -79,95 +129,141 @@ async function fetchJson(url: string) {
const json = await rs.json()
if (json.error) {
console.warn(`Fetch error: ${rs.status} ${JSON.stringify(json)}`)
throw new Error(json.error)
const err = new Error(json.error);
(err as any).status = rs.status;
throw err;
}
return json
}
async function getUserId(name: string) {
const cfg = config.value
if (!cfg) return
/**
* Returns the instance-local account ID for a given user name.
* Results are cached. Returns null if not found, or undefined on errors.
*/
async function getLocalUserId(user: string, domain: string) {
const key = `${user}@${domain}`
if (!userToId.hasOwnProperty(name)) {
if (!accountToLocalId.hasOwnProperty(key)) {
try {
userToId[name] = (await fetchJson(`https://${cfg.server}/api/v1/accounts/lookup?acct=${encodeURIComponent(name)}`)).id
accountToLocalId[key] = (await fetchJson(`https://${domain}/api/v1/accounts/lookup?acct=${encodeURIComponent(user)}`)).id
} catch (e) {
console.warn(`Failed to fetch id for user ${name}`)
if ((e as any).status === 404)
accountToLocalId[key] = null;
}
}
return userToId[name]
return accountToLocalId[key]
}
const filterStatus = (post: any) => {
/**
* Check if a mastodon status document should be accepted
*/
const filterStatus = (status: any) => {
if (status.reblog)
status = status.reblog
// Filter sensitive posts (TODO: Allow if configured)
if (post?.sensitive) return false;
if (status?.sensitive) return false;
// Filter replies (TODO: Allow if configured)
if (post?.in_reply_to_id) return false;
if (status?.in_reply_to_id) return false;
// Filter non-public posts
if (post?.visibility !== "public") return false;
if (status?.visibility !== "public") return false;
// Filter bad actors
if (post?.account?.suspended) return false;
if (post?.account?.limted) return false;
// Filter bots
if (status?.account?.suspended) return false;
if (status?.account?.limted) return false;
// TODO: Filter bots?
//if(post?.account?.bot) return false;
// Accept anything else
return true;
}
const statusToWallPost = (post: any): Post => {
var date = post.created_at
if (post.reblog)
post = post.reblog
/**
* Convert a mastdon status object to a Post.
*/
const statusToWallPost = (status: any): Post => {
var date = status.created_at
if (status.reblog)
status = status.reblog
var media;
const image = post.media_attachments?.find((m: any) => m.type == "image")
const image = status.media_attachments?.find((m: any) => m.type == "image")
if (image)
media = image.url
return {
id: post.id,
url: post.url,
id: status.id,
url: status.url,
author: {
name: post.account.display_name || post.account.username,
url: post.account.url,
avatar: post.account.avatar,
name: status.account.display_name || status.account.username,
url: status.account.url,
avatar: status.account.avatar,
},
content: post.content,
content: status.content,
date,
media,
}
}
async function fetchPosts() {
/**
* Fetch all new statuses from a given source.
*/
const fetchSource = async (source: SourceConfig) => {
const cfg = config.value
if (!cfg) return []
const posts = []
const posts: Array<Post> = [];
for (const tag of source.tags) {
const items = await fetchJson(`https://${source.domain}/api/v1/timelines/tag/${encodeURIComponent(tag)}?limit=${cfg.limit}`)
posts.push(...items)
}
for (let account of source.accounts) {
const localUserId = await getLocalUserId(account, source.domain)
if (!localUserId) continue;
const items = await fetchJson(`https://${source.domain}/api/v1/accounts/${encodeURIComponent(localUserId)}/statuses?limit=${cfg.limit}&exclude_replies=True`)
posts.push(...items)
}
return posts
}
/**
* Fetch Posts from all sources.
*/
async function fetchAllPosts() {
const cfg = config.value
if (!cfg) return []
const posts: Post[] = []
const addOrReplace = (post?: Post) => {
if (!post) return
const i = posts.findIndex(p => p.id === post.id)
const i = posts.findIndex(p => p.url === post.url)
if (i >= 0)
posts[i] = post
else
posts.unshift(post)
}
for (var tag of cfg.tags) {
const items = await fetchJson(`https://${cfg.server}/api/v1/timelines/tag/${encodeURIComponent(tag)}?limit=${cfg.limit}`) as any[];
items.filter(filterStatus).map(statusToWallPost).forEach(addOrReplace);
// Start all sources in parallel
const tasks = groupedSources.value.map(source => fetchSource(source));
const results = await Promise.allSettled(tasks);
for (const result of results) {
if (result.status === "fulfilled") {
result.value.filter(filterStatus).map(statusToWallPost).forEach(addOrReplace)
} else {
const err = result.reason;
}
}
for (var user of cfg.accounts) {
const userId = await getUserId(user)
if (!userId) continue;
const items = await fetchJson(`https://${cfg.server}/api/v1/accounts/${encodeURIComponent(userId)}/statuses?limit=${cfg.limit}&exclude_replies=True`) as any[];
items.filter(filterStatus).map(statusToWallPost).forEach(addOrReplace);
}
return posts;
return posts
}
/**
* Trigger a wall update.
*
* Does nothing if there is an update running already.
*/
async function updateWall() {
const cfg = config.value
if (!cfg) return
@ -180,12 +276,10 @@ async function updateWall() {
console.debug("Startung wall update...")
updateInProgress.value = true
try {
allPosts.value = await fetchPosts()
lastError.value = undefined
allPosts.value = await fetchAllPosts()
console.debug("Update completed")
} catch (e) {
console.warn("Update failed", e)
lastError.value = (e as Error).toString()
} finally {
updateInProgress.value = false;
}
@ -241,14 +335,14 @@ const toggleTheme = () => {
}
const aboutLink = computed(() => {
if (config.value?.server.length)
return `https://${config.value.server}/about`
if (config.value?.servers.length)
return `https://${config.value.servers[0]}/about`
return "#"
})
const privacyLink = computed(() => {
if (config.value?.server.length)
return `https://${config.value.server}/privacy-policy`
if (config.value?.servers.length)
return `https://${config.value.servers[0]}/privacy-policy`
return "#"
})
@ -257,22 +351,12 @@ const privacyLink = computed(() => {
<template>
<div id="page">
<span v-show="updateInProgress" class="position-fixed bottom-0 start-0 m-1 opacity-25 text-muted"></span>
<header v-if="config?.info === 'top'" class="secret-hover">
This wall shows <a :href="aboutLink" target="_blank" class="">{{ config.server }}</a> posts
<template v-if="config.accounts.length">by
<a v-for="a in config.accounts" class="me-1"
:href="`https://${config.server}/@${encodeURIComponent(a).replace('%40', '@')}`">@{{
a }}</a>
</template>
<template v-if="config.accounts.length && config?.tags.length"> or </template>
<template v-if="config.tags.length">tagged with
<a v-for="t in config.tags" class="me-1" :href="`https://${config.server}/tags/${encodeURIComponent(t)}`">#{{
t }}</a>
</template>
<small class="text-secondary secret">
[<a href="#" class="text-secondary" @click.prevent="config.info = 'hide'">hide</a> -
<a href="#" class="text-secondary" data-bs-toggle="modal" data-bs-target="#configModal">edit</a>]
</small>
<header v-if="config?.info === 'top'">
<InfoBar :config="config" class="secret-hover">
<small class="text-secondary secret float-end">
[<a href="#" class="text-secondary" data-bs-toggle="modal" data-bs-target="#configModal">edit</a>]
</small>
</InfoBar>
</header>
<main>
@ -283,7 +367,7 @@ const privacyLink = computed(() => {
:post="post">
<template v-slot:topleft>
<div class="dropdown secret">
<button class="btn btn-outline-secondary" type="button" data-bs-toggle="dropdown"
<button class="btn btn-sm btn-outline-secondary" type="button" data-bs-toggle="dropdown"
aria-expanded="false">...</button>
<ul class="dropdown-menu">
<li><a class="dropdown-item" href="#" @click.prevent="pin(post.id)">{{ post.pinned ? "Unpin" : "Pin"
@ -302,9 +386,8 @@ const privacyLink = computed(() => {
<ConfigModal v-if="config" v-model="config" id="configModal" />
<footer>
<button class="btn btn-link text-muted" @click="toggleTheme(); false">[{{ config?.theme == "dark" ? "Light" :
"Dark" }} mode]</button>
<button class="btn btn-link text-muted" data-bs-toggle="modal" data-bs-target="#configModal">[configure]</button>
<button class="btn btn-link text-muted" @click="toggleTheme(); false">[{{ config?.theme == "dark" ? "Light" : "Dark" }} mode]</button>
<button class="btn btn-link text-muted" data-bs-toggle="modal" data-bs-target="#configModal">[Customize]</button>
<div>
<a :href="privacyLink" target="_blank" class="mx-1">Privacy policy</a>
- <a href="https://github.com/defnull/fediwall" target="_blank" class="mx-1">Github</a>
@ -329,7 +412,7 @@ body {
}
#page main {
padding: 1rem 2rem;
margin: 1rem 2rem;
}
.secret-hover .secret {
@ -346,6 +429,7 @@ body {
display: block;
width: 100%;
text-align: center;
background-color: var(--bs-light-bg-subtle);
}
#page footer {

View File

@ -1,5 +1,6 @@
<script setup lang="ts">
import { sanatizeConfig } from '@/config';
import { isServer } from '@/config';
import { toQuery, type Config } from '@/config';
import { computed, ref } from 'vue';
@ -17,20 +18,21 @@ const config = computed({
set: (value) => emit('update:modelValue', sanatizeConfig(value)),
});
const serverValue = computed({
get: () => config.value.server || "",
set: (value) => config.value.server = value.replace(/(^.*\/\/|[^a-z0-9.-]+)/i, ""),
const formServers = computed({
get: () => config.value.servers.join(" "),
set: (value) => config.value.servers = (value || "").split(" ").filter(isServer),
});
const tagPattern = /\b([\p{Letter}\p{Number}\p{Mark}\p{Connector_Punctuation}_]+)\b/igu
const formTags = computed({
get: () => config.value.tags.map(t => "#" + t).join(" "),
set: (value) => config.value.tags = (value || "").replace(/[^a-z0-9]+/ig, " ").split(" ").filter(t => t.length),
set: (value) => config.value.tags = [...(value || "").matchAll(tagPattern)].map(m => m[0]),
});
const accountPattern = /\b([A-Z0-9._%+-]+)(@([A-Z0-9.-]+\.[A-Z]{2,}))?\b/ig;
const accountPattern = /\b([a-z0-9_]+)(@([a-z0-9.-]+\.[a-z]{2,}))?\b/ig;
const formAccounts = computed({
get: () => config.value.accounts.map(t => "@" + t).join(" "),
set: (value) => config.value.accounts = [...(value || "").matchAll(accountPattern)].map(m => m[0]).filter(t => t.length),
set: (value) => config.value.accounts = [...(value || "").matchAll(accountPattern)].map(m => m[0]),
});
const formLimit = computed({
@ -76,8 +78,8 @@ const onSubmit = () => {
<div class="mb-3 row">
<label for="edit-server" class="col-sm-2 col-form-label">Server</label>
<div class="col-sm-10">
<input type="text" class="form-control" id="edit-server" v-model="serverValue">
<div class="form-text">Mastodon (or compatible) server domain.</div>
<input type="text" class="form-control" id="edit-server" v-model.lazy="formServers">
<div class="form-text">Mastodon (or compatible) server domains.</div>
</div>
</div>
@ -85,7 +87,7 @@ const onSubmit = () => {
<label for="edit-tags" class="col-sm-2 col-form-label">Hashtags</label>
<div class="col-sm-10">
<input type="text" class="form-control" id="edit-tags" v-model.lazy="formTags">
<div class="form-text">Hashtags to follow.</div>
<div class="form-text">Hashtags to follow on each server.</div>
</div>
</div>

View File

@ -0,0 +1,37 @@
<script setup lang="ts">
import { type Config } from '@/config';
import { ref } from 'vue';
const props = defineProps<{
config: Config
}>()
</script>
<template>
<div>
This Fediwall shows
<template v-if="config.tags.length">
posts tagged with
<template v-for="(t, index) in config.tags" :key="t">
<code>#{{ t }}</code>
<template v-if="index < config.tags.length - 2">, </template>
<template v-else-if="index == config.tags.length - 2"> or </template>
</template>
</template>
<template v-if="config.accounts.length && config?.tags.length"> and </template>
<template v-if="config.accounts.length">
posts or boosts by
<template v-for="(acc, index) in config.accounts" :key="t">
<code>@{{ acc }}</code>
<template v-if="index < config.accounts.length - 2">, </template>
<template v-else-if="index == config.accounts.length - 2"> or </template>
</template>
</template>
<slot></slot>
</div>
</template>
<style></style>

View File

@ -1,89 +1,112 @@
import { arrayEquals } from "./utils";
import {fallbackConfig, siteConfigUrl} from "@/defaults";
import { arrayEquals, deepClone, isString } from "./utils";
import { fallbackConfig, siteConfigUrl } from "@/defaults";
export type Config = {
server: string,
servers: Array<string>,
tags: Array<string>,
accounts: Array<string>,
limit: number,
interval: number,
theme: string | "dark" | "light",
info: string | "top" | "hide"
theme: string,
info: string,
}
var siteConfig: Config | undefined | false;
var siteConfig: Config = null;
export function fromQuery(query: string): Partial<Config> {
const themes = ["dark", "light"] as const;
const infoLineModes = ["top", "hide"] as const;
const choice = <T>(choices: readonly T[], value?: T, fallback?: T): T => {
return choices.includes(value) ? value : fallback;
}
export function fromQuery(query: string): Config {
const params = new URLSearchParams(query);
return {
server: params.get("server") || undefined,
tags: params.get("tags")?.split(",").filter(a => a.replace(/(^#|\s+)/ig, "")) || undefined,
accounts: params.get("accounts")?.split(",").filter(a => a.replace(/(^@|\s+)/ig, "")) || undefined,
limit: parseInt(params.get("limit") || "0") || undefined,
interval: parseInt(params.get("interval") || "0") || undefined,
theme: params.get("theme") || undefined,
info: params.get("info") || undefined,
}
const config: Partial<Config> = {}
// Keep URLs backwards compatible
if (params.has("server"))
params.set("servers", params.get("server"))
// Parse URL parameters very roughly
config.servers = params.get("servers")?.split(",")
config.tags = params.get("tags")?.split(",")
config.accounts = params.get("accounts")?.split(",")
config.limit = parseInt(params.get("limit") || "0")
config.interval = parseInt(params.get("interval") || "0")
config.theme = params.get("theme")
config.info = params.get("info")
// Clean, fix and return a valid config
return sanatizeConfig(config);
}
export function toQuery(config: Config): string {
const params = new URLSearchParams();
const defaults = siteConfig || fallbackConfig;
if (config.server !== defaults.server) params.set("server", config.server)
if (!arrayEquals(config.tags, defaults.tags)) params.set("tags", config.tags.join(","))
if (!arrayEquals(config.accounts, defaults.accounts)) params.set("accounts", config.accounts.join(","))
if (config.limit !== defaults.limit) params.set("limit", config.limit.toString())
if (config.interval !== defaults.interval) params.set("interval", config.interval.toString())
if (config.theme !== defaults.theme) params.set("theme", config.theme)
if (config.info !== defaults.info) params.set("info", config.info)
if (config.server !== defaults.server) params.set("server", config.server)
if (!arrayEquals(config.servers, defaults.servers))
params.set("servers", config.servers.join(","))
if (!arrayEquals(config.tags, defaults.tags))
params.set("tags", config.tags.join(","))
if (!arrayEquals(config.accounts, defaults.accounts))
params.set("accounts", config.accounts.join(","))
if (config.limit !== defaults.limit)
params.set("limit", config.limit.toString())
if (config.interval !== defaults.interval)
params.set("interval", config.interval.toString())
if (config.theme !== defaults.theme)
params.set("theme", config.theme)
if (config.info !== defaults.info)
params.set("info", config.info)
return params.toString().replace(/%2C/g, ',').replace(/%40/g, '@')
}
function isTag(tag: string) {
return tag.match(/^[a-z0-9]+$/i)
export function isTag(tag: string) {
return isString(tag) && tag.match(/^[\p{Letter}\p{Number}\p{Mark}\p{Connector_Punctuation}_]+$/iu)
}
function isAccount(acc: string) {
return acc.match(/^\b([A-Z0-9._%+-]+)(@([A-Z0-9.-]+\.[A-Z]{2,}))?\b$/i)
export function isAccount(acc: string) {
return isString(acc) && acc.match(/^([a-z0-9_]+)(@([a-z0-9.-]+\.[a-z]{2,}))?$/i)
}
export function sanatizeConfig(config: Partial<Config>): Config {
const defaults = (siteConfig || fallbackConfig)
return {
server: config.server?.replace(/(.*\/|[^a-z0-9.-]+)/i, '') || defaults.server,
tags: Array.isArray(config.tags) ? [...config.tags].filter(isTag).sort() : [],
accounts: Array.isArray(config.accounts) ? [...config.accounts].filter(isAccount).sort() : [],
limit: Math.max(1, Math.min(100, config?.limit || 0)),
interval: Math.max(1, Math.min(600, config?.interval || 0)),
theme: config.theme?.replace(/[^a-z]+/i, '') || defaults.theme,
info: config.info?.replace(/[^a-z]+/i, '') || defaults.info,
export function isServer(server: string) {
return isString(server) && server.match(/^([a-z0-9.-]+\.[a-z]{2,})$/i)
}
export function sanatizeConfig(config: any): Config {
// Migrate old configuration within same minor release
if (isString(config.server) && !config.servers) {
console.warn("DEPRECATED: Config parameter 'server' is now an array and called 'servers'.")
config.servers = [config.server]
}
const defaults = siteConfig ? siteConfig : fallbackConfig;
const result: Partial<Config> = {}
result.servers = Array.isArray(config.servers) ? [...config.servers].filter(isServer).sort() : [...defaults.servers];
result.tags = Array.isArray(config.tags) ? [...config.tags].filter(isTag).sort() : [...defaults.tags]
result.accounts = Array.isArray(config.accounts) ? [...config.accounts].filter(isAccount).sort() : [...defaults.accounts]
result.limit = Math.max(1, Math.min(100, config?.limit || defaults.limit))
result.interval = Math.max(1, Math.min(600, config?.interval || defaults.interval))
result.theme = choice(themes, config.theme, defaults.theme)
result.info = choice(infoLineModes, config.info, defaults.info)
return result as Config;
}
export async function loadConfig() {
if (siteConfig === undefined && siteConfigUrl) {
if (siteConfig === null && siteConfigUrl) {
try {
siteConfig = sanatizeConfig(await (await fetch(siteConfigUrl)).json() || {})
} catch (e) {
siteConfig = false
console.warn("Site config failed to load, falling back to hard-coded defaults!")
}
}
const config: Partial<Config> = {... (siteConfig || fallbackConfig)};
if (siteConfig === null)
siteConfig = sanatizeConfig(deepClone(fallbackConfig))
// Merge url parameters into site config, if present
if (window.location.search) {
const urlConfig = fromQuery(window.location.search.toString())
for (const key in urlConfig) {
// TODO: Fighting typescript here :/ I'm sure there is a better way
const value = (urlConfig as any)[key];
if (value !== undefined)
(config as any)[key] = value
}
}
if (window.location.search)
return fromQuery(window.location.search)
return sanatizeConfig(config);
return deepClone({ ... (siteConfig || fallbackConfig) })
}

View File

@ -4,13 +4,13 @@ import type { Config } from "./config"
// Fallback configuration in case the site config fails to load or is missing required fields.
// TODO: Maybe just fail in that case and not hard-code mastodon.social?
export const fallbackConfig: Config = {
"server": "mastodon.social",
"tags": ["foss", "cats", "dogs"],
"accounts": [],
"limit": 20,
"interval": 10,
"theme": "light",
"info": "top",
servers: ["mastodon.social"],
tags: ["foss", "cats", "dogs"],
accounts: [],
limit: 20,
interval: 10,
theme: "light",
info: "top",
}
// URL for a site-config file that overrides the default configuration above, if present.

View File

@ -3,4 +3,14 @@ export function arrayEquals(a: any, b: any) {
Array.isArray(b) &&
a.length === b.length &&
a.every((val, index) => val === b[index]);
}
}
export function isString(test: any) {
return typeof test === 'string' || test instanceof String
}
export function deepClone(obj: any) {
if(window.structuredClone)
return window.structuredClone(obj)
return JSON.parse(JSON.stringify(obj))
}