extern/GetMoarFediverse
1
0
Fork 1
mirror of https://github.com/g3rv4/GetMoarFediverse.git synced 2024-11-21 23:23:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use null reference checks (and enforce them) (#11)

Browse Source
This commit is contained in:
Gervasio Marchand 2022-12-20 18:11:57 -08:00 committed by GitHub
parent ac83a1f8a9
commit 80bae5652b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 72 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/Config.cs
View File

@ -34,6 +34,10 @@ public class Config
}
var data = JsonSerializer.Deserialize(File.ReadAllText(path), JsonContext.Default.ConfigData);
if (data == null)
{
throw new Exception("Could not deserialize the config file");
}
var importedPath = Path.Join(Path.GetDirectoryName(path), "imported.txt");
var apiKey = string.IsNullOrEmpty(data.FakeRelayApiKey)
@ -47,7 +51,9 @@ public class Config
if (data.Sites is { Length: > 0 })
{
Console.WriteLine("Warning: Sites is deprecated, please use Instances instead");
Console.WriteLine("|============================================================|");
Console.WriteLine("| Warning: Sites is deprecated, please use Instances instead |");
Console.WriteLine("|============================================================|\n");
}
data.Tags ??= Array.Empty<string>();
@ -56,13 +62,18 @@ public class Config
throw new Exception("You can't specify both MastodonPostgresConnectionString and Tags");
}
if (data.FakeRelayUrl.IsNullOrEmpty())
{
throw new Exception("Missing FakeRelayUrl");
}
Instance = new Config(importedPath, data.FakeRelayUrl, apiKey, data.MastodonPostgresConnectionString,
data.Tags.ToImmutableArray(), data.GetImmutableSites());
}
public class ConfigData
{
public string FakeRelayUrl { get; set; }
public string? FakeRelayUrl { get; set; }
public string? FakeRelayApiKey { get; set; }
public string? MastodonPostgresConnectionString { get; set; }
public string[]? Instances { get; set; }
@ -72,11 +83,11 @@ public class Config
public ImmutableArray<SiteData> GetImmutableSites()
{
// the plan is to stop supporting Sites in favor of Instances. SiteSpecificTags add complexity and
// don't make sense when pulling tags from Mastodon. Also, pulling is fast and multithreaded!
// don't make sense when pulling tags from Mastodon. Also, pulling is fast and multi threaded!
if (Instances != null)
{
return Instances
.Select(i => new SiteData { Host = i, SiteSpecificTags = ImmutableArray<string>.Empty })
.Select(i => new SiteData(i, ImmutableArray<string>.Empty))
.ToImmutableArray();
}
@ -88,24 +99,18 @@ public class Config
public class InternalSiteData
{
public string Host { get; set; }
public string[]? SiteSpecificTags { get; set; }
public InternalSiteData(string host, string[]? siteSpecificTags)
{
Host = host;
SiteSpecificTags = siteSpecificTags;
}
public string Host { get; }
public string[]? SiteSpecificTags { get; }
public SiteData ToSiteData() =>
new()
{
Host = Host,
SiteSpecificTags =
SiteSpecificTags == null
? ImmutableArray<string>.Empty
: SiteSpecificTags.ToImmutableArray()
};
new(Host, SiteSpecificTags?.ToImmutableArray() ?? ImmutableArray<string>.Empty);
}
}
public class SiteData
{
public string Host { get; init; }
public ImmutableArray<string> SiteSpecificTags { get; init; }
}
public record SiteData(string Host, ImmutableArray<string> SiteSpecificTags);
}

6
src/Extensions.cs
View File

@ -1,8 +1,10 @@
using System.Diagnostics.CodeAnalysis;
namespace GetMoarFediverse;
public static class Extensions
{
public static bool IsNullOrEmpty(this string s) => string.IsNullOrEmpty(s);
public static bool IsNullOrEmpty([NotNullWhen(returnValue: false)]this string? s) => string.IsNullOrEmpty(s);
public static bool HasValue(this string s) => !s.IsNullOrEmpty();
public static bool HasValue([NotNullWhen(returnValue: true)]this string? s) => !s.IsNullOrEmpty();
}

1
src/GetMoarFediverse.csproj
View File

@ -5,6 +5,7 @@
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>

15
src/MastodonConnectionHelper.cs
View File

@ -8,15 +8,18 @@ public static class MastodonConnectionHelper
{
var res = new List<string>();
if (Config.Instance == null)
{
throw new Exception("Config object is not initialized");
}
await using var conn = new NpgsqlConnection(Config.Instance.MastodonPostgresConnectionString);
await conn.OpenAsync();
await using (var cmd = new NpgsqlCommand("SELECT DISTINCT tags.name FROM tag_follows JOIN tags ON tag_id = tags.id ORDER BY tags.name ASC;", conn))
await using (var reader = await cmd.ExecuteReaderAsync())
{
while (await reader.ReadAsync())
res.Add(reader.GetString(0));
}
await using var cmd = new NpgsqlCommand("SELECT DISTINCT tags.name FROM tag_follows JOIN tags ON tag_id = tags.id ORDER BY tags.name ASC;", conn);
await using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
res.Add(reader.GetString(0));
return res;
}

44
src/Program.cs
View File

@ -7,8 +7,18 @@ if (args.Length == 1){
configPath = args[0];
}
if (configPath.IsNullOrEmpty())
{
throw new Exception("Missing config path");
}
Config.Init(configPath);
if (Config.Instance == null)
{
throw new Exception("Error initializing config object");
}
var client = new HttpClient();
var authClient = new HttpClient
{
@ -27,7 +37,14 @@ var imported = importedList.ToHashSet();
var statusesToLoadBag = new ConcurrentBag<string>();
List<(string host, string tag)> sitesTags;
if (string.IsNullOrEmpty(Config.Instance.MastodonPostgresConnectionString))
if (Config.Instance.MastodonPostgresConnectionString.HasValue())
{
var tags = await MastodonConnectionHelper.GetFollowedTagsAsync();
sitesTags = Config.Instance.Sites
.SelectMany(s => tags.Select(t => (s.Host, t)))
.ToList();
}
else
{
sitesTags = Config.Instance.Sites
.SelectMany(s => Config.Instance.Tags.Select(tag => (s.Host, tag)))
@ -35,13 +52,6 @@ if (string.IsNullOrEmpty(Config.Instance.MastodonPostgresConnectionString))
.OrderBy(t => t.tag)
.ToList();
}
else
{
var tags = await MastodonConnectionHelper.GetFollowedTagsAsync();
sitesTags = Config.Instance.Sites
.SelectMany(s => tags.Select(t => (s.Host, t)))
.ToList();
}
ParallelOptions parallelOptions = new()
{
@ -66,6 +76,11 @@ await Parallel.ForEachAsync(sitesTags, parallelOptions, async (st, _) =>
var json = await response.Content.ReadAsStringAsync();
var data = JsonSerializer.Deserialize(json, CamelCaseJsonContext.Default.TagResponse);
if (data == null)
{
Console.WriteLine($"Error deserializing the response when pulling #{tag} posts from {site}");
return;
}
foreach (var statusLink in data.OrderedItems.Where(i=>!imported.Contains(i)))
{
@ -79,8 +94,10 @@ foreach (var statusLink in statusesToLoad)
Console.WriteLine($"Bringing in {statusLink}");
try
{
var content = new List<KeyValuePair<string, string>>();
content.Add(new KeyValuePair<string, string>("statusUrl", statusLink));
var content = new List<KeyValuePair<string, string>>
{
new("statusUrl", statusLink)
};
var res = await authClient.PostAsync("index", new FormUrlEncodedContent(content));
res.EnsureSuccessStatusCode();
@ -103,5 +120,10 @@ File.WriteAllLines(importedPath, importedList);
public class TagResponse
{
public string[] OrderedItems { get; set; }
public string[] OrderedItems { get; }
public TagResponse(string[] orderedItems)
{
OrderedItems = orderedItems;
}
}