extern/GetMoarFediverse
1
0
Fork 1
mirror of https://github.com/g3rv4/GetMoarFediverse.git synced 2025-02-16 17:20:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use Mastodon's API to fetch hashtagged statuses (#39)

Browse Source
This commit is contained in:
Gervasio Marchand 2023-02-11 13:14:15 -03:00 committed by GitHub
parent 6fd7804d6c
commit d9faf231a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 11 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/JsonContexts.cs
View File

@ -7,7 +7,7 @@ internal partial class JsonContext : JsonSerializerContext
{ {
} }
[JsonSerializable(typeof(TagResponse))] [JsonSerializable(typeof(Status[]))]
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] [JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)]
internal partial class CamelCaseJsonContext : JsonSerializerContext internal partial class CamelCaseJsonContext : JsonSerializerContext
{ {

23
src/Program.cs
View File

@ -85,7 +85,7 @@ await Parallel.ForEachAsync(sitesTags, new ParallelOptions{MaxDegreeOfParallelis
var (site, tag) = st; var (site, tag) = st;
Console.WriteLine($"Fetching tag #{tag} from {site}"); Console.WriteLine($"Fetching tag #{tag} from {site}");
var url = $"https://{site}/tags/{tag}.json"; var url = $"https://{site}/api/v1/timelines/tag/{tag}?limit=40";
if (sitesRobotFile.TryGetValue(site, out var robotsFile)) if (sitesRobotFile.TryGetValue(site, out var robotsFile))
{ {
var allowed = robotsFile.IsAllowedAccess(new Uri(url), "GetMoarFediverse"); var allowed = robotsFile.IsAllowedAccess(new Uri(url), "GetMoarFediverse");
@ -114,23 +114,18 @@ await Parallel.ForEachAsync(sitesTags, new ParallelOptions{MaxDegreeOfParallelis
} }
var json = await response.Content.ReadAsStringAsync(); var json = await response.Content.ReadAsStringAsync();
var data = JsonSerializer.Deserialize(json, CamelCaseJsonContext.Default.TagResponse);
var data = JsonSerializer.Deserialize(json, CamelCaseJsonContext.Default.StatusArray);
if (data == null) if (data == null)
{ {
Console.WriteLine($"Error deserializing the response when pulling #{tag} posts from {site}"); Console.WriteLine($"Error deserializing the response when pulling #{tag} posts from {site}");
return; return;
} }
if (data.OrderedItems == null)
{
Console.WriteLine($"Got null on OrderedItems when pulling #{tag} posts from {site}");
return;
}
var count = 0; var count = 0;
foreach (var statusLink in data.OrderedItems.Where(i => !imported.Contains(i))) foreach (var statusLink in data.Where(i => !imported.Contains(i.Uri)))
{ {
statusesToLoadBag.Add(statusLink); statusesToLoadBag.Add(statusLink.Uri);
count++; count++;
} }
@ -168,12 +163,12 @@ if (importedList.Count > 5000)
File.WriteAllLines(importedPath, importedList); File.WriteAllLines(importedPath, importedList);
public class TagResponse public class Status
{ {
public string[]? OrderedItems { get; } public string Uri { get; }
public TagResponse(string[] orderedItems) public Status(string uri)
{ {
OrderedItems = orderedItems; Uri = uri;
} }
} }