Use FakeRelay, build a docker image and set things in a config file

.github/workflows/build.yml

@@ -0,0 +1,30 @@
+name: build
+on:
+  push:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # get entire git tree, required for nerdbank gitversioning
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push the Docker image
+        shell: pwsh
+        run: |
+          Push-Location src
+          $version = (nbgv get-version -f json | ConvertFrom-Json).SimpleVersion
+          Write-Host "Version $version"
+          Pop-Location
+
+          docker build . --tag ghcr.io/g3rv4/importdataasrelay:latest --tag "ghcr.io/g3rv4/importdataasrelay:$version"
+          docker push ghcr.io/g3rv4/importdataasrelay:latest
+          docker push "ghcr.io/g3rv4/importdataasrelay:$version"

.gitignore

@@ -6,3 +6,4 @@ obj/
 launchSettings.json
 .idea/
 .DS_Store
+data/

Dockerfile

@@ -0,0 +1,13 @@
+# syntax=docker/dockerfile:1
+
+FROM mcr.microsoft.com/dotnet/sdk:6.0.403-alpine3.16 AS builder
+WORKDIR /src
+COPY src /src/
+RUN dotnet publish -c Release /src/ImportDataAsRelay.csproj -o /app
+
+FROM mcr.microsoft.com/dotnet/aspnet:6.0.11-alpine3.16  
+VOLUME ["/data"]
+ENV CONFIG_PATH=/data/config.json
+COPY --from=builder /app /app
+RUN ln -s /app/ImportDataAsRelay /bin/run
+CMD ["/bin/run"]

Helpers/CryptoHelper.cs

@@ -1,31 +0,0 @@
-using System.Security.Cryptography;
-using System.Text;
-
-namespace ImportDataAsRelay.Helpers;
-
-public static class CryptoHelper
-{
-    public static string GetSHA256Digest(string content)
-    {
-        using var sha256 = SHA256.Create();
-        var hashValue = sha256.ComputeHash(Encoding.UTF8.GetBytes(content));
-        return Convert.ToBase64String(hashValue);
-    }
-
-    public static string ConvertPemToBase64(string filename)
-    {
-        var rsa = RSA.Create();
-        rsa.ImportFromPem(File.ReadAllText(filename).ToCharArray());
-        return Convert.ToBase64String(rsa.ExportRSAPrivateKey());
-    }
-
-    public static string Sign(string stringToSign)
-    {
-        using var rsaProvider = new RSACryptoServiceProvider();
-        using var sha256 = SHA256.Create();
-        rsaProvider.ImportRSAPrivateKey(Convert.FromBase64String(Environment.GetEnvironmentVariable("PRIVATE_KEY")), out _);
-        
-        var signature = rsaProvider.SignData(Encoding.UTF8.GetBytes(stringToSign), sha256);
-        return Convert.ToBase64String(signature);
-    }
-}

Helpers/MastodonHelper.cs

@@ -1,54 +0,0 @@
-using System.Net.Http.Headers;
-
-namespace ImportDataAsRelay.Helpers;
-
-public static class MastodonHelper
-{
-    private static string TargetHost = Environment.GetEnvironmentVariable("TARGET_HOST");
-    private static string RelayHost = Environment.GetEnvironmentVariable("RELAY_HOST");
-    
-    public static async Task EnqueueStatusToFetch(string statusUrl)
-    {
-        var client = new HttpClient();
-
-        var date = DateTime.UtcNow;
-        
-        var content = $@"{{
-    ""@context"": ""https://www.w3.org/ns/activitystreams"",
-    ""actor"": ""https://{RelayHost}/actor"",
-    ""id"": ""https://{RelayHost}/activities/23af173e-e1fd-4283-93eb-514f1e5e5408"",
-    ""object"": ""{statusUrl}"",
-    ""to"": [
-        ""https://{RelayHost}/followers""
-    ],
-    ""type"": ""Announce""
-}}";
-        var digest = CryptoHelper.GetSHA256Digest(content);
-        var requestContent = new StringContent(content);
-
-        requestContent.Headers.Add("Digest", "SHA-256=" + digest);
-
-        var stringToSign = $"(request-target): post /inbox\ndate: {date.ToString("R")}\nhost: {TargetHost}\ndigest: SHA-256={digest}\ncontent-length: {content.Length}";
-        var signature = CryptoHelper.Sign(stringToSign);
-        requestContent.Headers.Add("Signature", $@"keyId=""https://{RelayHost}/actor#main-key"",algorithm=""rsa-sha256"",headers=""(request-target) date host digest content-length"",signature=""{signature}""");
-
-        requestContent.Headers.ContentType = new MediaTypeHeaderValue("application/activity+json");
-        client.DefaultRequestHeaders.Date = date;
-
-        var response = await client.PostAsync($"https://{TargetHost}/inbox", requestContent);
-        try
-        {
-            response.EnsureSuccessStatusCode();
-        }
-        catch (Exception e)
-        {
-            Console.WriteLine("Error: " + e.Message);
-            Console.WriteLine("Status code: " + response.StatusCode);
-
-            var body = await response.Content.ReadAsStringAsync();
-            Console.WriteLine("Response content: " + body);
-
-            throw;
-        }
-    }
-}

Program.cs

@@ -1,53 +0,0 @@
-using ImportDataAsRelay.Helpers;
-using Jil;
-
-var interestingTagsEverywhere = new[] { "dotnet", "csharp" };
-var sources = new Dictionary<string, string[]>
-{
-    ["hachyderm.io"] = new [] { "hachyderm" },
-    ["mastodon.social"] = Array.Empty<string>(),
-    ["dotnet.social"] = Array.Empty<string>(),
-};
-
-var client = new HttpClient();
-
-foreach (var (site, specificTags) in sources)
-{
-    var tags = specificTags.Concat(interestingTagsEverywhere).ToList();
-    foreach (var tag in tags)
-    {
-        Console.WriteLine($"Fetching tag #{tag} from {site}");
-        var response = await client.GetAsync($"https://{site}/tags/{tag}.json");
-        try
-        {
-            response.EnsureSuccessStatusCode();
-        }
-        catch (Exception e)
-        {
-            Console.WriteLine($"Error fetching tag, status code: {response.StatusCode}. Error: {e.Message}");
-            continue;
-        }
-
-        var json = await response.Content.ReadAsStringAsync();
-        var data = JSON.Deserialize<TagResponse>(json, Options.CamelCase);
-
-        foreach (var statusLink in data.OrderedItems)
-        {
-            Console.WriteLine($"Bringing in {statusLink}");
-            try
-            {
-                await MastodonHelper.EnqueueStatusToFetch(statusLink);
-                await Task.Delay(500);
-            }
-            catch (Exception e)
-            {
-                Console.WriteLine($"{e.Message}");
-            }
-        }
-    }
-}
-
-public class TagResponse
-{
-    public string[] OrderedItems { get; private set; }
-}

README.md

@@ -0,0 +1,54 @@
+# ImportDataAsRelay
+
+This is a small app that shows how you could use [FakeRelay](https://github.com/g3rv4/FakeRelay/) to import content into your instance that's tagged with hashtags you're interested in.
+
+This doesn't paginate over the tags, that means it will import up to 20 statuses per instance. This also keeps a txt file with all the statuses it has imported.
+
+## How can I run it?
+
+The easiest way is with docker compose. This `docker-compose.yml` shows how it can be used:
+
+```
+version: '2'
+services:
+  importdata:
+    image: 'ghcr.io/g3rv4/importdataasrelay:latest'
+    volumes:
+      - '/path/to/data:/data'
+```
+
+On `/path/to/data`, you need to place a `config.json` that tells the system what you want. You could use something like this:
+
+```
+{
+    "FakeRelayUrl": "https://fakerelay.gervas.io",
+    "FakeRelayApiKey": "1TxL6m1Esx6tnv4EPxscvAmdQN7qSn0nKeyoM7LD8b9mz+GNfrKaHiWgiT3QcNMUA+dWLyWD8qyl1MuKJ+4uHA==",
+    "Tags": [
+        "dotnet",
+        "csharp"
+    ],
+    "Sites": [
+        {
+            "Host": "hachyderm.io",
+            "SiteSpecificTags": [
+                "hachyderm"
+            ]
+        },
+        {
+            "Host": "mastodon.social"
+        }
+    ]
+}
+```
+
+Once you have that set up, you can just execute it! and it will output what's going on. You can run this on a cron!
+
+```
+g3rv4@s1:~/docker/FakeRelay$ docker-compose run --rm import
+Fetching tag #dotnet from mastodon.social
+Fetching tag #hachyderm from hachyderm.io
+Fetching tag #dotnet from hachyderm.io
+Fetching tag #csharp from mastodon.social
+Fetching tag #csharp from hachyderm.io
+Bringing in https://dotnet.social/users/mzikmund/statuses/109458968117245196
+```

src/Config.cs

@@ -0,0 +1,75 @@
+using System.Collections.Immutable;
+using Jil;
+
+namespace ImportDataAsRelay;
+
+public class Config
+{
+    public static Config? Instance { get; private set; }
+    
+    public string ImportedPath { get; }
+    public string FakeRelayUrl { get; }
+    public string FakeRelayApiKey { get; }
+    public ImmutableArray<string> Tags { get; }
+    public ImmutableArray<SiteData> Sites { get; }
+
+
+    private Config(string importedPath, string fakeRelayUrl, string fakeRelayApiKey, ImmutableArray<string> tags, ImmutableArray<SiteData> sites)
+    {
+        ImportedPath = importedPath;
+        FakeRelayUrl = fakeRelayUrl;
+        FakeRelayApiKey = fakeRelayApiKey;
+        Tags = tags;
+        Sites = sites;
+    }
+
+    public static void Init(string path)
+    {
+        if (Instance != null)
+        {
+            return;
+        }
+
+        var data = JSON.Deserialize<ConfigData>(File.ReadAllText(path));
+
+        var importedPath = Path.Join(Path.GetDirectoryName(path), "imported.txt");
+        
+        Instance = new Config(importedPath, data.FakeRelayUrl, data.FakeRelayApiKey, data.Tags.ToImmutableArray(), data.ImmutableSites);
+    }
+
+    private class ConfigData
+    {
+        public string FakeRelayUrl { get; set; }
+        public string FakeRelayApiKey { get; set; }
+        public string[] Tags { get; set; }
+        public InternalSiteData[]? Sites { get; set; }
+
+        public ImmutableArray<SiteData> ImmutableSites =>
+            Sites == null
+                ? ImmutableArray<SiteData>.Empty
+                : Sites.Select(s => s.ToSiteData())
+                       .ToImmutableArray();
+
+        public class InternalSiteData
+        {
+            public string Host { get; private set; }
+            public string[]? SiteSpecificTags { get; private set; }
+
+            public SiteData ToSiteData() =>
+                new()
+                {
+                    Host = Host,
+                    SiteSpecificTags =
+                        SiteSpecificTags == null
+                        ? ImmutableArray<string>.Empty
+                        : SiteSpecificTags.ToImmutableArray()
+                };
+        }
+    }
+
+    public class SiteData
+    {
+        public string Host { get; init; }
+        public ImmutableArray<string> SiteSpecificTags { get; init; }
+    }
+}

src/Directory.Build.props

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <PackageReference Include="Nerdbank.GitVersioning" Condition="!Exists('packages.config')">
+      <Version>3.4.255</Version>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+</Project>

src/Program.cs

@@ -0,0 +1,80 @@
+using System.Collections.Concurrent;
+using ImportDataAsRelay;
+using Jil;
+
+Config.Init(Environment.GetEnvironmentVariable("CONFIG_PATH"));
+
+var client = new HttpClient();
+var authClient = new HttpClient
+{
+    BaseAddress = new Uri(Config.Instance.FakeRelayUrl)
+};
+authClient.DefaultRequestHeaders.Add("Authorization", "Bearer " + Config.Instance.FakeRelayApiKey);
+
+var importedPath = Config.Instance.ImportedPath;
+if (!File.Exists(importedPath))
+{
+    File.WriteAllText(importedPath, "");
+}
+
+var imported = File.ReadAllLines(importedPath).ToHashSet();
+var statusesToLoadBag = new ConcurrentBag<string>();
+
+ParallelOptions parallelOptions = new()
+{
+    MaxDegreeOfParallelism = 8
+};
+
+await Parallel.ForEachAsync(Config.Instance.Sites, parallelOptions, async (site, _) =>
+{
+    var tags = site.SiteSpecificTags.Concat(Config.Instance.Tags).ToList();
+    foreach (var tag in tags)
+    {
+        Console.WriteLine($"Fetching tag #{tag} from {site.Host}");
+        var response = await client.GetAsync($"https://{site.Host}/tags/{tag}.json");
+        try
+        {
+            response.EnsureSuccessStatusCode();
+        }
+        catch (Exception e)
+        {
+            Console.WriteLine($"Error fetching tag, status code: {response.StatusCode}. Error: {e.Message}");
+            continue;
+        }
+
+        var json = await response.Content.ReadAsStringAsync();
+        var data = JSON.Deserialize<TagResponse>(json, Options.CamelCase);
+
+        foreach (var statusLink in data.OrderedItems.Where(i=>!imported.Contains(i)))
+        {
+            statusesToLoadBag.Add(statusLink);
+        }
+    }
+});
+
+var statusesToLoad = statusesToLoadBag.ToHashSet();
+var importedOnThisRun = new List<string>();
+foreach (var statusLink in statusesToLoad)
+{
+    Console.WriteLine($"Bringing in {statusLink}");
+    try
+    {
+        var content = new List<KeyValuePair<string, string>>();
+        content.Add(new KeyValuePair<string, string>("statusUrl", statusLink));
+
+        var res = await authClient.PostAsync("index", new FormUrlEncodedContent(content));
+        res.EnsureSuccessStatusCode();
+        importedOnThisRun.Add(statusLink);
+    }
+    catch (Exception e)
+    {
+        Console.WriteLine($"{e.Message}");
+    }
+}
+
+File.AppendAllLines(importedPath, importedOnThisRun);
+
+public class TagResponse
+{
+    public string[] OrderedItems { get; private set; }
+}

src/version.json

@@ -0,0 +1,13 @@
+{
+    "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json",
+    "version": "1.0",
+    "publicReleaseRefSpec": [
+      "^refs/heads/main$",
+      "^refs/heads/v\\d+(?:\\.\\d+)?$"
+    ],
+    "cloudBuild": {
+      "buildNumber": {
+        "enabled": true
+      }
+    }
+  }