diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..63c4462 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,30 @@ +name: build +on: + push: + branches: [ main ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 # get entire git tree, required for nerdbank gitversioning + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push the Docker image + shell: pwsh + run: | + Push-Location src + $version = (nbgv get-version -f json | ConvertFrom-Json).SimpleVersion + Write-Host "Version $version" + Pop-Location + + docker build . --tag ghcr.io/g3rv4/importdataasrelay:latest --tag "ghcr.io/g3rv4/importdataasrelay:$version" + docker push ghcr.io/g3rv4/importdataasrelay:latest + docker push "ghcr.io/g3rv4/importdataasrelay:$version" diff --git a/.gitignore b/.gitignore index 7c826dd..a46452c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ obj/ launchSettings.json .idea/ .DS_Store +data/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..972ddca --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +# syntax=docker/dockerfile:1 + +FROM mcr.microsoft.com/dotnet/sdk:6.0.403-alpine3.16 AS builder +WORKDIR /src +COPY src /src/ +RUN dotnet publish -c Release /src/ImportDataAsRelay.csproj -o /app + +FROM mcr.microsoft.com/dotnet/aspnet:6.0.11-alpine3.16 +VOLUME ["/data"] +ENV CONFIG_PATH=/data/config.json +COPY --from=builder /app /app +RUN ln -s /app/ImportDataAsRelay /bin/run +CMD ["/bin/run"] diff --git a/Helpers/CryptoHelper.cs b/Helpers/CryptoHelper.cs deleted file mode 100644 index 2015d42..0000000 --- a/Helpers/CryptoHelper.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Security.Cryptography; -using System.Text; - -namespace ImportDataAsRelay.Helpers; - -public static class CryptoHelper -{ - public static string GetSHA256Digest(string content) - { - using var sha256 = SHA256.Create(); - var hashValue = sha256.ComputeHash(Encoding.UTF8.GetBytes(content)); - return Convert.ToBase64String(hashValue); - } - - public static string ConvertPemToBase64(string filename) - { - var rsa = RSA.Create(); - rsa.ImportFromPem(File.ReadAllText(filename).ToCharArray()); - return Convert.ToBase64String(rsa.ExportRSAPrivateKey()); - } - - public static string Sign(string stringToSign) - { - using var rsaProvider = new RSACryptoServiceProvider(); - using var sha256 = SHA256.Create(); - rsaProvider.ImportRSAPrivateKey(Convert.FromBase64String(Environment.GetEnvironmentVariable("PRIVATE_KEY")), out _); - - var signature = rsaProvider.SignData(Encoding.UTF8.GetBytes(stringToSign), sha256); - return Convert.ToBase64String(signature); - } -} diff --git a/Helpers/MastodonHelper.cs b/Helpers/MastodonHelper.cs deleted file mode 100644 index 01c1e55..0000000 --- a/Helpers/MastodonHelper.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System.Net.Http.Headers; - -namespace ImportDataAsRelay.Helpers; - -public static class MastodonHelper -{ - private static string TargetHost = Environment.GetEnvironmentVariable("TARGET_HOST"); - private static string RelayHost = Environment.GetEnvironmentVariable("RELAY_HOST"); - - public static async Task EnqueueStatusToFetch(string statusUrl) - { - var client = new HttpClient(); - - var date = DateTime.UtcNow; - - var content = $@"{{ - ""@context"": ""https://www.w3.org/ns/activitystreams"", - ""actor"": ""https://{RelayHost}/actor"", - ""id"": ""https://{RelayHost}/activities/23af173e-e1fd-4283-93eb-514f1e5e5408"", - ""object"": ""{statusUrl}"", - ""to"": [ - ""https://{RelayHost}/followers"" - ], - ""type"": ""Announce"" -}}"; - var digest = CryptoHelper.GetSHA256Digest(content); - var requestContent = new StringContent(content); - - requestContent.Headers.Add("Digest", "SHA-256=" + digest); - - var stringToSign = $"(request-target): post /inbox\ndate: {date.ToString("R")}\nhost: {TargetHost}\ndigest: SHA-256={digest}\ncontent-length: {content.Length}"; - var signature = CryptoHelper.Sign(stringToSign); - requestContent.Headers.Add("Signature", $@"keyId=""https://{RelayHost}/actor#main-key"",algorithm=""rsa-sha256"",headers=""(request-target) date host digest content-length"",signature=""{signature}"""); - - requestContent.Headers.ContentType = new MediaTypeHeaderValue("application/activity+json"); - client.DefaultRequestHeaders.Date = date; - - var response = await client.PostAsync($"https://{TargetHost}/inbox", requestContent); - try - { - response.EnsureSuccessStatusCode(); - } - catch (Exception e) - { - Console.WriteLine("Error: " + e.Message); - Console.WriteLine("Status code: " + response.StatusCode); - - var body = await response.Content.ReadAsStringAsync(); - Console.WriteLine("Response content: " + body); - - throw; - } - } -} diff --git a/Program.cs b/Program.cs deleted file mode 100644 index 9ccca4a..0000000 --- a/Program.cs +++ /dev/null @@ -1,53 +0,0 @@ -using ImportDataAsRelay.Helpers; -using Jil; - -var interestingTagsEverywhere = new[] { "dotnet", "csharp" }; -var sources = new Dictionary -{ - ["hachyderm.io"] = new [] { "hachyderm" }, - ["mastodon.social"] = Array.Empty(), - ["dotnet.social"] = Array.Empty(), -}; - -var client = new HttpClient(); - -foreach (var (site, specificTags) in sources) -{ - var tags = specificTags.Concat(interestingTagsEverywhere).ToList(); - foreach (var tag in tags) - { - Console.WriteLine($"Fetching tag #{tag} from {site}"); - var response = await client.GetAsync($"https://{site}/tags/{tag}.json"); - try - { - response.EnsureSuccessStatusCode(); - } - catch (Exception e) - { - Console.WriteLine($"Error fetching tag, status code: {response.StatusCode}. Error: {e.Message}"); - continue; - } - - var json = await response.Content.ReadAsStringAsync(); - var data = JSON.Deserialize(json, Options.CamelCase); - - foreach (var statusLink in data.OrderedItems) - { - Console.WriteLine($"Bringing in {statusLink}"); - try - { - await MastodonHelper.EnqueueStatusToFetch(statusLink); - await Task.Delay(500); - } - catch (Exception e) - { - Console.WriteLine($"{e.Message}"); - } - } - } -} - -public class TagResponse -{ - public string[] OrderedItems { get; private set; } -} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..95fb407 --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# ImportDataAsRelay + +This is a small app that shows how you could use [FakeRelay](https://github.com/g3rv4/FakeRelay/) to import content into your instance that's tagged with hashtags you're interested in. + +This doesn't paginate over the tags, that means it will import up to 20 statuses per instance. This also keeps a txt file with all the statuses it has imported. + +## How can I run it? + +The easiest way is with docker compose. This `docker-compose.yml` shows how it can be used: + +``` +version: '2' +services: + importdata: + image: 'ghcr.io/g3rv4/importdataasrelay:latest' + volumes: + - '/path/to/data:/data' +``` + +On `/path/to/data`, you need to place a `config.json` that tells the system what you want. You could use something like this: + +``` +{ + "FakeRelayUrl": "https://fakerelay.gervas.io", + "FakeRelayApiKey": "1TxL6m1Esx6tnv4EPxscvAmdQN7qSn0nKeyoM7LD8b9mz+GNfrKaHiWgiT3QcNMUA+dWLyWD8qyl1MuKJ+4uHA==", + "Tags": [ + "dotnet", + "csharp" + ], + "Sites": [ + { + "Host": "hachyderm.io", + "SiteSpecificTags": [ + "hachyderm" + ] + }, + { + "Host": "mastodon.social" + } + ] +} +``` + +Once you have that set up, you can just execute it! and it will output what's going on. You can run this on a cron! + +``` +g3rv4@s1:~/docker/FakeRelay$ docker-compose run --rm import +Fetching tag #dotnet from mastodon.social +Fetching tag #hachyderm from hachyderm.io +Fetching tag #dotnet from hachyderm.io +Fetching tag #csharp from mastodon.social +Fetching tag #csharp from hachyderm.io +Bringing in https://dotnet.social/users/mzikmund/statuses/109458968117245196 +``` \ No newline at end of file diff --git a/src/Config.cs b/src/Config.cs new file mode 100644 index 0000000..20f3b23 --- /dev/null +++ b/src/Config.cs @@ -0,0 +1,75 @@ +using System.Collections.Immutable; +using Jil; + +namespace ImportDataAsRelay; + +public class Config +{ + public static Config? Instance { get; private set; } + + public string ImportedPath { get; } + public string FakeRelayUrl { get; } + public string FakeRelayApiKey { get; } + public ImmutableArray Tags { get; } + public ImmutableArray Sites { get; } + + + private Config(string importedPath, string fakeRelayUrl, string fakeRelayApiKey, ImmutableArray tags, ImmutableArray sites) + { + ImportedPath = importedPath; + FakeRelayUrl = fakeRelayUrl; + FakeRelayApiKey = fakeRelayApiKey; + Tags = tags; + Sites = sites; + } + + public static void Init(string path) + { + if (Instance != null) + { + return; + } + + var data = JSON.Deserialize(File.ReadAllText(path)); + + var importedPath = Path.Join(Path.GetDirectoryName(path), "imported.txt"); + + Instance = new Config(importedPath, data.FakeRelayUrl, data.FakeRelayApiKey, data.Tags.ToImmutableArray(), data.ImmutableSites); + } + + private class ConfigData + { + public string FakeRelayUrl { get; set; } + public string FakeRelayApiKey { get; set; } + public string[] Tags { get; set; } + public InternalSiteData[]? Sites { get; set; } + + public ImmutableArray ImmutableSites => + Sites == null + ? ImmutableArray.Empty + : Sites.Select(s => s.ToSiteData()) + .ToImmutableArray(); + + public class InternalSiteData + { + public string Host { get; private set; } + public string[]? SiteSpecificTags { get; private set; } + + public SiteData ToSiteData() => + new() + { + Host = Host, + SiteSpecificTags = + SiteSpecificTags == null + ? ImmutableArray.Empty + : SiteSpecificTags.ToImmutableArray() + }; + } + } + + public class SiteData + { + public string Host { get; init; } + public ImmutableArray SiteSpecificTags { get; init; } + } +} diff --git a/src/Directory.Build.props b/src/Directory.Build.props new file mode 100644 index 0000000..f036334 --- /dev/null +++ b/src/Directory.Build.props @@ -0,0 +1,9 @@ + + + + + 3.4.255 + all + + + \ No newline at end of file diff --git a/ImportDataAsRelay.csproj b/src/ImportDataAsRelay.csproj similarity index 100% rename from ImportDataAsRelay.csproj rename to src/ImportDataAsRelay.csproj diff --git a/src/Program.cs b/src/Program.cs new file mode 100644 index 0000000..a1a2ccd --- /dev/null +++ b/src/Program.cs @@ -0,0 +1,80 @@ +using System.Collections.Concurrent; +using ImportDataAsRelay; +using Jil; + +Config.Init(Environment.GetEnvironmentVariable("CONFIG_PATH")); + +var client = new HttpClient(); +var authClient = new HttpClient +{ + BaseAddress = new Uri(Config.Instance.FakeRelayUrl) +}; +authClient.DefaultRequestHeaders.Add("Authorization", "Bearer " + Config.Instance.FakeRelayApiKey); + +var importedPath = Config.Instance.ImportedPath; +if (!File.Exists(importedPath)) +{ + File.WriteAllText(importedPath, ""); +} + +var imported = File.ReadAllLines(importedPath).ToHashSet(); +var statusesToLoadBag = new ConcurrentBag(); + +ParallelOptions parallelOptions = new() +{ + MaxDegreeOfParallelism = 8 +}; + +await Parallel.ForEachAsync(Config.Instance.Sites, parallelOptions, async (site, _) => +{ + var tags = site.SiteSpecificTags.Concat(Config.Instance.Tags).ToList(); + foreach (var tag in tags) + { + Console.WriteLine($"Fetching tag #{tag} from {site.Host}"); + var response = await client.GetAsync($"https://{site.Host}/tags/{tag}.json"); + try + { + response.EnsureSuccessStatusCode(); + } + catch (Exception e) + { + Console.WriteLine($"Error fetching tag, status code: {response.StatusCode}. Error: {e.Message}"); + continue; + } + + var json = await response.Content.ReadAsStringAsync(); + var data = JSON.Deserialize(json, Options.CamelCase); + + foreach (var statusLink in data.OrderedItems.Where(i=>!imported.Contains(i))) + { + statusesToLoadBag.Add(statusLink); + } + } +}); + +var statusesToLoad = statusesToLoadBag.ToHashSet(); +var importedOnThisRun = new List(); +foreach (var statusLink in statusesToLoad) +{ + Console.WriteLine($"Bringing in {statusLink}"); + try + { + var content = new List>(); + content.Add(new KeyValuePair("statusUrl", statusLink)); + + var res = await authClient.PostAsync("index", new FormUrlEncodedContent(content)); + res.EnsureSuccessStatusCode(); + importedOnThisRun.Add(statusLink); + } + catch (Exception e) + { + Console.WriteLine($"{e.Message}"); + } +} + +File.AppendAllLines(importedPath, importedOnThisRun); + +public class TagResponse +{ + public string[] OrderedItems { get; private set; } +} \ No newline at end of file diff --git a/src/version.json b/src/version.json new file mode 100644 index 0000000..be8e5de --- /dev/null +++ b/src/version.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", + "version": "1.0", + "publicReleaseRefSpec": [ + "^refs/heads/main$", + "^refs/heads/v\\d+(?:\\.\\d+)?$" + ], + "cloudBuild": { + "buildNumber": { + "enabled": true + } + } + } \ No newline at end of file