From 3a710fbbd11988d9b68307766eeb836a10773450 Mon Sep 17 00:00:00 2001 From: Chris Caron Date: Sat, 11 May 2024 13:28:13 -0400 Subject: [PATCH] Added payload field mapper (#189) --- README.md | 46 +++- apprise_api/api/payload_mapper.py | 81 +++++++ apprise_api/api/tests/test_payload_mapper.py | 226 ++++++++++++++++++ apprise_api/api/tests/test_stateful_notify.py | 56 ++++- .../api/tests/test_stateless_notify.py | 33 +++ apprise_api/api/views.py | 38 ++- 6 files changed, 465 insertions(+), 15 deletions(-) create mode 100644 apprise_api/api/payload_mapper.py create mode 100644 apprise_api/api/tests/test_payload_mapper.py diff --git a/README.md b/README.md index 31d1777..021797b 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ curl -X POST \ http://localhost:8000/notify ``` -### Persistent Storage Solution +### Persistent (Stateful) Storage Solution You can pre-save all of your Apprise configuration and/or set of Apprise URLs and associate them with a `{KEY}` of your choosing. Once set, the configuration persists for retrieval by the `apprise` [CLI tool](https://github.com/caronc/apprise/wiki/CLI_Usage) or any other custom integration you've set up. The built in website with comes with a user interface that you can use to leverage these API calls as well. Those who wish to build their own application around this can use the following API end points: @@ -512,3 +512,47 @@ a.add(config) a.notify('test message') ``` +## Third Party Webhook Support +It can be understandable that third party applications can't always publish the format expected by this API tool. To work-around this, you can re-map the fields just before they're processed. For example; consider that we expect the follow minimum payload items for a stateful notification: +```json +{ + "body": "Message body" +} +``` + +But what if your tool you're using is only capable of sending: +```json +{ + "subject": "My Title", + "payload": "My Body" +} +``` + +We would want to map `subject` to `title` in this case and `payload` to `body`. This can easily be done using the `:` (colon) argument when we prepare our payload: + +```bash +# Note the keyword arguments prefixed with a `:` (colon). These +# instruct the API to map the payload (which we may not have control over) +# to align with what the Apprise API expects. +# +# We also convert `subject` to `title` too: +curl -X POST \ + -F "subject=Mesage Title" \ + -F "payload=Message Body" \ + "http://localhost:8000/notify/{KEY}?:subject=title&:payload=body" + +``` + +Here is the JSON Version and tests out the Stateless query (which requires at a minimum the `urls` and `body`: +```bash +# We also convert `subject` to `title` too: +curl -X POST -d '{"href": "mailto://user:pass@gmail.com", "subject":"My Title", "payload":"Body"}' \ + -H "Content-Type: application/json" \ + "http://localhost:8000/notify/{KEY}?:subject=title&:payload=body&:href=urls" +``` + +The colon `:` prefix is the switch that starts the re-mapping rule engine. You can do 3 possible things with the rule engine: +1. `:existing_key=expected_key`: Rename an existing (expected) payload key to one Apprise expects +1. `:existing_key=`: By setting no value, the existing key is simply removed from the payload entirely +1. `:expected_key=A value to give it`: You can also fix an expected apprise key to a pre-generated string value. + diff --git a/apprise_api/api/payload_mapper.py b/apprise_api/api/payload_mapper.py new file mode 100644 index 0000000..489662b --- /dev/null +++ b/apprise_api/api/payload_mapper.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 Chris Caron +# All rights reserved. +# +# This code is licensed under the MIT License. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files(the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and / or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions : +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from api.forms import NotifyForm + +# import the logging library +import logging + +# Get an instance of a logger +logger = logging.getLogger('django') + + +def remap_fields(rules, payload, form=None): + """ + Remaps fields in the payload provided based on the rules provided + + The key value of the dictionary identifies the payload key type you + wish to alter. If there is no value defined, then the entry is removed + + If there is a value provided, then it's key is swapped into the new key + provided. + + The purpose of this function is to allow people to re-map the fields + that are being posted to the Apprise API before hand. Mapping them + can allow 3rd party programs that post 'subject' and 'content' to + be remapped to say 'title' and 'body' respectively + + """ + + # Prepare our Form (identifies our expected keys) + form = NotifyForm() if form is None else form + + # First generate our expected keys; only these can be mapped + expected_keys = set(form.fields.keys()) + for _key, value in rules.items(): + + key = _key.lower() + if key in payload and not value: + # Remove element + del payload[key] + continue + + vkey = value.lower() + if vkey in expected_keys and key in payload: + if key not in expected_keys or vkey not in payload: + # replace + payload[vkey] = payload[key] + del payload[key] + + elif vkey in payload: + # swap + _tmp = payload[vkey] + payload[vkey] = payload[key] + payload[key] = _tmp + + elif key in expected_keys or key in payload: + # assignment + payload[key] = value + + return True diff --git a/apprise_api/api/tests/test_payload_mapper.py b/apprise_api/api/tests/test_payload_mapper.py new file mode 100644 index 0000000..da7dd4a --- /dev/null +++ b/apprise_api/api/tests/test_payload_mapper.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 Chris Caron +# All rights reserved. +# +# This code is licensed under the MIT License. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files(the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and / or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions : +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from django.test import SimpleTestCase +from ..payload_mapper import remap_fields + + +class NotifyPayloadMapper(SimpleTestCase): + """ + Test Payload Mapper + """ + + def test_remap_fields(self): + """ + Test payload re-mapper + """ + + # + # No rules defined + # + rules = {} + payload = { + 'format': 'markdown', + 'title': 'title', + 'body': '# body', + } + payload_orig = payload.copy() + + # Map our fields + remap_fields(rules, payload) + + # no change is made + assert payload == payload_orig + + # + # rules defined - test 1 + # + rules = { + # map 'as' to 'format' + 'as': 'format', + # map 'subject' to 'title' + 'subject': 'title', + # map 'content' to 'body' + 'content': 'body', + # 'missing' is an invalid entry so this will be skipped + 'unknown': 'missing', + + # Empty field + 'attachment': '', + + # Garbage is an field that can be removed since it doesn't + # conflict with the form + 'garbage': '', + + # Tag + 'tag': 'test', + } + payload = { + 'as': 'markdown', + 'subject': 'title', + 'content': '# body', + 'tag': '', + 'unknown': 'hmm', + 'attachment': '', + 'garbage': '', + } + + # Map our fields + remap_fields(rules, payload) + + # Our field mappings have taken place + assert payload == { + 'tag': 'test', + 'unknown': 'missing', + 'format': 'markdown', + 'title': 'title', + 'body': '# body', + } + + # + # rules defined - test 2 + # + rules = { + # + # map 'content' to 'body' + 'content': 'body', + # a double mapping to body will trigger an error + 'message': 'body', + # Swapping fields + 'body': 'another set of data', + } + payload = { + 'as': 'markdown', + 'subject': 'title', + 'content': '# content body', + 'message': '# message body', + 'body': 'another set of data', + } + + # Map our fields + remap_fields(rules, payload) + + # Our information gets swapped + assert payload == { + 'as': 'markdown', + 'subject': 'title', + 'body': 'another set of data', + } + + # + # swapping fields - test 3 + # + rules = { + # + # map 'content' to 'body' + 'title': 'body', + } + payload = { + 'format': 'markdown', + 'title': 'body', + 'body': '# title', + } + + # Map our fields + remap_fields(rules, payload) + + # Our information gets swapped + assert payload == { + 'format': 'markdown', + 'title': '# title', + 'body': 'body', + } + + # + # swapping fields - test 4 + # + rules = { + # + # map 'content' to 'body' + 'title': 'body', + } + payload = { + 'format': 'markdown', + 'title': 'body', + } + + # Map our fields + remap_fields(rules, payload) + + # Our information gets swapped + assert payload == { + 'format': 'markdown', + 'body': 'body', + } + + # + # swapping fields - test 5 + # + rules = { + # + # map 'content' to 'body' + 'content': 'body', + } + payload = { + 'format': 'markdown', + 'content': 'the message', + 'body': 'to-be-replaced', + } + + # Map our fields + remap_fields(rules, payload) + + # Our information gets swapped + assert payload == { + 'format': 'markdown', + 'body': 'the message', + } + + + # + # mapping of fields don't align - test 6 + # + rules = { + 'payload': 'body', + 'fmt': 'format', + 'extra': 'tag', + } + payload = { + 'format': 'markdown', + 'type': 'info', + 'title': '', + 'body': '## test notifiction', + 'attachment': None, + 'tag': 'general', + 'tags': '', + } + + # Make a copy of our original payload + payload_orig = payload.copy() + + # Map our fields + remap_fields(rules, payload) + + # There are no rules applied since nothing aligned + assert payload == payload_orig diff --git a/apprise_api/api/tests/test_stateful_notify.py b/apprise_api/api/tests/test_stateful_notify.py index 63983cc..790cb50 100644 --- a/apprise_api/api/tests/test_stateful_notify.py +++ b/apprise_api/api/tests/test_stateful_notify.py @@ -27,6 +27,7 @@ from django.test.utils import override_settings from unittest.mock import patch, Mock from ..forms import NotifyForm from ..utils import ConfigCache +from json import dumps import os import re import apprise @@ -107,7 +108,7 @@ class StatefulNotifyTests(SimpleTestCase): assert len(entries) == 3 form_data = { - 'body': '## test notifiction', + 'body': '## test notification', 'format': apprise.NotifyFormat.MARKDOWN, 'tag': 'general', } @@ -128,7 +129,40 @@ class StatefulNotifyTests(SimpleTestCase): mock_post.reset_mock() form_data = { - 'body': '## test notifiction', + 'payload': '## test notification', + 'fmt': apprise.NotifyFormat.MARKDOWN, + 'extra': 'general', + } + + # We sent the notification successfully (use our rule mapping) + # FORM + response = self.client.post( + f'/notify/{key}/?:payload=body&:fmt=format&:extra=tag', + form_data) + assert response.status_code == 200 + assert mock_post.call_count == 1 + + mock_post.reset_mock() + + form_data = { + 'payload': '## test notification', + 'fmt': apprise.NotifyFormat.MARKDOWN, + 'extra': 'general', + } + + # We sent the notification successfully (use our rule mapping) + # JSON + response = self.client.post( + f'/notify/{key}/?:payload=body&:fmt=format&:extra=tag', + dumps(form_data), + content_type="application/json") + assert response.status_code == 200 + assert mock_post.call_count == 1 + + mock_post.reset_mock() + + form_data = { + 'body': '## test notification', 'format': apprise.NotifyFormat.MARKDOWN, 'tag': 'no-on-with-this-tag', } @@ -180,7 +214,7 @@ class StatefulNotifyTests(SimpleTestCase): assert len(entries) == 3 form_data = { - 'body': '## test notifiction', + 'body': '## test notification', 'format': apprise.NotifyFormat.MARKDOWN, } @@ -204,7 +238,7 @@ class StatefulNotifyTests(SimpleTestCase): # Test tagging now # form_data = { - 'body': '## test notifiction', + 'body': '## test notification', 'format': apprise.NotifyFormat.MARKDOWN, 'tag': 'general+json', } @@ -226,7 +260,7 @@ class StatefulNotifyTests(SimpleTestCase): mock_post.reset_mock() form_data = { - 'body': '## test notifiction', + 'body': '## test notification', 'format': apprise.NotifyFormat.MARKDOWN, # Plus with space inbetween 'tag': 'general + json', @@ -248,7 +282,7 @@ class StatefulNotifyTests(SimpleTestCase): mock_post.reset_mock() form_data = { - 'body': '## test notifiction', + 'body': '## test notification', 'format': apprise.NotifyFormat.MARKDOWN, # Space (AND) 'tag': 'general json', @@ -269,7 +303,7 @@ class StatefulNotifyTests(SimpleTestCase): mock_post.reset_mock() form_data = { - 'body': '## test notifiction', + 'body': '## test notification', 'format': apprise.NotifyFormat.MARKDOWN, # Comma (OR) 'tag': 'general, devops', @@ -351,7 +385,7 @@ class StatefulNotifyTests(SimpleTestCase): for tag in ('user1', 'user2'): form_data = { - 'body': '## test notifiction', + 'body': '## test notification', 'format': apprise.NotifyFormat.MARKDOWN, 'tag': tag, } @@ -374,7 +408,7 @@ class StatefulNotifyTests(SimpleTestCase): # Now let's notify by our group form_data = { - 'body': '## test notifiction', + 'body': '## test notification', 'format': apprise.NotifyFormat.MARKDOWN, 'tag': 'mygroup', } @@ -448,7 +482,7 @@ class StatefulNotifyTests(SimpleTestCase): for tag in ('user1', 'user2'): form_data = { - 'body': '## test notifiction', + 'body': '## test notification', 'format': apprise.NotifyFormat.MARKDOWN, 'tag': tag, } @@ -471,7 +505,7 @@ class StatefulNotifyTests(SimpleTestCase): # Now let's notify by our group form_data = { - 'body': '## test notifiction', + 'body': '## test notification', 'format': apprise.NotifyFormat.MARKDOWN, 'tag': 'mygroup', } diff --git a/apprise_api/api/tests/test_stateless_notify.py b/apprise_api/api/tests/test_stateless_notify.py index 6ee0a98..3eccb71 100644 --- a/apprise_api/api/tests/test_stateless_notify.py +++ b/apprise_api/api/tests/test_stateless_notify.py @@ -120,6 +120,39 @@ class StatelessNotifyTests(SimpleTestCase): # Reset our mock object mock_notify.reset_mock() + form_data = { + 'payload': '## test notification', + 'fmt': apprise.NotifyFormat.MARKDOWN, + 'extra': 'mailto://user:pass@hotmail.com', + } + + # We sent the notification successfully (use our rule mapping) + # FORM + response = self.client.post( + f'/notify/?:payload=body&:fmt=format&:extra=urls', + form_data) + assert response.status_code == 200 + assert mock_notify.call_count == 1 + + mock_notify.reset_mock() + + form_data = { + 'payload': '## test notification', + 'fmt': apprise.NotifyFormat.MARKDOWN, + 'extra': 'mailto://user:pass@hotmail.com', + } + + # We sent the notification successfully (use our rule mapping) + # JSON + response = self.client.post( + '/notify/?:payload=body&:fmt=format&:extra=urls', + json.dumps(form_data), + content_type="application/json") + assert response.status_code == 200 + assert mock_notify.call_count == 1 + + mock_notify.reset_mock() + # Long Filename attach_data = { 'attachment': SimpleUploadedFile( diff --git a/apprise_api/api/views.py b/apprise_api/api/views.py index 281e68c..3e569c2 100644 --- a/apprise_api/api/views.py +++ b/apprise_api/api/views.py @@ -35,6 +35,7 @@ from django.views.decorators.gzip import gzip_page from django.utils.translation import gettext_lazy as _ from django.core.serializers.json import DjangoJSONEncoder +from .payload_mapper import remap_fields from .utils import parse_attachments from .utils import ConfigCache from .utils import apply_global_filters @@ -662,10 +663,22 @@ class NotifyView(View): and ACCEPT_ALL.match(request.headers.get('accept', '')) else \ MIME_IS_JSON.match(request.headers.get('accept', '')) is not None + # rules + rules = {k[1:]: v for k,v in request.GET.items() if k[0] == ':'} + # our content content = {} if not json_payload: - form = NotifyForm(data=request.POST, files=request.FILES) + if rules: + # Create a copy + data = request.POST.copy() + remap_fields(rules, data) + + else: + # Just create a pointer + data = request.POST + + form = NotifyForm(data=data, files=request.FILES) if form.is_valid(): content.update(form.cleaned_data) @@ -675,6 +688,10 @@ class NotifyView(View): # load our JSON content content = json.loads(request.body.decode('utf-8')) + # Apply content rules + if rules: + remap_fields(rules, content) + except (RequestDataTooBig): # DATA_UPLOAD_MAX_MEMORY_SIZE exceeded it's value; this is usually the case # when there is a very large flie attachment that can't be pulled out of the @@ -1169,11 +1186,22 @@ class StatelessNotifyView(View): and ACCEPT_ALL.match(request.headers.get('accept', '')) else \ MIME_IS_JSON.match(request.headers.get('accept', '')) is not None + # rules + rules = {k[1:]: v for k,v in request.GET.items() if k[0] == ':'} + # our content content = {} if not json_payload: - content = {} - form = NotifyByUrlForm(request.POST, request.FILES) + if rules: + # Create a copy + data = request.POST.copy() + remap_fields(rules, data, form=NotifyByUrlForm()) + + else: + # Just create a pointer + data = request.POST + + form = NotifyByUrlForm(data=data, files=request.FILES) if form.is_valid(): content.update(form.cleaned_data) @@ -1183,6 +1211,10 @@ class StatelessNotifyView(View): # load our JSON content content = json.loads(request.body.decode('utf-8')) + # Apply content rules + if rules: + remap_fields(rules, content, form=NotifyByUrlForm()) + except (RequestDataTooBig): # DATA_UPLOAD_MAX_MEMORY_SIZE exceeded it's value; this is usually the case # when there is a very large flie attachment that can't be pulled out of the