Added payload field mapper (#189)

This commit is contained in:
Chris Caron 2024-05-11 13:28:13 -04:00 committed by GitHub
parent 1c2e5ff1ea
commit 3a710fbbd1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 465 additions and 15 deletions

View File

@ -210,7 +210,7 @@ curl -X POST \
http://localhost:8000/notify
```
### Persistent Storage Solution
### Persistent (Stateful) Storage Solution
You can pre-save all of your Apprise configuration and/or set of Apprise URLs and associate them with a `{KEY}` of your choosing. Once set, the configuration persists for retrieval by the `apprise` [CLI tool](https://github.com/caronc/apprise/wiki/CLI_Usage) or any other custom integration you've set up. The built in website with comes with a user interface that you can use to leverage these API calls as well. Those who wish to build their own application around this can use the following API end points:
@ -512,3 +512,47 @@ a.add(config)
a.notify('test message')
```
## Third Party Webhook Support
It can be understandable that third party applications can't always publish the format expected by this API tool. To work-around this, you can re-map the fields just before they're processed. For example; consider that we expect the follow minimum payload items for a stateful notification:
```json
{
"body": "Message body"
}
```
But what if your tool you're using is only capable of sending:
```json
{
"subject": "My Title",
"payload": "My Body"
}
```
We would want to map `subject` to `title` in this case and `payload` to `body`. This can easily be done using the `:` (colon) argument when we prepare our payload:
```bash
# Note the keyword arguments prefixed with a `:` (colon). These
# instruct the API to map the payload (which we may not have control over)
# to align with what the Apprise API expects.
#
# We also convert `subject` to `title` too:
curl -X POST \
-F "subject=Mesage Title" \
-F "payload=Message Body" \
"http://localhost:8000/notify/{KEY}?:subject=title&:payload=body"
```
Here is the JSON Version and tests out the Stateless query (which requires at a minimum the `urls` and `body`:
```bash
# We also convert `subject` to `title` too:
curl -X POST -d '{"href": "mailto://user:pass@gmail.com", "subject":"My Title", "payload":"Body"}' \
-H "Content-Type: application/json" \
"http://localhost:8000/notify/{KEY}?:subject=title&:payload=body&:href=urls"
```
The colon `:` prefix is the switch that starts the re-mapping rule engine. You can do 3 possible things with the rule engine:
1. `:existing_key=expected_key`: Rename an existing (expected) payload key to one Apprise expects
1. `:existing_key=`: By setting no value, the existing key is simply removed from the payload entirely
1. `:expected_key=A value to give it`: You can also fix an expected apprise key to a pre-generated string value.

View File

@ -0,0 +1,81 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2024 Chris Caron <lead2gold@gmail.com>
# All rights reserved.
#
# This code is licensed under the MIT License.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files(the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions :
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
from api.forms import NotifyForm
# import the logging library
import logging
# Get an instance of a logger
logger = logging.getLogger('django')
def remap_fields(rules, payload, form=None):
"""
Remaps fields in the payload provided based on the rules provided
The key value of the dictionary identifies the payload key type you
wish to alter. If there is no value defined, then the entry is removed
If there is a value provided, then it's key is swapped into the new key
provided.
The purpose of this function is to allow people to re-map the fields
that are being posted to the Apprise API before hand. Mapping them
can allow 3rd party programs that post 'subject' and 'content' to
be remapped to say 'title' and 'body' respectively
"""
# Prepare our Form (identifies our expected keys)
form = NotifyForm() if form is None else form
# First generate our expected keys; only these can be mapped
expected_keys = set(form.fields.keys())
for _key, value in rules.items():
key = _key.lower()
if key in payload and not value:
# Remove element
del payload[key]
continue
vkey = value.lower()
if vkey in expected_keys and key in payload:
if key not in expected_keys or vkey not in payload:
# replace
payload[vkey] = payload[key]
del payload[key]
elif vkey in payload:
# swap
_tmp = payload[vkey]
payload[vkey] = payload[key]
payload[key] = _tmp
elif key in expected_keys or key in payload:
# assignment
payload[key] = value
return True

View File

@ -0,0 +1,226 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2024 Chris Caron <lead2gold@gmail.com>
# All rights reserved.
#
# This code is licensed under the MIT License.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files(the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions :
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
from django.test import SimpleTestCase
from ..payload_mapper import remap_fields
class NotifyPayloadMapper(SimpleTestCase):
"""
Test Payload Mapper
"""
def test_remap_fields(self):
"""
Test payload re-mapper
"""
#
# No rules defined
#
rules = {}
payload = {
'format': 'markdown',
'title': 'title',
'body': '# body',
}
payload_orig = payload.copy()
# Map our fields
remap_fields(rules, payload)
# no change is made
assert payload == payload_orig
#
# rules defined - test 1
#
rules = {
# map 'as' to 'format'
'as': 'format',
# map 'subject' to 'title'
'subject': 'title',
# map 'content' to 'body'
'content': 'body',
# 'missing' is an invalid entry so this will be skipped
'unknown': 'missing',
# Empty field
'attachment': '',
# Garbage is an field that can be removed since it doesn't
# conflict with the form
'garbage': '',
# Tag
'tag': 'test',
}
payload = {
'as': 'markdown',
'subject': 'title',
'content': '# body',
'tag': '',
'unknown': 'hmm',
'attachment': '',
'garbage': '',
}
# Map our fields
remap_fields(rules, payload)
# Our field mappings have taken place
assert payload == {
'tag': 'test',
'unknown': 'missing',
'format': 'markdown',
'title': 'title',
'body': '# body',
}
#
# rules defined - test 2
#
rules = {
#
# map 'content' to 'body'
'content': 'body',
# a double mapping to body will trigger an error
'message': 'body',
# Swapping fields
'body': 'another set of data',
}
payload = {
'as': 'markdown',
'subject': 'title',
'content': '# content body',
'message': '# message body',
'body': 'another set of data',
}
# Map our fields
remap_fields(rules, payload)
# Our information gets swapped
assert payload == {
'as': 'markdown',
'subject': 'title',
'body': 'another set of data',
}
#
# swapping fields - test 3
#
rules = {
#
# map 'content' to 'body'
'title': 'body',
}
payload = {
'format': 'markdown',
'title': 'body',
'body': '# title',
}
# Map our fields
remap_fields(rules, payload)
# Our information gets swapped
assert payload == {
'format': 'markdown',
'title': '# title',
'body': 'body',
}
#
# swapping fields - test 4
#
rules = {
#
# map 'content' to 'body'
'title': 'body',
}
payload = {
'format': 'markdown',
'title': 'body',
}
# Map our fields
remap_fields(rules, payload)
# Our information gets swapped
assert payload == {
'format': 'markdown',
'body': 'body',
}
#
# swapping fields - test 5
#
rules = {
#
# map 'content' to 'body'
'content': 'body',
}
payload = {
'format': 'markdown',
'content': 'the message',
'body': 'to-be-replaced',
}
# Map our fields
remap_fields(rules, payload)
# Our information gets swapped
assert payload == {
'format': 'markdown',
'body': 'the message',
}
#
# mapping of fields don't align - test 6
#
rules = {
'payload': 'body',
'fmt': 'format',
'extra': 'tag',
}
payload = {
'format': 'markdown',
'type': 'info',
'title': '',
'body': '## test notifiction',
'attachment': None,
'tag': 'general',
'tags': '',
}
# Make a copy of our original payload
payload_orig = payload.copy()
# Map our fields
remap_fields(rules, payload)
# There are no rules applied since nothing aligned
assert payload == payload_orig

View File

@ -27,6 +27,7 @@ from django.test.utils import override_settings
from unittest.mock import patch, Mock
from ..forms import NotifyForm
from ..utils import ConfigCache
from json import dumps
import os
import re
import apprise
@ -107,7 +108,7 @@ class StatefulNotifyTests(SimpleTestCase):
assert len(entries) == 3
form_data = {
'body': '## test notifiction',
'body': '## test notification',
'format': apprise.NotifyFormat.MARKDOWN,
'tag': 'general',
}
@ -128,7 +129,40 @@ class StatefulNotifyTests(SimpleTestCase):
mock_post.reset_mock()
form_data = {
'body': '## test notifiction',
'payload': '## test notification',
'fmt': apprise.NotifyFormat.MARKDOWN,
'extra': 'general',
}
# We sent the notification successfully (use our rule mapping)
# FORM
response = self.client.post(
f'/notify/{key}/?:payload=body&:fmt=format&:extra=tag',
form_data)
assert response.status_code == 200
assert mock_post.call_count == 1
mock_post.reset_mock()
form_data = {
'payload': '## test notification',
'fmt': apprise.NotifyFormat.MARKDOWN,
'extra': 'general',
}
# We sent the notification successfully (use our rule mapping)
# JSON
response = self.client.post(
f'/notify/{key}/?:payload=body&:fmt=format&:extra=tag',
dumps(form_data),
content_type="application/json")
assert response.status_code == 200
assert mock_post.call_count == 1
mock_post.reset_mock()
form_data = {
'body': '## test notification',
'format': apprise.NotifyFormat.MARKDOWN,
'tag': 'no-on-with-this-tag',
}
@ -180,7 +214,7 @@ class StatefulNotifyTests(SimpleTestCase):
assert len(entries) == 3
form_data = {
'body': '## test notifiction',
'body': '## test notification',
'format': apprise.NotifyFormat.MARKDOWN,
}
@ -204,7 +238,7 @@ class StatefulNotifyTests(SimpleTestCase):
# Test tagging now
#
form_data = {
'body': '## test notifiction',
'body': '## test notification',
'format': apprise.NotifyFormat.MARKDOWN,
'tag': 'general+json',
}
@ -226,7 +260,7 @@ class StatefulNotifyTests(SimpleTestCase):
mock_post.reset_mock()
form_data = {
'body': '## test notifiction',
'body': '## test notification',
'format': apprise.NotifyFormat.MARKDOWN,
# Plus with space inbetween
'tag': 'general + json',
@ -248,7 +282,7 @@ class StatefulNotifyTests(SimpleTestCase):
mock_post.reset_mock()
form_data = {
'body': '## test notifiction',
'body': '## test notification',
'format': apprise.NotifyFormat.MARKDOWN,
# Space (AND)
'tag': 'general json',
@ -269,7 +303,7 @@ class StatefulNotifyTests(SimpleTestCase):
mock_post.reset_mock()
form_data = {
'body': '## test notifiction',
'body': '## test notification',
'format': apprise.NotifyFormat.MARKDOWN,
# Comma (OR)
'tag': 'general, devops',
@ -351,7 +385,7 @@ class StatefulNotifyTests(SimpleTestCase):
for tag in ('user1', 'user2'):
form_data = {
'body': '## test notifiction',
'body': '## test notification',
'format': apprise.NotifyFormat.MARKDOWN,
'tag': tag,
}
@ -374,7 +408,7 @@ class StatefulNotifyTests(SimpleTestCase):
# Now let's notify by our group
form_data = {
'body': '## test notifiction',
'body': '## test notification',
'format': apprise.NotifyFormat.MARKDOWN,
'tag': 'mygroup',
}
@ -448,7 +482,7 @@ class StatefulNotifyTests(SimpleTestCase):
for tag in ('user1', 'user2'):
form_data = {
'body': '## test notifiction',
'body': '## test notification',
'format': apprise.NotifyFormat.MARKDOWN,
'tag': tag,
}
@ -471,7 +505,7 @@ class StatefulNotifyTests(SimpleTestCase):
# Now let's notify by our group
form_data = {
'body': '## test notifiction',
'body': '## test notification',
'format': apprise.NotifyFormat.MARKDOWN,
'tag': 'mygroup',
}

View File

@ -120,6 +120,39 @@ class StatelessNotifyTests(SimpleTestCase):
# Reset our mock object
mock_notify.reset_mock()
form_data = {
'payload': '## test notification',
'fmt': apprise.NotifyFormat.MARKDOWN,
'extra': 'mailto://user:pass@hotmail.com',
}
# We sent the notification successfully (use our rule mapping)
# FORM
response = self.client.post(
f'/notify/?:payload=body&:fmt=format&:extra=urls',
form_data)
assert response.status_code == 200
assert mock_notify.call_count == 1
mock_notify.reset_mock()
form_data = {
'payload': '## test notification',
'fmt': apprise.NotifyFormat.MARKDOWN,
'extra': 'mailto://user:pass@hotmail.com',
}
# We sent the notification successfully (use our rule mapping)
# JSON
response = self.client.post(
'/notify/?:payload=body&:fmt=format&:extra=urls',
json.dumps(form_data),
content_type="application/json")
assert response.status_code == 200
assert mock_notify.call_count == 1
mock_notify.reset_mock()
# Long Filename
attach_data = {
'attachment': SimpleUploadedFile(

View File

@ -35,6 +35,7 @@ from django.views.decorators.gzip import gzip_page
from django.utils.translation import gettext_lazy as _
from django.core.serializers.json import DjangoJSONEncoder
from .payload_mapper import remap_fields
from .utils import parse_attachments
from .utils import ConfigCache
from .utils import apply_global_filters
@ -662,10 +663,22 @@ class NotifyView(View):
and ACCEPT_ALL.match(request.headers.get('accept', '')) else \
MIME_IS_JSON.match(request.headers.get('accept', '')) is not None
# rules
rules = {k[1:]: v for k,v in request.GET.items() if k[0] == ':'}
# our content
content = {}
if not json_payload:
form = NotifyForm(data=request.POST, files=request.FILES)
if rules:
# Create a copy
data = request.POST.copy()
remap_fields(rules, data)
else:
# Just create a pointer
data = request.POST
form = NotifyForm(data=data, files=request.FILES)
if form.is_valid():
content.update(form.cleaned_data)
@ -675,6 +688,10 @@ class NotifyView(View):
# load our JSON content
content = json.loads(request.body.decode('utf-8'))
# Apply content rules
if rules:
remap_fields(rules, content)
except (RequestDataTooBig):
# DATA_UPLOAD_MAX_MEMORY_SIZE exceeded it's value; this is usually the case
# when there is a very large flie attachment that can't be pulled out of the
@ -1169,11 +1186,22 @@ class StatelessNotifyView(View):
and ACCEPT_ALL.match(request.headers.get('accept', '')) else \
MIME_IS_JSON.match(request.headers.get('accept', '')) is not None
# rules
rules = {k[1:]: v for k,v in request.GET.items() if k[0] == ':'}
# our content
content = {}
if not json_payload:
content = {}
form = NotifyByUrlForm(request.POST, request.FILES)
if rules:
# Create a copy
data = request.POST.copy()
remap_fields(rules, data, form=NotifyByUrlForm())
else:
# Just create a pointer
data = request.POST
form = NotifyByUrlForm(data=data, files=request.FILES)
if form.is_valid():
content.update(form.cleaned_data)
@ -1183,6 +1211,10 @@ class StatelessNotifyView(View):
# load our JSON content
content = json.loads(request.body.decode('utf-8'))
# Apply content rules
if rules:
remap_fields(rules, content, form=NotifyByUrlForm())
except (RequestDataTooBig):
# DATA_UPLOAD_MAX_MEMORY_SIZE exceeded it's value; this is usually the case
# when there is a very large flie attachment that can't be pulled out of the