Support custom field mappings for JSON, FORM and XML Services (#876)

apprise/plugins/NotifyForm.py

@@ -40,6 +40,16 @@ from ..common import NotifyType
 from ..AppriseLocale import gettext_lazy as _
 
 
+class FORMPayloadField:
+    """
+    Identifies the fields available in the FORM Payload
+    """
+    VERSION = 'version'
+    TITLE = 'title'
+    MESSAGE = 'message'
+    MESSAGETYPE = 'type'
+
+
 # Defines the method to send the notification
 METHODS = (
     'POST',
@@ -96,6 +106,12 @@ class NotifyForm(NotifyBase):
     # local anyway
     request_rate_per_sec = 0
 
+    # Define the FORM version to place in all payloads
+    # Version: Major.Minor,  Major is only updated if the entire schema is
+    # changed. If just adding new items (or removing old ones, only increment
+    # the Minor!
+    form_version = '1.0'
+
     # Define object templates
     templates = (
         '{schema}://{host}',
@@ -218,6 +234,18 @@ class NotifyForm(NotifyBase):
                     self.attach_as += self.attach_as_count
                     self.attach_multi_support = True
 
+        # A payload map allows users to over-ride the default mapping if
+        # they're detected with the :overide=value.  Normally this would
+        # create a new key and assign it the value specified.  However
+        # if the key you specify is actually an internally mapped one,
+        # then a re-mapping takes place using the value
+        self.payload_map = {
+            FORMPayloadField.VERSION: FORMPayloadField.VERSION,
+            FORMPayloadField.TITLE: FORMPayloadField.TITLE,
+            FORMPayloadField.MESSAGE: FORMPayloadField.MESSAGE,
+            FORMPayloadField.MESSAGETYPE: FORMPayloadField.MESSAGETYPE,
+        }
+
         self.params = {}
         if params:
             # Store our extra headers
@@ -228,10 +256,20 @@ class NotifyForm(NotifyBase):
             # Store our extra headers
             self.headers.update(headers)
 
+        self.payload_overrides = {}
         self.payload_extras = {}
         if payload:
             # Store our extra payload entries
             self.payload_extras.update(payload)
+            for key in list(self.payload_extras.keys()):
+                # Any values set in the payload to alter a system related one
+                # alters the system key.  Hence :message=msg maps the 'message'
+                # variable that otherwise already contains the payload to be
+                # 'msg' instead (containing the payload)
+                if key in self.payload_map:
+                    self.payload_map[key] = self.payload_extras[key]
+                    self.payload_overrides[key] = self.payload_extras[key]
+                    del self.payload_extras[key]
 
         return
 
@@ -257,6 +295,8 @@ class NotifyForm(NotifyBase):
         # Append our payload extra's into our parameters
         params.update(
             {':{}'.format(k): v for k, v in self.payload_extras.items()})
+        params.update(
+            {':{}'.format(k): v for k, v in self.payload_overrides.items()})
 
         if self.attach_as != self.attach_as_default:
             # Provide Attach-As extension details
@@ -337,15 +377,18 @@ class NotifyForm(NotifyBase):
                     'form:// Multi-Attachment Support not enabled')
 
         # prepare Form Object
-        payload = {
+        payload = {}
-            # Version: Major.Minor,  Major is only updated if the entire
+
-            # schema is changed. If just adding new items (or removing
+        for key, value in (
-            # old ones, only increment the Minor!
+                (FORMPayloadField.VERSION, self.form_version),
-            'version': '1.0',
+                (FORMPayloadField.TITLE, title),
-            'title': title,
+                (FORMPayloadField.MESSAGE, body),
-            'message': body,
+                (FORMPayloadField.MESSAGETYPE, notify_type)):
-            'type': notify_type,
+
-        }
+            if not self.payload_map[key]:
+                # Do not store element in payload response
+                continue
+            payload[self.payload_map[key]] = value
 
         # Apply any/all payload over-rides defined
         payload.update(self.payload_extras)

apprise/plugins/NotifyJSON.py

@@ -41,6 +41,17 @@ from ..common import NotifyType
 from ..AppriseLocale import gettext_lazy as _
 
 
+class JSONPayloadField:
+    """
+    Identifies the fields available in the JSON Payload
+    """
+    VERSION = 'version'
+    TITLE = 'title'
+    MESSAGE = 'message'
+    ATTACHMENTS = 'attachments'
+    MESSAGETYPE = 'type'
+
+
 # Defines the method to send the notification
 METHODS = (
     'POST',
@@ -76,6 +87,12 @@ class NotifyJSON(NotifyBase):
     # local anyway
     request_rate_per_sec = 0
 
+    # Define the JSON version to place in all payloads
+    # Version: Major.Minor,  Major is only updated if the entire schema is
+    # changed. If just adding new items (or removing old ones, only increment
+    # the Minor!
+    json_version = '1.0'
+
     # Define object templates
     templates = (
         '{schema}://{host}',
@@ -162,6 +179,19 @@ class NotifyJSON(NotifyBase):
             self.logger.warning(msg)
             raise TypeError(msg)
 
+        # A payload map allows users to over-ride the default mapping if
+        # they're detected with the :overide=value.  Normally this would
+        # create a new key and assign it the value specified.  However
+        # if the key you specify is actually an internally mapped one,
+        # then a re-mapping takes place using the value
+        self.payload_map = {
+            JSONPayloadField.VERSION: JSONPayloadField.VERSION,
+            JSONPayloadField.TITLE: JSONPayloadField.TITLE,
+            JSONPayloadField.MESSAGE: JSONPayloadField.MESSAGE,
+            JSONPayloadField.ATTACHMENTS: JSONPayloadField.ATTACHMENTS,
+            JSONPayloadField.MESSAGETYPE: JSONPayloadField.MESSAGETYPE,
+        }
+
         self.params = {}
         if params:
             # Store our extra headers
@@ -172,10 +202,21 @@ class NotifyJSON(NotifyBase):
             # Store our extra headers
             self.headers.update(headers)
 
+        self.payload_overrides = {}
         self.payload_extras = {}
         if payload:
             # Store our extra payload entries
             self.payload_extras.update(payload)
+            for key in list(self.payload_extras.keys()):
+                # Any values set in the payload to alter a system related one
+                # alters the system key.  Hence :message=msg maps the 'message'
+                # variable that otherwise already contains the payload to be
+                # 'msg' instead (containing the payload)
+                if key in self.payload_map:
+                    self.payload_map[key] = self.payload_extras[key].strip()
+                    self.payload_overrides[key] = \
+                        self.payload_extras[key].strip()
+                    del self.payload_extras[key]
 
         return
 
@@ -201,6 +242,8 @@ class NotifyJSON(NotifyBase):
         # Append our payload extra's into our parameters
         params.update(
             {':{}'.format(k): v for k, v in self.payload_extras.items()})
+        params.update(
+            {':{}'.format(k): v for k, v in self.payload_overrides.items()})
 
         # Determine Authentication
         auth = ''
@@ -275,16 +318,18 @@ class NotifyJSON(NotifyBase):
                     return False
 
         # prepare JSON Object
-        payload = {
+        payload = {}
-            # Version: Major.Minor,  Major is only updated if the entire
+        for key, value in (
-            # schema is changed. If just adding new items (or removing
+                (JSONPayloadField.VERSION, self.json_version),
-            # old ones, only increment the Minor!
+                (JSONPayloadField.TITLE, title),
-            'version': '1.0',
+                (JSONPayloadField.MESSAGE, body),
-            'title': title,
+                (JSONPayloadField.ATTACHMENTS, attachments),
-            'message': body,
+                (JSONPayloadField.MESSAGETYPE, notify_type)):
-            'attachments': attachments,
+
-            'type': notify_type,
+            if not self.payload_map[key]:
-        }
+                # Do not store element in payload response
+                continue
+            payload[self.payload_map[key]] = value
 
         # Apply any/all payload over-rides defined
         payload.update(self.payload_extras)

apprise/plugins/NotifyXML.py

@@ -41,6 +41,16 @@ from ..common import NotifyType
 from ..AppriseLocale import gettext_lazy as _
 
 
+class XMLPayloadField:
+    """
+    Identifies the fields available in the JSON Payload
+    """
+    VERSION = 'Version'
+    TITLE = 'Subject'
+    MESSAGE = 'Message'
+    MESSAGETYPE = 'MessageType'
+
+
 # Defines the method to send the notification
 METHODS = (
     'POST',
@@ -78,7 +88,8 @@ class NotifyXML(NotifyBase):
 
     # XSD Information
     xsd_ver = '1.1'
-    xsd_url = 'https://raw.githubusercontent.com/caronc/apprise/master' \
+    xsd_default_url = \
+        'https://raw.githubusercontent.com/caronc/apprise/master' \
         '/apprise/assets/NotifyXML-{version}.xsd'
 
     # Define object templates
@@ -161,7 +172,7 @@ class NotifyXML(NotifyBase):
     xmlns:xsd="http://www.w3.org/2001/XMLSchema"
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
     <soapenv:Body>
-        <Notification xmlns:xsi="{{XSD_URL}}">
+        <Notification{{XSD_URL}}>
             {{CORE}}
             {{ATTACHMENTS}}
        </Notification>
@@ -180,6 +191,18 @@ class NotifyXML(NotifyBase):
             self.logger.warning(msg)
             raise TypeError(msg)
 
+        # A payload map allows users to over-ride the default mapping if
+        # they're detected with the :overide=value.  Normally this would
+        # create a new key and assign it the value specified.  However
+        # if the key you specify is actually an internally mapped one,
+        # then a re-mapping takes place using the value
+        self.payload_map = {
+            XMLPayloadField.VERSION: XMLPayloadField.VERSION,
+            XMLPayloadField.TITLE: XMLPayloadField.TITLE,
+            XMLPayloadField.MESSAGE: XMLPayloadField.MESSAGE,
+            XMLPayloadField.MESSAGETYPE: XMLPayloadField.MESSAGETYPE,
+        }
+
         self.params = {}
         if params:
             # Store our extra headers
@@ -190,6 +213,10 @@ class NotifyXML(NotifyBase):
             # Store our extra headers
             self.headers.update(headers)
 
+        # Set our xsd url
+        self.xsd_url = self.xsd_default_url.format(version=self.xsd_ver)
+
+        self.payload_overrides = {}
         self.payload_extras = {}
         if payload:
             # Store our extra payload entries (but tidy them up since they will
@@ -201,8 +228,20 @@ class NotifyXML(NotifyBase):
                         'Ignoring invalid XML Stanza element name({})'
                         .format(k))
                     continue
-                self.payload_extras[key] = v
 
+                # Any values set in the payload to alter a system related one
+                # alters the system key.  Hence :message=msg maps the 'message'
+                # variable that otherwise already contains the payload to be
+                # 'msg' instead (containing the payload)
+                if key in self.payload_map:
+                    self.payload_map[key] = v
+                    self.payload_overrides[key] = v
+
+                    # Over-ride XSD URL as data is no longer known
+                    self.xsd_url = None
+
+                else:
+                    self.payload_extras[key] = v
         return
 
     def url(self, privacy=False, *args, **kwargs):
@@ -227,6 +266,8 @@ class NotifyXML(NotifyBase):
         # Append our payload extra's into our parameters
         params.update(
             {':{}'.format(k): v for k, v in self.payload_extras.items()})
+        params.update(
+            {':{}'.format(k): v for k, v in self.payload_overrides.items()})
 
         # Determine Authentication
         auth = ''
@@ -273,14 +314,21 @@ class NotifyXML(NotifyBase):
         # Our XML Attachmement subsitution
         xml_attachments = ''
 
-        # Our Payload Base
+        payload_base = {}
-        payload_base = {
+
-            'Version': self.xsd_ver,
+        for key, value in (
-            'Subject': NotifyXML.escape_html(title, whitespace=False),
+                (XMLPayloadField.VERSION, self.xsd_ver),
-            'MessageType': NotifyXML.escape_html(
+                (XMLPayloadField.TITLE, NotifyXML.escape_html(
-                notify_type, whitespace=False),
+                    title, whitespace=False)),
-            'Message': NotifyXML.escape_html(body, whitespace=False),
+                (XMLPayloadField.MESSAGE, NotifyXML.escape_html(
-        }
+                    body, whitespace=False)),
+                (XMLPayloadField.MESSAGETYPE, NotifyXML.escape_html(
+                    notify_type, whitespace=False))):
+
+            if not self.payload_map[key]:
+                # Do not store element in payload response
+                continue
+            payload_base[self.payload_map[key]] = value
 
         # Apply our payload extras
         payload_base.update(
@@ -328,7 +376,8 @@ class NotifyXML(NotifyBase):
                 ''.join(attachments) + '</Attachments>'
 
         re_map = {
-            '{{XSD_URL}}': self.xsd_url.format(version=self.xsd_ver),
+            '{{XSD_URL}}':
+            f' xmlns:xsi="{self.xsd_url}"' if self.xsd_url else '',
             '{{ATTACHMENTS}}': xml_attachments,
             '{{CORE}}': xml_base,
         }

test/test_plugin_custom_form.py

@@ -318,7 +318,7 @@ def test_plugin_custom_form_edge_cases(mock_get, mock_post):
     mock_get.return_value = response
 
     results = NotifyForm.parse_url(
-        'form://localhost:8080/command?:abcd=test&method=POST')
+        'form://localhost:8080/command?:message=msg&:abcd=test&method=POST')
 
     assert isinstance(results, dict)
     assert results['user'] is None
@@ -332,6 +332,7 @@ def test_plugin_custom_form_edge_cases(mock_get, mock_post):
     assert results['url'] == 'form://localhost:8080/command'
     assert isinstance(results['qsd:'], dict) is True
     assert results['qsd:']['abcd'] == 'test'
+    assert results['qsd:']['message'] == 'msg'
 
     instance = NotifyForm(**results)
     assert isinstance(instance, NotifyForm)
@@ -347,8 +348,11 @@ def test_plugin_custom_form_edge_cases(mock_get, mock_post):
     assert details[1]['data']['abcd'] == 'test'
     assert 'title' in details[1]['data']
     assert details[1]['data']['title'] == 'title'
-    assert 'message' in details[1]['data']
+    assert 'message' not in details[1]['data']
-    assert details[1]['data']['message'] == 'body'
+    # message over-ride was provided; the body is now in `msg` and not
+    # `message`
+    assert 'msg' in details[1]['data']
+    assert details[1]['data']['msg'] == 'body'
 
     assert instance.url(privacy=False).startswith(
         'form://localhost:8080/command?')
@@ -364,7 +368,7 @@ def test_plugin_custom_form_edge_cases(mock_get, mock_post):
     mock_get.reset_mock()
 
     results = NotifyForm.parse_url(
-        'form://localhost:8080/command?:message=test&method=POST')
+        'form://localhost:8080/command?:type=&:message=msg&method=POST')
 
     assert isinstance(results, dict)
     assert results['user'] is None
@@ -377,7 +381,7 @@ def test_plugin_custom_form_edge_cases(mock_get, mock_post):
     assert results['schema'] == 'form'
     assert results['url'] == 'form://localhost:8080/command'
     assert isinstance(results['qsd:'], dict) is True
-    assert results['qsd:']['message'] == 'test'
+    assert results['qsd:']['message'] == 'msg'
 
     instance = NotifyForm(**results)
     assert isinstance(instance, NotifyForm)
@@ -391,9 +395,18 @@ def test_plugin_custom_form_edge_cases(mock_get, mock_post):
     assert details[0][0] == 'http://localhost:8080/command'
     assert 'title' in details[1]['data']
     assert details[1]['data']['title'] == 'title'
+
+    # type was removed from response object
+    assert 'type' not in details[1]['data']
+
+    # message over-ride was provided; the body is now in `msg` and not
+    # `message`
+    assert details[1]['data']['msg'] == 'body'
+
     # 'body' is over-ridden by 'test' passed inline with the URL
-    assert 'message' in details[1]['data']
+    assert 'message' not in details[1]['data']
-    assert details[1]['data']['message'] == 'test'
+    assert 'msg' in details[1]['data']
+    assert details[1]['data']['msg'] == 'body'
 
     assert instance.url(privacy=False).startswith(
         'form://localhost:8080/command?')
@@ -438,8 +451,9 @@ def test_plugin_custom_form_edge_cases(mock_get, mock_post):
     assert 'title' in details[1]['params']
     assert details[1]['params']['title'] == 'title'
     # 'body' is over-ridden by 'test' passed inline with the URL
-    assert 'message' in details[1]['params']
+    assert 'message' not in details[1]['params']
-    assert details[1]['params']['message'] == 'test'
+    assert 'test' in details[1]['params']
+    assert details[1]['params']['test'] == 'body'
 
     assert instance.url(privacy=False).startswith(
         'form://localhost:8080/command?')

test/test_plugin_custom_json.py

@@ -176,8 +176,11 @@ def test_plugin_custom_json_edge_cases(mock_get, mock_post):
     mock_post.return_value = response
     mock_get.return_value = response
 
+    # This string also tests that type is set to nothing
     results = NotifyJSON.parse_url(
-        'json://localhost:8080/command?:message=test&method=GET')
+        'json://localhost:8080/command?'
+        ':message=msg&:test=value&method=GET'
+        '&:type=')
 
     assert isinstance(results, dict)
     assert results['user'] is None
@@ -190,7 +193,9 @@ def test_plugin_custom_json_edge_cases(mock_get, mock_post):
     assert results['schema'] == 'json'
     assert results['url'] == 'json://localhost:8080/command'
     assert isinstance(results['qsd:'], dict) is True
-    assert results['qsd:']['message'] == 'test'
+    assert results['qsd:']['message'] == 'msg'
+    # empty special mapping
+    assert results['qsd:']['type'] == ''
 
     instance = NotifyJSON(**results)
     assert isinstance(instance, NotifyJSON)
@@ -205,9 +210,16 @@ def test_plugin_custom_json_edge_cases(mock_get, mock_post):
     assert 'title' in details[1]['data']
     dataset = json.loads(details[1]['data'])
     assert dataset['title'] == 'title'
-    assert 'message' in dataset
+    assert 'message' not in dataset
-    # message over-ride was provided
+    assert 'msg' in dataset
-    assert dataset['message'] == 'test'
+    # type was set to nothing which implies it should be removed
+    assert 'type' not in dataset
+    # message over-ride was provided; the body is now in `msg` and not
+    # `message`
+    assert dataset['msg'] == 'body'
+
+    assert 'test' in dataset
+    assert dataset['test'] == 'value'
 
     assert instance.url(privacy=False).startswith(
         'json://localhost:8080/command?')

test/test_plugin_custom_xml.py

@@ -251,8 +251,8 @@ def test_plugin_custom_xml_edge_cases(mock_get, mock_post):
     mock_get.return_value = response
 
     results = NotifyXML.parse_url(
-        'xml://localhost:8080/command?:Message=test&method=GET'
+        'xml://localhost:8080/command?:Message=Body&method=GET'
-        '&:Key=value&:,=invalid')
+        '&:Key=value&:,=invalid&:MessageType=')
 
     assert isinstance(results, dict)
     assert results['user'] is None
@@ -265,13 +265,16 @@ def test_plugin_custom_xml_edge_cases(mock_get, mock_post):
     assert results['schema'] == 'xml'
     assert results['url'] == 'xml://localhost:8080/command'
     assert isinstance(results['qsd:'], dict) is True
-    assert results['qsd:']['Message'] == 'test'
+    assert results['qsd:']['Message'] == 'Body'
     assert results['qsd:']['Key'] == 'value'
     assert results['qsd:'][','] == 'invalid'
 
     instance = NotifyXML(**results)
     assert isinstance(instance, NotifyXML)
 
+    # XSD URL is disabled due to custom formatting
+    assert instance.xsd_url is None
+
     response = instance.send(title='title', body='body')
     assert response is True
     assert mock_post.call_count == 0
@@ -290,9 +293,63 @@ def test_plugin_custom_xml_edge_cases(mock_get, mock_post):
 
     # Test our data set for our key/value pair
     assert re.search(r'<Version>[1-9]+\.[0-9]+</Version>', details[1]['data'])
-    assert re.search('<MessageType>info</MessageType>', details[1]['data'])
     assert re.search('<Subject>title</Subject>', details[1]['data'])
-    # Custom entry Message acts as Over-ride and kicks in here
+
-    assert re.search('<Message>test</Message>', details[1]['data'])
+    assert re.search('<Message>test</Message>', details[1]['data']) is None
+    assert re.search('<Message>', details[1]['data']) is None
+    # MessageType was removed from the payload
+    assert re.search('<MessageType>', details[1]['data']) is None
+    # However we can find our mapped Message to the new value Body
+    assert re.search('<Body>body</Body>', details[1]['data'])
     # Custom entry
     assert re.search('<Key>value</Key>', details[1]['data'])
+
+    mock_post.reset_mock()
+    mock_get.reset_mock()
+
+    results = NotifyXML.parse_url(
+        'xml://localhost:8081/command?method=POST&:New=Value')
+
+    assert isinstance(results, dict)
+    assert results['user'] is None
+    assert results['password'] is None
+    assert results['port'] == 8081
+    assert results['host'] == 'localhost'
+    assert results['fullpath'] == '/command'
+    assert results['path'] == '/'
+    assert results['query'] == 'command'
+    assert results['schema'] == 'xml'
+    assert results['url'] == 'xml://localhost:8081/command'
+    assert isinstance(results['qsd:'], dict) is True
+    assert results['qsd:']['New'] == 'Value'
+
+    instance = NotifyXML(**results)
+    assert isinstance(instance, NotifyXML)
+
+    # XSD URL is disabled due to custom formatting
+    assert instance.xsd_url is not None
+
+    response = instance.send(title='title', body='body')
+    assert response is True
+    assert mock_post.call_count == 1
+    assert mock_get.call_count == 0
+
+    details = mock_post.call_args_list[0]
+    assert details[0][0] == 'http://localhost:8081/command'
+    assert instance.url(privacy=False).startswith(
+        'xml://localhost:8081/command?')
+
+    # Generate a new URL based on our last and verify key values are the same
+    new_results = NotifyXML.parse_url(instance.url(safe=False))
+    for k in ('user', 'password', 'port', 'host', 'fullpath', 'path', 'query',
+              'schema', 'url', 'method'):
+        assert new_results[k] == results[k]
+
+    # Test our data set for our key/value pair
+    assert re.search(r'<Version>[1-9]+\.[0-9]+</Version>', details[1]['data'])
+    assert re.search(r'<MessageType>info</MessageType>', details[1]['data'])
+    assert re.search(r'<Subject>title</Subject>', details[1]['data'])
+    # No over-ride
+    assert re.search(r'<Message>body</Message>', details[1]['data'])
+    # since there is no over-ride, an xmlns:xsi is provided
+    assert re.search(r'<Notification xmlns:xsi=', details[1]['data'])