From 7144a37bcd9c69126f5c407b6d238ac88b6fedbe Mon Sep 17 00:00:00 2001 From: Chris Caron Date: Sat, 20 Apr 2024 18:48:34 -0400 Subject: [PATCH] Logging improvements and code tidying --- README.md | 2 + apprise_api/api/tests/test_add.py | 14 + apprise_api/api/tests/test_notify.py | 70 ++- .../api/tests/test_stateless_notify.py | 17 + apprise_api/api/views.py | 553 ++++++++++++------ apprise_api/core/settings/__init__.py | 5 + setup.cfg | 1 + 7 files changed, 493 insertions(+), 169 deletions(-) diff --git a/README.md b/README.md index f5b39e7..1fd2138 100644 --- a/README.md +++ b/README.md @@ -336,6 +336,7 @@ curl -X POST -d '{"tag":"leaders teamA, leaders teamB", "body":"meeting now"}' \ | 400 | bad request | Your API call did not conform to what was documented here | 405 | method not accepted | Your API call identified an action that has been disabled due to the Server configuration (such as a `apprise://` `APPRISE_RECURSION_MAX` being exceeded). | 424 | failed dependency | At least one notification could not be sent. This can be due to
- Not all notifications intended to be actioned could follow through (due to upstrem failures).
You didn't idenify a tag associated with what was defined in your configuration.
The tag(s) you specified do not match with those defined in your configuration. +| 431 | fields too large | This can happen if you're payload is larger then 3MB (default value). See `APPRISE_UPLOAD_MAX_MEMORY_SIZE` to adjust this. | 500 | internal server error | This can occur if there was an issue saving your configuration to disk (usually the cause of permission issues). @@ -358,6 +359,7 @@ The use of environment variables allow you to provide over-rides to default sett | `APPRISE_CONFIG_DIR` | Defines an (optional) persistent store location of all configuration files saved. By default:
- Configuration is written to the `apprise_api/var/config` directory when just using the _Django_ `manage runserver` script. However for the path for the container is `/config`. | `APPRISE_ATTACH_DIR` | The directory the uploaded attachments are placed in. By default:
- Attachments are written to the `apprise_api/var/attach` directory when just using the _Django_ `manage runserver` script. However for the path for the container is `/attach`. | `APPRISE_ATTACH_SIZE` | Over-ride the attachment size (defined in MB). By default it is set to `200` (Megabytes). You can set this up to a maximum value of `500` which is the restriction in place for NginX (internal hosting ervice) at this time. If you set this to zero (`0`) then attachments will not be passed along even if provided. +| `APPRISE_UPLOAD_MAX_MEMORY_SIZE` | Over-ride the in-memory accepted payload size (defined in MB). By default it is set to `3` (Megabytes). There is no reason the HTTP payload (excluding attachments) should exceed this limit. This value is only configurable for those who have edge cases where there are exceptions to this rule. | `APPRISE_STATELESS_URLS` | For a non-persistent solution, you can take advantage of this global variable. Use this to define a default set of Apprise URLs to notify when using API calls to `/notify`. If no `{KEY}` is defined when calling `/notify` then the URLs defined here are used instead. By default, nothing is defined for this variable. | `APPRISE_STATEFUL_MODE` | This can be set to the following possible modes:
📌 **hash**: This is also the default. It stores the server configuration in a hash formatted that can be easily indexed and compressed.
📌 **simple**: Configuration is written straight to disk using the `{KEY}.cfg` (if `TEXT` based) and `{KEY}.yml` (if `YAML` based).
📌 **disabled**: Straight up deny any read/write queries to the servers stateful store. Effectively turn off the Apprise Stateful feature completely. | `APPRISE_CONFIG_LOCK` | Locks down your API hosting so that you can no longer delete/update/access stateful information. Your configuration is still referenced when stateful calls are made to `/notify`. The idea of this switch is to allow someone to set their (Apprise) configuration up and then as an added security tactic, they may choose to lock their configuration down (in a read-only state). Those who use the Apprise CLI tool may still do it, however the `--config` (`-c`) switch will not successfully reference this access point anymore. You can however use the `apprise://` plugin without any problem ([see here for more details](https://github.com/caronc/apprise/wiki/Notify_apprise_api)). This defaults to `no` and can however be set to `yes` by simply defining the global variable as such. diff --git a/apprise_api/api/tests/test_add.py b/apprise_api/api/tests/test_add.py index 0b5f4a4..ef16e7f 100644 --- a/apprise_api/api/tests/test_add.py +++ b/apprise_api/api/tests/test_add.py @@ -23,8 +23,10 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. from django.test import SimpleTestCase +from django.core.exceptions import RequestDataTooBig from apprise import ConfigFormat from unittest.mock import patch +from unittest import mock from django.test.utils import override_settings from ..forms import AUTO_DETECT_CONFIG_KEYWORD import json @@ -139,6 +141,18 @@ class AddTests(SimpleTestCase): ) assert response.status_code == 200 + with mock.patch('json.loads') as mock_loads: + mock_loads.side_effect = RequestDataTooBig() + # Send our notification by specifying the tag in the parameters + response = self.client.post( + '/add/{}'.format(key), + data=json.dumps({'urls': 'mailto://user:pass@yahoo.ca'}), + content_type='application/json', + ) + + # Our notification failed + assert response.status_code == 431 + # Test with JSON (and no payload provided) response = self.client.post( '/add/{}'.format(key), diff --git a/apprise_api/api/tests/test_notify.py b/apprise_api/api/tests/test_notify.py index 949e504..0d8e349 100644 --- a/apprise_api/api/tests/test_notify.py +++ b/apprise_api/api/tests/test_notify.py @@ -24,6 +24,7 @@ # THE SOFTWARE. from django.test import SimpleTestCase, override_settings from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.exceptions import RequestDataTooBig from unittest import mock import requests from ..forms import NotifyForm @@ -577,6 +578,22 @@ class NotifyTests(SimpleTestCase): assert response['message'] == form_data['body'] assert response['type'] == apprise.NotifyType.WARNING + # Test case where RequestDataTooBig thrown + # Reset our mock object + mock_post.reset_mock() + + with mock.patch('json.loads') as mock_loads: + mock_loads.side_effect = RequestDataTooBig() + # Send our notification by specifying the tag in the parameters + response = self.client.post( + f'/notify/{key}?tag=home&body=test', + form_data, + content_type='application/json') + + # Our notification failed + assert response.status_code == 431 + assert mock_post.call_count == 0 + @mock.patch('requests.post') def test_advanced_notify_with_tags(self, mock_post): """ @@ -883,7 +900,7 @@ class NotifyTests(SimpleTestCase): # Preare our JSON data json_data = { - 'body': 'test notifiction', + 'body': 'test notification', 'type': apprise.NotifyType.WARNING, } @@ -1059,6 +1076,7 @@ class NotifyTests(SimpleTestCase): headers = { 'HTTP_X_APPRISE_LOG_LEVEL': 'debug', + # Accept is over-ridden to be that of the content type 'HTTP_ACCEPT': 'text/plain', } @@ -1074,13 +1092,14 @@ class NotifyTests(SimpleTestCase): assert mock_notify.call_count == 1 assert response['content-type'] == 'text/plain' + mock_notify.reset_mock() + headers = { 'HTTP_X_APPRISE_LOG_LEVEL': 'debug', + # Accept is over-ridden to be that of the content type 'HTTP_ACCEPT': 'text/html', } - mock_notify.reset_mock() - # Test referencing a key that doesn't exist response = self.client.post( '/notify/{}'.format(key), @@ -1093,6 +1112,38 @@ class NotifyTests(SimpleTestCase): assert mock_notify.call_count == 1 assert response['content-type'] == 'text/html' + mock_notify.reset_mock() + + # Test referencing a key that doesn't exist + response = self.client.post( + '/notify/{}'.format(key), + data={'body': 'test'}, + **headers, + ) + + assert response.status_code == 200 + assert mock_notify.call_count == 1 + assert response['content-type'].startswith('text/html') + + mock_notify.reset_mock() + + headers = { + 'HTTP_X_APPRISE_LOG_LEVEL': 'debug', + 'HTTP_ACCEPT': '*/*', + } + + # Test referencing a key that doesn't exist + response = self.client.post( + '/notify/{}'.format(key), + data=json.dumps(json_data), + content_type='application/json', + **headers, + ) + + assert response.status_code == 200 + assert mock_notify.call_count == 1 + assert response['content-type'] == 'application/json' + headers = { 'HTTP_X_APPRISE_LOG_LEVEL': 'invalid', 'HTTP_ACCEPT': 'text/*', @@ -1112,6 +1163,19 @@ class NotifyTests(SimpleTestCase): assert mock_notify.call_count == 1 assert response['content-type'] == 'text/html' + mock_notify.reset_mock() + + # Test referencing a key that doesn't exist + response = self.client.post( + '/notify/{}'.format(key), + data=json_data, + **headers, + ) + + assert response.status_code == 200 + assert mock_notify.call_count == 1 + assert response['content-type'].startswith('text/html') + @mock.patch('apprise.plugins.NotifyEmail.NotifyEmail.send') def test_notify_with_filters(self, mock_send): """ diff --git a/apprise_api/api/tests/test_stateless_notify.py b/apprise_api/api/tests/test_stateless_notify.py index 2a32d66..6ee0a98 100644 --- a/apprise_api/api/tests/test_stateless_notify.py +++ b/apprise_api/api/tests/test_stateless_notify.py @@ -24,6 +24,7 @@ # THE SOFTWARE. from django.test import SimpleTestCase from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.exceptions import RequestDataTooBig from django.test.utils import override_settings from unittest import mock from ..forms import NotifyByUrlForm @@ -442,6 +443,22 @@ class StatelessNotifyTests(SimpleTestCase): assert response.status_code == 200 assert mock_notify.call_count == 1 + # Reset our count + mock_notify.reset_mock() + + with mock.patch('json.loads') as mock_loads: + mock_loads.side_effect = RequestDataTooBig() + # Send our notification + response = self.client.post( + '/notify/?title=my%20title&format=text&type=info', + data=json.dumps(json_data), + content_type='application/json', + ) + + # Our notification failed + assert response.status_code == 431 + assert mock_notify.call_count == 0 + @mock.patch('apprise.Apprise.notify') def test_notify_by_loaded_urls_with_json(self, mock_notify): """ diff --git a/apprise_api/api/views.py b/apprise_api/api/views.py index c122589..281e68c 100644 --- a/apprise_api/api/views.py +++ b/apprise_api/api/views.py @@ -27,6 +27,7 @@ from django.http import HttpResponse from django.http import JsonResponse from django.views import View from django.conf import settings +from django.core.exceptions import RequestDataTooBig from django.utils.html import escape from django.utils.decorators import method_decorator from django.views.decorators.cache import never_cache @@ -55,7 +56,6 @@ logger = logging.getLogger('django') # Content-Type Parsing # application/x-www-form-urlencoded -# application/x-www-form-urlencoded # multipart/form-data MIME_IS_FORM = re.compile( r'(multipart|application)/(x-www-)?form-(data|urlencoded)', re.I) @@ -68,6 +68,11 @@ MIME_IS_FORM = re.compile( MIME_IS_JSON = re.compile( r'(text|application)/(x-)?json', re.I) +# Parsing of Accept; the following amounts to Accept All +# */* +# +ACCEPT_ALL = re.compile(r'^\s*([*]/[*]|)\s*$', re.I) + # Tags separated by space , &, or + are and'ed together # Tags separated by commas (even commas wrapped in spaces) are "or'ed" together # We start with a regular expression used to clean up provided tags @@ -111,6 +116,7 @@ class ResponseCode(object): method_not_allowed = 405 method_not_accepted = 406 failed_dependency = 424 + fields_too_large = 431 internal_server_error = 500 @@ -215,20 +221,27 @@ class AddView(View): """ Handle a POST request """ - # Detect the format our response should be in - json_response = \ + # Detect the format our incoming payload + json_payload = \ MIME_IS_JSON.match( request.content_type if request.content_type else request.headers.get( - 'accept', request.headers.get( - 'content-type', ''))) is not None + 'content-type', '')) is not None + + # Detect the format our response should be in + json_response = True if json_payload \ + and ACCEPT_ALL.match(request.headers.get('accept', '')) else \ + MIME_IS_JSON.match(request.headers.get('accept', '')) is not None if settings.APPRISE_CONFIG_LOCK: # General Access Control - msg = _('The site has been configured to deny this request.') + logger.warning( + 'ADD - %s - Config Lock Active - Request Denied', + request.META['REMOTE_ADDR']) + msg = _('The site has been configured to deny this request') status = ResponseCode.no_access - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -239,7 +252,7 @@ class AddView(View): # our content content = {} - if MIME_IS_FORM.match(request.content_type): + if not json_payload: content = {} form = AddByConfigForm(request.POST) if form.is_valid(): @@ -249,27 +262,49 @@ class AddView(View): if form.is_valid(): content.update(form.cleaned_data) - elif json_response: + else: # JSON Payload # Prepare our default response try: # load our JSON content content = json.loads(request.body.decode('utf-8')) + except (RequestDataTooBig): + # DATA_UPLOAD_MAX_MEMORY_SIZE exceeded it's value; this is usually the case + # when there is a very large flie attachment that can't be pulled out of the + # payload without exceeding memory limitations (default is 3MB) + logger.warning( + 'ADD - %s - JSON Payload Exceeded %dMB; operaton aborted using KEY: %s', + request.META['REMOTE_ADDR'], (settings.DATA_UPLOAD_MAX_MEMORY_SIZE / 1048576), key) + + status = ResponseCode.fields_too_large + msg = _('JSON Payload provided is to large') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) + except (AttributeError, ValueError): # could not parse JSON response... - return JsonResponse({ - 'error': _('Invalid JSON specified.'), - }, - encoder=JSONEncoder, - safe=False, - status=ResponseCode.bad_request, - ) + logger.warning( + 'ADD - %s - Invalid JSON Payload provided using KEY: %s', + request.META['REMOTE_ADDR'], key) + + status = ResponseCode.bad_request + msg = _('Invalid JSON Payload provided') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) if not content: # No information was posted - msg = _('The message format is not supported.') + logger.warning( + 'ADD - %s - Invalid payload structure provided using KEY: %s', + request.META['REMOTE_ADDR'], key) + + msg = _('Invalid payload structure provided') status = ResponseCode.bad_request - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -285,9 +320,13 @@ class AddView(View): a_obj.add(content['urls']) if not len(a_obj): # No URLs were loaded - msg = _('No valid URLs were found.') + logger.warning( + 'ADD - %s - No valid URLs defined using KEY: %s', + request.META['REMOTE_ADDR'], key) + + msg = _('No valid URLs defined') status = ResponseCode.bad_request - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -300,9 +339,12 @@ class AddView(View): key, '\r\n'.join([s.url() for s in a_obj]), apprise.ConfigFormat.TEXT): - msg = _('The configuration could not be saved.') + logger.warning( + 'ADD - %s - configuration could not be saved using KEY: %s', + request.META['REMOTE_ADDR'], key) + msg = _('The configuration could not be saved') status = ResponseCode.internal_server_error - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -315,9 +357,12 @@ class AddView(View): fmt = content.get('format', '').lower() if fmt not in [i[0] for i in CONFIG_FORMATS]: # Format must be one supported by apprise - msg = _('The format specified is invalid.') + logger.warning( + 'ADD - %s - Invalid configuration format specified (%s) using KEY: %s', + request.META['REMOTE_ADDR'], fmt, key) + msg = _('Invalid configuration format specified') status = ResponseCode.bad_request - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -337,9 +382,12 @@ class AddView(View): # Load our configuration if not ac_obj.add_config(content['config'], format=fmt): # The format could not be detected - msg = _('The configuration format could not be detected.') + logger.warning( + 'ADD - %s - The configuration format could not be auto-detected using KEY: %s', + request.META['REMOTE_ADDR'], key) + msg = _('The configuration format could not be auto-detected') status = ResponseCode.bad_request - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -354,9 +402,12 @@ class AddView(View): if not len(a_obj): # No specified URL(s) were loaded due to # mis-configuration on the caller's part - msg = _('No valid URL(s) were specified.') + logger.warning( + 'ADD - %s - No valid URL(s) defined using KEY: %s', + request.META['REMOTE_ADDR'], key) + msg = _('No valid URL(s) defined') status = ResponseCode.bad_request - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -368,9 +419,12 @@ class AddView(View): if not ConfigCache.put( key, content['config'], fmt=ac_obj[0].config_format): # Something went very wrong; return 500 - msg = _('An error occured saving configuration.') + logger.error( + 'ADD - %s - Configuration could not be saved using KEY: %s', + request.META['REMOTE_ADDR'], key) + msg = _('An error occured saving configuration') status = ResponseCode.internal_server_error - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -381,9 +435,12 @@ class AddView(View): else: # No configuration specified; we're done - msg = _('No configuration specified.') + msg = _('No configuration provided') + logger.warning( + 'ADD - %s - No configuration provided using KEY: %s', + request.META['REMOTE_ADDR'], key) status = ResponseCode.bad_request - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -394,10 +451,16 @@ class AddView(View): # If we reach here; we successfully loaded the configuration so we can # go ahead and write it to disk and alert our caller of the success. - return HttpResponse( - _('Successfully saved configuration.'), - status=ResponseCode.okay, - ) + logger.info( + 'ADD - %s - Configuration saved using KEY: %s', + request.META['REMOTE_ADDR'], key) + + status = ResponseCode.okay + msg = _('Successfully saved configuration') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + "error": None, + }, encoder=JSONEncoder, safe=False, status=status) @method_decorator(never_cache, name='dispatch') @@ -420,9 +483,12 @@ class DelView(View): if settings.APPRISE_CONFIG_LOCK: # General Access Control - msg = _('The site has been configured to deny this request.') + logger.warning( + 'DEL - %s - Config Lock Active - Request Denied', + request.META['REMOTE_ADDR']) + msg = _('The site has been configured to deny this request') status = ResponseCode.no_access - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -434,9 +500,12 @@ class DelView(View): # Clear the key result = ConfigCache.clear(key) if result is None: - msg = _('There was no configuration to remove.') + logger.warning( + 'DEL - %s - No configuration associated using KEY: %s', + request.META['REMOTE_ADDR'], key) + msg = _('There was no configuration to remove') status = ResponseCode.no_content - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -447,9 +516,13 @@ class DelView(View): elif result is False: # There was a failure at the os level - msg = _('The configuration could not be removed.') + logger.error( + 'DEL - %s - Configuration could not be removed associated using KEY: %s', + request.META['REMOTE_ADDR'], key) + + msg = _('The configuration could not be removed') status = ResponseCode.internal_server_error - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -459,10 +532,16 @@ class DelView(View): ) # Removed content - return HttpResponse( - _('Successfully removed configuration.'), - status=ResponseCode.okay, - ) + logger.info( + 'DEL - %s - Removed configuration associated using KEY: %s', + request.META['REMOTE_ADDR'], key) + + status = ResponseCode.okay + msg = _('Successfully removed configuration') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + "error": None, + }, encoder=JSONEncoder, safe=False, status=status) @method_decorator((gzip_page, never_cache), name='dispatch') @@ -486,9 +565,13 @@ class GetView(View): if settings.APPRISE_CONFIG_LOCK: # General Access Control - msg = _('The site has been configured to deny this request.') + logger.warning( + 'VIEW - %s - Config Lock Active - Request Denied', + request.META['REMOTE_ADDR']) + + msg = _('The site has been configured to deny this request') status = ResponseCode.no_access - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse( {'error': msg}, encoder=JSONEncoder, @@ -505,9 +588,12 @@ class GetView(View): # config != None: we simply have no data if format is not None: # no content to return - msg = _('There was no configuration found.') + logger.warning( + 'VIEW - %s - No configuration associated using KEY: %s', + request.META['REMOTE_ADDR'], key) + msg = _('There was no configuration found') status = ResponseCode.no_content - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse( {'error': msg}, encoder=JSONEncoder, @@ -515,9 +601,12 @@ class GetView(View): status=status) # Something went very wrong; return 500 - msg = _('An error occured accessing configuration.') + logger.error( + 'VIEW - %s - Configuration could not be accessed associated using KEY: %s', + request.META['REMOTE_ADDR'], key) + msg = _('An error occured accessing configuration') status = ResponseCode.internal_server_error - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -532,9 +621,12 @@ class GetView(View): # reference to it through the --config (-c) option in the CLI. content_type = 'text/yaml; charset=utf-8' \ if format == apprise.ConfigFormat.YAML \ - else 'text/html; charset=utf-8' + else 'text/plain; charset=utf-8' # Return our retrieved content + logger.info( + 'VIEW - %s - Retrieved configuration associated using KEY: %s', + request.META['REMOTE_ADDR'], key) return HttpResponse( config, content_type=content_type, @@ -557,49 +649,69 @@ class NotifyView(View): """ Handle a POST request """ - # Detect the format our response should be in - json_response = \ + # Detect the format our incoming payload + json_payload = \ MIME_IS_JSON.match( request.content_type if request.content_type else request.headers.get( - 'accept', request.headers.get( - 'content-type', ''))) is not None + 'content-type', '')) is not None + + # Detect the format our response should be in + json_response = True if json_payload \ + and ACCEPT_ALL.match(request.headers.get('accept', '')) else \ + MIME_IS_JSON.match(request.headers.get('accept', '')) is not None # our content content = {} - if MIME_IS_FORM.match(request.content_type): + if not json_payload: form = NotifyForm(data=request.POST, files=request.FILES) if form.is_valid(): content.update(form.cleaned_data) - elif json_response: + else: # JSON Payload # Prepare our default response try: # load our JSON content content = json.loads(request.body.decode('utf-8')) + except (RequestDataTooBig): + # DATA_UPLOAD_MAX_MEMORY_SIZE exceeded it's value; this is usually the case + # when there is a very large flie attachment that can't be pulled out of the + # payload without exceeding memory limitations (default is 3MB) + logger.warning( + 'NOTIFY - %s - JSON Payload Exceeded %dMB using KEY: %s', + request.META['REMOTE_ADDR'], (settings.DATA_UPLOAD_MAX_MEMORY_SIZE / 1048576), key) + + status = ResponseCode.fields_too_large + msg = _('JSON Payload provided is to large') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) + except (AttributeError, ValueError): # could not parse JSON response... logger.warning( - 'NOTIFY - %s - Invalid JSON Payload provided', - request.META['REMOTE_ADDR']) + 'NOTIFY - %s - Invalid JSON Payload provided using KEY: %s', + request.META['REMOTE_ADDR'], key) - return JsonResponse( - _('Invalid JSON provided.'), - encoder=JSONEncoder, - safe=False, - status=ResponseCode.bad_request) + status = ResponseCode.bad_request + msg = _('Invalid JSON Payload provided') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) if not content: # We could not handle the Content-Type logger.warning( - 'NOTIFY - %s - Invalid FORM Payload provided', - request.META['REMOTE_ADDR']) + 'NOTIFY - %s - Invalid FORM Payload provided using KEY: %s', + request.META['REMOTE_ADDR'], key) - msg = _('Bad FORM Payload provided.') + msg = _('Bad FORM Payload provided') status = ResponseCode.bad_request - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -632,12 +744,15 @@ class NotifyView(View): except (TypeError, ValueError) as e: # Invalid entry found in list logger.warning( - 'NOTIFY - %s - Bad attachment: %s', - request.META['REMOTE_ADDR'], str(e)) + 'NOTIFY - %s - Bad attachment using KEY: %s - %s', + request.META['REMOTE_ADDR'], key, str(e)) - return HttpResponse( - _('Bad attachment'), - status=ResponseCode.bad_request) + status = ResponseCode.bad_request + msg = _('Bad Attachment') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) # # Allow 'tag' value to be specified as part of the URL parameters @@ -670,12 +785,12 @@ class NotifyView(View): # Invalid entry found in list logger.warning( 'NOTIFY - %s - Ignored invalid tag specified ' - '(type %s): %s', request.META['REMOTE_ADDR'], - str(type(tag)), str(tag)[:12]) + '(type %s): %s using KEY: %s', request.META['REMOTE_ADDR'], + str(type(tag)), str(tag)[:12], key) - msg = _('Unsupported characters found in tag definition.') + msg = _('Unsupported characters found in tag definition') status = ResponseCode.bad_request - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -704,12 +819,12 @@ class NotifyView(View): else: # Could be int, float or some other unsupported type logger.warning( 'NOTIFY - %s - Ignored invalid tag specified (type %s): ' - '%s', request.META['REMOTE_ADDR'], - str(type(tag)), str(tag)[:12]) + '%s using KEY: %s', request.META['REMOTE_ADDR'], + str(type(tag)), str(tag)[:12], key) - msg = _('Unsupported characters found in tag definition.') + msg = _('Unsupported characters found in tag definition') status = ResponseCode.bad_request - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -744,12 +859,14 @@ class NotifyView(View): not in apprise.NOTIFY_TYPES: logger.warning( - 'NOTIFY - %s - Payload lacks minimum requirements', - request.META['REMOTE_ADDR']) + 'NOTIFY - %s - Payload lacks minimum requirements using KEY: %s', + request.META['REMOTE_ADDR'], key) - return HttpResponse(msg, status=status) \ + status = ResponseCode.bad_request + msg = _('Payload lacks minimum requirements') + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ - 'error': _('Payload lacks minimum requirements.'), + 'error': msg, }, encoder=JSONEncoder, safe=False, @@ -761,11 +878,11 @@ class NotifyView(View): if body_format and body_format not in apprise.NOTIFY_FORMATS: logger.warning( 'NOTIFY - %s - Format parameter contains an unsupported ' - 'value (%s)', request.META['REMOTE_ADDR'], str(body_format)) + 'value (%s) using KEY: %s', request.META['REMOTE_ADDR'], str(body_format), key) - msg = _('An invalid body input format was specified.') + msg = _('An invalid body input format was specified') status = ResponseCode.bad_request - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -789,9 +906,10 @@ class NotifyView(View): logger.debug( 'NOTIFY - %s - Empty configuration found using KEY: %s', request.META['REMOTE_ADDR'], key) - msg = _('There was no configuration found.') + + msg = _('There was no configuration found') status = ResponseCode.no_content - return HttpResponse(msg, status=status) \ + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -800,13 +918,14 @@ class NotifyView(View): status=status, ) - # Something went very wrong; return 500 - msg = _('An error occured accessing configuration.') - status = ResponseCode.internal_server_error logger.error( 'NOTIFY - %s - I/O error accessing configuration ' 'using KEY: %s', request.META['REMOTE_ADDR'], key) - return HttpResponse(msg, status=status) \ + + # Something went very wrong; return 500 + msg = _('An error occured accessing configuration') + status = ResponseCode.internal_server_error + return HttpResponse(msg, status=status, content_type='text/plain') \ if not json_response else JsonResponse({ 'error': msg, }, @@ -839,9 +958,13 @@ class NotifyView(View): 'NOTIFY - %s - Recursion limit reached (%d > %d)', request.META['REMOTE_ADDR'], recursion, settings.APPRISE_RECURSION_MAX) - return HttpResponse( - _('The recursion limit has been reached.'), - status=ResponseCode.method_not_accepted) + + status = ResponseCode.method_not_accepted + msg = _('The recursion limit has been reached') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) # Store our recursion value for our AppriseAsset() initialization kwargs['_recursion'] = recursion @@ -850,9 +973,13 @@ class NotifyView(View): logger.warning( 'NOTIFY - %s - Invalid recursion value (%s) provided', request.META['REMOTE_ADDR'], str(recursion)) - return HttpResponse( - _('An invalid recursion value was specified.'), - status=ResponseCode.bad_request) + + status = ResponseCode.bad_request + msg = _('An invalid recursion value was specified') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) # Acquire our unique identifier (if defined) uid = request.headers.get('X-Apprise-ID', '').strip() @@ -884,11 +1011,15 @@ class NotifyView(View): # we return the logs as they're processed in their text format. # The HTML response type has a bit of overhead where as it's not # the case with text/plain - content_type = \ - 'text/html' if re.search(r'text\/(\*|html)', - request.headers.get('Accept', ''), - re.IGNORECASE) \ - else 'text/plain' + if not json_response: + content_type = \ + 'text/html' if re.search(r'text\/(\*|html)', + request.headers.get('Accept', ''), + re.IGNORECASE) \ + else 'text/plain' + + else: + content_type = 'application/json' # Acquire our log level from headers if defined, otherwise use # the global one set in the settings @@ -923,13 +1054,17 @@ class NotifyView(View): esc = '' - # Format is only updated if the content_type is html - fmt = '
  • ' \ - '
    %(asctime)s
    ' \ - '
    %(levelname)s
    ' \ - f'
    {esc}%(message)s{esc}
    • ' \ - if content_type == 'text/html' else \ - settings.LOGGING['formatters']['standard']['format'] + if json_response: + fmt = f'["%(levelname)s","%(asctime)s","{esc}%(message)s{esc}"]' + + else: + # Format is only updated if the content_type is html + fmt = '
  • ' \ + '
    %(asctime)s
    ' \ + '
    %(levelname)s
    ' \ + f'
    {esc}%(message)s{esc}
    • ' \ + if content_type == 'text/html' else \ + settings.LOGGING['formatters']['standard']['format'] # Now specify our format (and over-ride the default): with apprise.LogCapture(level=level, fmt=fmt) as logs: @@ -942,15 +1077,25 @@ class NotifyView(View): attach=attach, ) - if content_type == 'text/html': + if json_response: + esc = re.escape(esc) + entries = re.findall( + r'\r*\n?(?P\["[^"]+","[^"]+",)"{}(?P.+?)' + r'{}"(?P\]\r*\n?$)(?=$|\r*\n?\["[^"]+","[^"]+","{})'.format( + esc, esc, esc), logs.getvalue(), re.DOTALL | re.MULTILINE) + + # Prepare ourselves a JSON Response + response = json.loads('[{}]'.format( + ','.join([e[0] + json.dumps(e[1].rstrip()) + e[2] for e in entries]))) + + elif content_type == 'text/html': # Iterate over our entries so that we can prepare to escape # things to be presented as HTML esc = re.escape(esc) entries = re.findall( - r'(?P
  • .*?)' - r'{}(?P.+li>$)(?=$|
  • ` tag and escape our message body: response = '
      {}
    '.format( @@ -972,15 +1117,16 @@ class NotifyView(View): if not result: # If at least one notification couldn't be sent; change up # the response to a 424 error code - msg = _('One or more notification could not be sent.') + msg = _('One or more notification could not be sent') status = ResponseCode.failed_dependency logger.warning( 'NOTIFY - %s - One or more notifications not ' 'sent%s using KEY: %s', request.META['REMOTE_ADDR'], '' if not tag else f' (Tags: {tag})', key) - return HttpResponse(response if response else msg, status=status) \ + return HttpResponse(response if response else msg, status=status, content_type=content_type) \ if not json_response else JsonResponse({ 'error': msg, + 'details': response, }, encoder=JSONEncoder, safe=False, @@ -992,13 +1138,13 @@ class NotifyView(View): request.META['REMOTE_ADDR'], '' if not tag else f'Tags: {tag}, ', key) - # Return our retrieved content - return HttpResponse( - response if response is not None else - _('Notification(s) sent.'), - content_type=content_type, - status=ResponseCode.okay, - ) + # Return our success message + status = ResponseCode.okay + return HttpResponse(response, status=status, content_type=content_type) \ + if not json_response else JsonResponse({ + 'error': None, + 'details': response, + }, encoder=JSONEncoder, safe=False, status=status) @method_decorator((gzip_page, never_cache), name='dispatch') @@ -1010,29 +1156,60 @@ class StatelessNotifyView(View): """ Handle a POST request """ + # Detect the format our incoming payload + json_payload = \ + MIME_IS_JSON.match( + request.content_type + if request.content_type + else request.headers.get( + 'content-type', '')) is not None + + # Detect the format our response should be in + json_response = True if json_payload \ + and ACCEPT_ALL.match(request.headers.get('accept', '')) else \ + MIME_IS_JSON.match(request.headers.get('accept', '')) is not None + # our content content = {} - if MIME_IS_FORM.match(request.content_type): + if not json_payload: content = {} form = NotifyByUrlForm(request.POST, request.FILES) if form.is_valid(): content.update(form.cleaned_data) - elif MIME_IS_JSON.match(request.content_type): + else: # JSON Payload # Prepare our default response try: # load our JSON content content = json.loads(request.body.decode('utf-8')) + except (RequestDataTooBig): + # DATA_UPLOAD_MAX_MEMORY_SIZE exceeded it's value; this is usually the case + # when there is a very large flie attachment that can't be pulled out of the + # payload without exceeding memory limitations (default is 3MB) + logger.warning( + 'NOTIFY - %s - JSON Payload Exceeded %dMB; operaton aborted', + request.META['REMOTE_ADDR'], (settings.DATA_UPLOAD_MAX_MEMORY_SIZE / 1048576)) + + status = ResponseCode.fields_too_large + msg = _('JSON Payload provided is to large') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) + except (AttributeError, ValueError): # could not parse JSON response... logger.warning( 'NOTIFY - %s - Invalid JSON Payload provided', request.META['REMOTE_ADDR']) - return HttpResponse( - _('Invalid JSON specified.'), - status=ResponseCode.bad_request) + status = ResponseCode.bad_request + msg = _('Invalid JSON Payload provided') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) if not content: # We could not handle the Content-Type @@ -1040,9 +1217,12 @@ class StatelessNotifyView(View): 'NOTIFY - %s - Invalid FORM Payload provided', request.META['REMOTE_ADDR']) - return HttpResponse( - _('Bad FORM Payload provided.'), - status=ResponseCode.bad_request) + status = ResponseCode.bad_request + msg = _('Bad FORM Payload provided') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) if not content.get('urls') and settings.APPRISE_STATELESS_URLS: # fallback to settings.APPRISE_STATELESS_URLS if no urls were @@ -1079,9 +1259,12 @@ class StatelessNotifyView(View): 'NOTIFY - %s - Payload lacks minimum requirements', request.META['REMOTE_ADDR']) - return HttpResponse( - _('Payload lacks minimum requirements.'), - status=ResponseCode.bad_request) + status = ResponseCode.bad_request + msg = _('Payload lacks minimum requirements') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) # Acquire our body format (if identified) body_format = content.get('format', apprise.NotifyFormat.TEXT) @@ -1089,9 +1272,13 @@ class StatelessNotifyView(View): logger.warning( 'NOTIFY - %s - Format parameter contains an unsupported ' 'value (%s)', request.META['REMOTE_ADDR'], str(body_format)) - return HttpResponse( - _('An invalid body input format was specified.'), - status=ResponseCode.bad_request) + + status = ResponseCode.bad_request + msg = _('An invalid body input format was specified') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) # Prepare our keyword arguments (to be passed into an AppriseAsset # object) @@ -1117,9 +1304,13 @@ class StatelessNotifyView(View): 'NOTIFY - %s - Recursion limit reached (%d > %d)', request.META['REMOTE_ADDR'], recursion, settings.APPRISE_RECURSION_MAX) - return HttpResponse( - _('The recursion limit has been reached.'), - status=ResponseCode.method_not_accepted) + + status = ResponseCode.method_not_accepted + msg = _('The recursion limit has been reached') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) # Store our recursion value for our AppriseAsset() initialization kwargs['_recursion'] = recursion @@ -1128,9 +1319,13 @@ class StatelessNotifyView(View): logger.warning( 'NOTIFY - %s - Invalid recursion value (%s) provided', request.META['REMOTE_ADDR'], str(recursion)) - return HttpResponse( - _('An invalid recursion value was specified.'), - status=ResponseCode.bad_request) + + status = ResponseCode.bad_request + msg = _('An invalid recursion value was specified') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) # Acquire our unique identifier (if defined) uid = request.headers.get('X-Apprise-ID', '').strip() @@ -1151,10 +1346,16 @@ class StatelessNotifyView(View): # Add URLs a_obj.add(content.get('urls')) if not len(a_obj): - return HttpResponse( - _('There was no services to notify.'), - status=ResponseCode.no_content, - ) + logger.warning( + 'NOTIFY - %s - No valid URLs provided', + request.META['REMOTE_ADDR']) + + status = ResponseCode.no_content + msg = _('There was no valid URLs provided to notify') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) # Handle Attachments attach = None @@ -1183,9 +1384,12 @@ class StatelessNotifyView(View): 'NOTIFY - %s - Bad attachment: %s', request.META['REMOTE_ADDR'], str(e)) - return HttpResponse( - _('Bad attachment'), - status=ResponseCode.bad_request) + status = ResponseCode.bad_request + msg = _('Bad Attachment') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) # Acquire our log level from headers if defined, otherwise use # the global one set in the settings @@ -1216,7 +1420,8 @@ class StatelessNotifyView(View): level = logging.DEBUG - 1 if settings.APPRISE_WEBHOOK_URL: - fmt = settings.LOGGING['formatters']['standard']['format'] + esc = '' + fmt = f'["%(levelname)s","%(asctime)s","{esc}%(message)s{esc}"]' with apprise.LogCapture(level=level, fmt=fmt) as logs: # Perform our notification at this point result = a_obj.notify( @@ -1227,7 +1432,15 @@ class StatelessNotifyView(View): attach=attach, ) - response = logs.getvalue() + esc = re.escape(esc) + entries = re.findall( + r'\r*\n?(?P\["[^"]+","[^"]+",)"{}(?P.+?)' + r'{}"(?P\]\r*\n?$)(?=$|\r*\n?\["[^"]+","[^"]+","{})'.format( + esc, esc, esc), logs.getvalue(), re.DOTALL | re.MULTILINE) + + # Prepare ourselves a JSON Response + response = json.loads('[{}]'.format( + ','.join([e[0] + json.dumps(e[1].rstrip()) + e[2] for e in entries]))) webhook_payload = { 'source': request.META['REMOTE_ADDR'], @@ -1251,19 +1464,27 @@ class StatelessNotifyView(View): if not result: # If at least one notification couldn't be sent; change up the # response to a 424 error code - return HttpResponse( - _('One or more notification could not be sent.'), - status=ResponseCode.failed_dependency) + logger.warning( + 'NOTIFY - %s - One or more stateless notification(s) could not be actioned', + request.META['REMOTE_ADDR']) + + status = ResponseCode.failed_dependency + msg = _('One or more notification could not be sent') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + 'error': msg, + }, encoder=JSONEncoder, safe=False, status=status) logger.info( 'NOTIFY - %s - Delivered Stateless Notification(s)', request.META['REMOTE_ADDR']) - # Return our retrieved content - return HttpResponse( - _('Notification(s) sent.'), - status=ResponseCode.okay, - ) + # Return our success message + status = ResponseCode.okay + msg = _('Notification(s) sent') + return HttpResponse(msg, status=status, content_type='text/plain') \ + if not json_response else JsonResponse({ + }, encoder=JSONEncoder, safe=False, status=status) @method_decorator((gzip_page, never_cache), name='dispatch') @@ -1323,7 +1544,7 @@ class JsonUrlView(View): ) # Something went very wrong; return 500 - response['error'] = _('There was no configuration found.') + response['error'] = _('There was no configuration found') return JsonResponse( response, encoder=JSONEncoder, diff --git a/apprise_api/core/settings/__init__.py b/apprise_api/core/settings/__init__.py index 4f8d547..45921c0 100644 --- a/apprise_api/core/settings/__init__.py +++ b/apprise_api/core/settings/__init__.py @@ -150,6 +150,11 @@ APPRISE_ATTACH_DIR = os.environ.get( # The maximum file attachment size allowed by the API (defined in MB) APPRISE_ATTACH_SIZE = int(os.environ.get('APPRISE_ATTACH_SIZE', 200)) * 1048576 +# The maximum size in bytes that a request body may be before raising an error +# (defined in MB) +DATA_UPLOAD_MAX_MEMORY_SIZE = abs(int(os.environ.get( + 'APPRISE_UPLOAD_MAX_MEMORY_SIZE', 3))) * 1048576 + # When set Apprise API Locks itself down so that future (configuration) # changes can not be made or accessed. It disables access to: # - the configuration screen: /cfg/{token} diff --git a/setup.cfg b/setup.cfg index 96b40a2..b2dd51a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,6 +8,7 @@ exclude = .eggs,.tox ignore = E741,E722,W503,W504,W605 statistics = true builtins = _ +max-line-length = 160 [aliases] test=pytest