From f6c48d066bf334468c19196e881234f12c09b80d Mon Sep 17 00:00:00 2001 From: Chris Caron Date: Mon, 16 Sep 2024 17:23:05 -0400 Subject: [PATCH] Matrix Well Known URI Server Discovery (#1198) --- apprise/plugins/matrix.py | 298 +++++++++++++++++++++++++++++++++---- test/test_plugin_matrix.py | 221 +++++++++++++++++++++++++-- 2 files changed, 481 insertions(+), 38 deletions(-) diff --git a/apprise/plugins/matrix.py b/apprise/plugins/matrix.py index bb9c6dbb..3dde574e 100644 --- a/apprise/plugins/matrix.py +++ b/apprise/plugins/matrix.py @@ -39,6 +39,7 @@ from time import time from .base import NotifyBase from ..url import PrivacyMode +from ..exception import AppriseException from ..common import NotifyType from ..common import NotifyImageSize from ..common import NotifyFormat @@ -56,6 +57,13 @@ MATRIX_V3_API_PATH = '/_matrix/client/v3' MATRIX_V3_MEDIA_PATH = '/_matrix/media/v3' MATRIX_V2_MEDIA_PATH = '/_matrix/media/r0' + +class MatrixDiscoveryException(AppriseException): + """ + Apprise Matrix Exception Class + """ + + # Extend HTTP Error Messages MATRIX_HTTP_ERROR_MAP = { 403: 'Unauthorized - Invalid Token.', @@ -165,9 +173,6 @@ class NotifyMatrix(NotifyBase): # Throttle a wee-bit to avoid thrashing request_rate_per_sec = 0.5 - # Our Matrix API Version - matrix_api_version = '3' - # How many retry attempts we'll make in the event the server asks us to # throttle back. default_retries = 2 @@ -183,6 +188,13 @@ class NotifyMatrix(NotifyBase): # Keep our cache for 20 days default_cache_expiry_sec = 60 * 60 * 24 * 20 + # Used for server discovery + discovery_base_key = '__discovery_base' + discovery_identity_key = '__discovery_identity' + + # Defines how long we cache our discovery for + discovery_cache_length_sec = 86400 + # Define object templates templates = ( # Targets are ignored when using t2bot mode; only a token is required @@ -256,6 +268,11 @@ class NotifyMatrix(NotifyBase): 'default': False, 'map_to': 'include_image', }, + 'discovery': { + 'name': _('Server Discovery'), + 'type': 'bool', + 'default': True, + }, 'mode': { 'name': _('Webhook Mode'), 'type': 'choice:string', @@ -283,7 +300,7 @@ class NotifyMatrix(NotifyBase): }) def __init__(self, targets=None, mode=None, msgtype=None, version=None, - include_image=False, **kwargs): + include_image=None, discovery=None, **kwargs): """ Initialize Matrix Object """ @@ -305,7 +322,12 @@ class NotifyMatrix(NotifyBase): self.transaction_id = 0 # Place an image inline with the message body - self.include_image = include_image + self.include_image = self.template_args['image']['default'] \ + if include_image is None else include_image + + # Prepare Delegate Server Lookup Check + self.discovery = self.template_args['discovery']['default'] \ + if discovery is None else discovery # Setup our mode self.mode = self.template_args['mode']['default'] \ @@ -358,6 +380,10 @@ class NotifyMatrix(NotifyBase): self.logger.warning(msg) raise TypeError(msg) + if self.mode != MatrixWebhookMode.DISABLED: + # Discovery only works when we're not using webhooks + self.discovery = False + # # Initialize from cache if present # @@ -1180,14 +1206,16 @@ class NotifyMatrix(NotifyBase): return None - def _fetch(self, path, payload=None, params=None, attachment=None, - method='POST'): + def _fetch(self, path, payload=None, params={}, attachment=None, + method='POST', url_override=None): """ Wrapper to request.post() to manage it's response better and make the send() function cleaner and easier to maintain. This function returns True if the _post was successful and False if it wasn't. + + this function returns the status code if url_override is used """ # Define our headers @@ -1200,14 +1228,20 @@ class NotifyMatrix(NotifyBase): if self.access_token is not None: headers["Authorization"] = 'Bearer %s' % self.access_token - default_port = 443 if self.secure else 80 + # Server Discovery / Well-known URI + if url_override: + url = url_override - url = \ - '{schema}://{hostname}{port}'.format( - schema='https' if self.secure else 'http', - hostname=self.host, - port='' if self.port is None - or self.port == default_port else f':{self.port}') + else: + try: + url = self.base_url + + except MatrixDiscoveryException: + # Discovery failed; we're done + return (False, {}) + + # Default return status code + status_code = requests.codes.internal_server_error if path == '/upload': # FUTURE if self.version == MatrixVersion.V3: @@ -1217,14 +1251,14 @@ class NotifyMatrix(NotifyBase): # FUTURE url += MATRIX_V2_MEDIA_PATH + path url += MATRIX_V2_MEDIA_PATH + path - params = {'filename': attachment.name} + params.update({'filename': attachment.name}) with open(attachment.path, 'rb') as fp: payload = fp.read() # Update our content type headers['Content-Type'] = attachment.mimetype - else: + elif not url_override: if self.version == MatrixVersion.V3: url += MATRIX_V3_API_PATH + path @@ -1246,7 +1280,9 @@ class NotifyMatrix(NotifyBase): # Decrement our throttle retry count retries -= 1 - self.logger.debug('Matrix POST URL: %s (cert_verify=%r)' % ( + self.logger.debug('Matrix %s URL: %s (cert_verify=%r)' % ( + 'POST' if method == 'POST' else ( + requests.put if method == 'PUT' else 'GET'), url, self.verify_certificate, )) self.logger.debug('Matrix Payload: %s' % str(payload)) @@ -1258,18 +1294,21 @@ class NotifyMatrix(NotifyBase): r = fn( url, data=dumps(payload) if not attachment else payload, - params=params, + params=None if not params else params, headers=headers, verify=self.verify_certificate, timeout=self.request_timeout, ) + # Store status code + status_code = r.status_code + self.logger.debug( 'Matrix Response: code=%d, %s' % ( r.status_code, str(r.content))) response = loads(r.content) - if r.status_code == 429: + if r.status_code == requests.codes.too_many_requests: wait = self.default_wait_ms / 1000 try: wait = response['retry_after_ms'] / 1000 @@ -1310,7 +1349,8 @@ class NotifyMatrix(NotifyBase): 'Response Details:\r\n{}'.format(r.content)) # Return; we're done - return (False, response) + return ( + False if not url_override else status_code, response) except (AttributeError, TypeError, ValueError): # This gets thrown if we can't parse our JSON Response @@ -1320,27 +1360,27 @@ class NotifyMatrix(NotifyBase): self.logger.warning('Invalid response from Matrix server.') self.logger.debug( 'Response Details:\r\n{}'.format(r.content)) - return (False, {}) + return (False if not url_override else status_code, {}) - except requests.RequestException as e: + except (requests.TooManyRedirects, requests.RequestException) as e: self.logger.warning( 'A Connection error occurred while registering with Matrix' ' server.') - self.logger.debug('Socket Exception: %s' % str(e)) + self.logger.debug('Socket Exception: %s', str(e)) # Return; we're done - return (False, response) + return (False if not url_override else status_code, response) except (OSError, IOError) as e: self.logger.warning( 'An I/O error occurred while reading {}.'.format( attachment.name if attachment else 'unknown file')) - self.logger.debug('I/O Exception: %s' % str(e)) - return (False, {}) + self.logger.debug('I/O Exception: %s', str(e)) + return (False if not url_override else status_code, {}) - return (True, response) + return (True if not url_override else status_code, response) # If we get here, we ran out of retries - return (False, {}) + return (False if not url_override else status_code, {}) def __del__(self): """ @@ -1426,6 +1466,7 @@ class NotifyMatrix(NotifyBase): 'mode': self.mode, 'version': self.version, 'msgtype': self.msgtype, + 'discovery': 'yes' if self.discovery else 'no', } # Extend our parameters @@ -1495,6 +1536,10 @@ class NotifyMatrix(NotifyBase): results['include_image'] = parse_bool(results['qsd'].get( 'image', NotifyMatrix.template_args['image']['default'])) + # Boolean to perform a server discovery + results['discovery'] = parse_bool(results['qsd'].get( + 'discovery', NotifyMatrix.template_args['discovery']['default'])) + # Get our mode results['mode'] = results['qsd'].get('mode') @@ -1554,3 +1599,200 @@ class NotifyMatrix(NotifyBase): else '{}&{}'.format(result.group('params'), mode))) return None + + def server_discovery(self): + """ + Home Server Discovery as documented here: + https://spec.matrix.org/v1.11/client-server-api/#well-known-uri + """ + + if not (self.discovery and self.secure): + # Nothing further to do with insecure server setups + return '' + + # Get our content from cache + base_url, identity_url = ( + self.store.get(self.discovery_base_key), + self.store.get(self.discovery_identity_key), + ) + + if not (base_url is None and identity_url is None): + # We can use our cached value and return early + return base_url + + # 1. Extract the server name from the user’s Matrix ID by splitting + # the Matrix ID at the first colon. + verify_url = f'https://{self.host}/.well-known/matrix/client' + code, wk_response = self._fetch( + None, method='GET', url_override=verify_url) + + # Output may look as follows: + # { + # "m.homeserver": { + # "base_url": "https://matrix.example.com" + # }, + # "m.identity_server": { + # "base_url": "https://nuxref.com" + # } + # } + + if code == requests.codes.not_found: + # This is an acceptable response; we're done + self.logger.debug( + 'Matrix Well-Known Base URI not found at %s', verify_url) + + # Set our keys out for fast recall later on + self.store.set( + self.discovery_base_key, '', + expires=self.discovery_cache_length_sec) + self.store.set( + self.discovery_identity_key, '', + expires=self.discovery_cache_length_sec) + return '' + + elif code != requests.codes.ok: + # We're done early as we couldn't load the results + msg = 'Matrix Well-Known Base URI Discovery Failed' + self.logger.warning( + '%s - %s returned error code: %d', msg, verify_url, code) + raise MatrixDiscoveryException(msg, error_code=code) + + if not wk_response: + # This is an acceptable response; we simply do nothing + self.logger.debug( + 'Matrix Well-Known Base URI not defined %s', verify_url) + + # Set our keys out for fast recall later on + self.store.set( + self.discovery_base_key, '', + expires=self.discovery_cache_length_sec) + self.store.set( + self.discovery_identity_key, '', + expires=self.discovery_cache_length_sec) + return '' + + # + # Parse our m.homeserver information + # + try: + base_url = wk_response['m.homeserver']['base_url'].rstrip('/') + results = NotifyBase.parse_url(base_url, verify_host=True) + + except (AttributeError, TypeError, KeyError): + # AttributeError: result wasn't a string (rstrip failed) + # TypeError : wk_response wasn't a dictionary + # KeyError : wk_response not to standards + results = None + + if not results: + msg = 'Matrix Well-Known Base URI Discovery Failed' + self.logger.warning( + '%s - m.homeserver payload is missing or invalid: %s', + msg, str(wk_response)) + raise MatrixDiscoveryException(msg) + + # + # Our .well-known extraction was successful; now we need to verify + # that the version information resolves. + # + verify_url = f'{base_url}/_matrix/client/versions' + # Post our content + code, response = self._fetch( + None, method='GET', url_override=verify_url) + if code != requests.codes.ok: + # We're done early as we couldn't load the results + msg = 'Matrix Well-Known Base URI Discovery Verification Failed' + self.logger.warning( + '%s - %s returned error code: %d', msg, verify_url, code) + raise MatrixDiscoveryException(msg, error_code=code) + + # + # Phase 2: Handle m.identity_server IF defined + # + if 'm.identity_server' in wk_response: + try: + identity_url = \ + wk_response['m.identity_server']['base_url'].rstrip('/') + results = NotifyBase.parse_url(identity_url, verify_host=True) + + except (AttributeError, TypeError, KeyError): + # AttributeError: result wasn't a string (rstrip failed) + # TypeError : wk_response wasn't a dictionary + # KeyError : wk_response not to standards + results = None + + if not results: + msg = 'Matrix Well-Known Identity URI Discovery Failed' + self.logger.warning( + '%s - m.identity_server payload is missing or invalid: %s', + msg, str(wk_response)) + raise MatrixDiscoveryException(msg) + + # + # Verify identity server found + # + verify_url = f'{identity_url}/_matrix/identity/v2' + + # Post our content + code, response = self._fetch( + None, method='GET', url_override=verify_url) + if code != requests.codes.ok: + # We're done early as we couldn't load the results + msg = 'Matrix Well-Known Identity URI Discovery Failed' + self.logger.warning( + '%s - %s returned error code: %d', msg, verify_url, code) + raise MatrixDiscoveryException(msg, error_code=code) + + # Update our cache + self.store.set( + self.discovery_identity_key, identity_url, + # Add 2 seconds to prevent this key from expiring before base + expires=self.discovery_cache_length_sec + 2) + else: + # No identity server + self.store.set( + self.discovery_identity_key, '', + # Add 2 seconds to prevent this key from expiring before base + expires=self.discovery_cache_length_sec + 2) + + # Update our cache + self.store.set( + self.discovery_base_key, base_url, + expires=self.discovery_cache_length_sec) + + return base_url + + @property + def base_url(self): + """ + Returns the base_url if known + """ + try: + base_url = self.server_discovery() + if base_url: + # We can use our cached value and return early + return base_url + + except MatrixDiscoveryException: + self.store.clear( + self.discovery_base_key, self.discovery_identity_key) + raise + + # If we get hear, we need to build our URL dynamically based on what + # was provided to us during the plugins initialization + default_port = 443 if self.secure else 80 + + return '{schema}://{hostname}{port}'.format( + schema='https' if self.secure else 'http', + hostname=self.host, + port='' if self.port is None + or self.port == default_port else f':{self.port}') + + @property + def identity_url(self): + """ + Returns the identity_url if known + """ + base_url = self.base_url + identity_url = self.store.get(self.discovery_identity_key) + return base_url if not identity_url else identity_url diff --git a/test/test_plugin_matrix.py b/test/test_plugin_matrix.py index 1473887a..5562ce1c 100644 --- a/test/test_plugin_matrix.py +++ b/test/test_plugin_matrix.py @@ -32,9 +32,10 @@ import requests import pytest from apprise import ( Apprise, AppriseAsset, AppriseAttachment, NotifyType, PersistentStoreMode) -from json import dumps +from json import dumps, loads from apprise.plugins.matrix import NotifyMatrix +from apprise.plugins.matrix import MatrixDiscoveryException from helpers import AppriseURLTester # Disable logging for a cleaner testing output @@ -47,6 +48,14 @@ MATRIX_GOOD_RESPONSE = dumps({ 'joined_rooms': ['!abc123:localhost', '!def456:localhost'], 'access_token': 'abcd1234', 'home_server': 'localhost', + + # Simulate .well-known + "m.homeserver": { + "base_url": "https://matrix.example.com" + }, + "m.identity_server": { + "base_url": "https://vector.im" + }, }) # Attachment Directory @@ -1012,6 +1021,197 @@ def test_plugin_matrix_attachments_api_v3(mock_post, mock_get, mock_put): del obj +@mock.patch('requests.get') +@mock.patch('requests.post') +def test_plugin_matrix_discovery_service(mock_post, mock_get): + """ + NotifyMatrix() Discovery Service + + """ + + # Prepare a good response + response = mock.Mock() + response.status_code = requests.codes.ok + response.content = MATRIX_GOOD_RESPONSE.encode('utf-8') + + # Prepare a good response + bad_response = mock.Mock() + bad_response.status_code = requests.codes.unauthorized + bad_response.content = MATRIX_GOOD_RESPONSE.encode('utf-8') + + # Prepare Mock return object + mock_post.return_value = response + mock_get.return_value = response + + # Instantiate our object + obj = Apprise.instantiate( + 'matrixs://user:pass@example.com/#general?v=2&discovery=yes') + assert obj.notify('body') is True + + response = mock.Mock() + response.status_code = requests.codes.unavailable + _resp = loads(MATRIX_GOOD_RESPONSE) + + mock_get.return_value = response + mock_post.return_value = response + obj = Apprise.instantiate( + 'matrixs://user:pass@example.com/#general?v=2&discovery=yes') + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + + # Invalid host / fallback is to resolve our own host + with pytest.raises(MatrixDiscoveryException): + obj.base_url + + # Verify cache is not saved + assert NotifyMatrix.discovery_base_key not in obj.store + assert NotifyMatrix.discovery_identity_key not in obj.store + + response.status_code = requests.codes.ok + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + + # bad data + _resp['m.homeserver'] = '!garbage!:303' + response.content = dumps(_resp).encode('utf-8') + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + + with pytest.raises(MatrixDiscoveryException): + obj.base_url + + # Verify cache is not saved + assert NotifyMatrix.discovery_base_key not in obj.store + assert NotifyMatrix.discovery_identity_key not in obj.store + + # We fail our discovery and therefore can't send our notification + assert obj.notify('hello world') is False + + # bad key + _resp['m.homeserver'] = {} + response.content = dumps(_resp).encode('utf-8') + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + with pytest.raises(MatrixDiscoveryException): + obj.base_url + + # Verify cache is not saved + assert NotifyMatrix.discovery_base_key not in obj.store + assert NotifyMatrix.discovery_identity_key not in obj.store + + _resp['m.homeserver'] = {'base_url': 'https://nuxref.com/base'} + response.content = dumps(_resp).encode('utf-8') + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + assert obj.base_url == 'https://nuxref.com/base' + assert obj.identity_url == "https://vector.im" + + # Verify cache saved + assert NotifyMatrix.discovery_base_key in obj.store + assert NotifyMatrix.discovery_identity_key in obj.store + + # Discovery passes so notifications work too + assert obj.notify('hello world') is True + + # bad data + _resp['m.identity_server'] = '!garbage!:303' + response.content = dumps(_resp).encode('utf-8') + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + + with pytest.raises(MatrixDiscoveryException): + obj.base_url + + # Verify cache is not saved + assert NotifyMatrix.discovery_base_key not in obj.store + assert NotifyMatrix.discovery_identity_key not in obj.store + + # no key + _resp['m.identity_server'] = {} + response.content = dumps(_resp).encode('utf-8') + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + + with pytest.raises(MatrixDiscoveryException): + obj.base_url + + # Verify cache is not saved + assert NotifyMatrix.discovery_base_key not in obj.store + assert NotifyMatrix.discovery_identity_key not in obj.store + + # remove + del _resp['m.identity_server'] + response.content = dumps(_resp).encode('utf-8') + + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + assert obj.base_url == 'https://nuxref.com/base' + assert obj.identity_url == 'https://nuxref.com/base' + + # restore + _resp['m.identity_server'] = {'base_url': '"https://vector.im'} + response.content = dumps(_resp).encode('utf-8') + + # Not found is an acceptable response (no exceptions thrown) + response.status_code = requests.codes.not_found + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + assert obj.base_url == 'https://example.com' + assert obj.identity_url == 'https://example.com' + + # Verify cache saved + assert NotifyMatrix.discovery_base_key in obj.store + assert NotifyMatrix.discovery_identity_key in obj.store + + # Discovery passes so notifications work too + response.status_code = requests.codes.ok + assert obj.notify('hello world') is True + + response.status_code = requests.codes.ok + mock_get.return_value = None + mock_get.side_effect = (response, bad_response) + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + + with pytest.raises(MatrixDiscoveryException): + obj.base_url + + # Verify cache is not saved + assert NotifyMatrix.discovery_base_key not in obj.store + assert NotifyMatrix.discovery_identity_key not in obj.store + + # Test case where ourIdentity URI fails to do it's check + mock_get.side_effect = (response, response, bad_response) + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + + with pytest.raises(MatrixDiscoveryException): + obj.base_url + + # Verify cache is not saved + assert NotifyMatrix.discovery_base_key not in obj.store + assert NotifyMatrix.discovery_identity_key not in obj.store + + # Test an empty block response + response.status_code = requests.codes.ok + response.content = '' + mock_get.return_value = response + mock_get.side_effect = None + mock_post.return_value = response + mock_post.side_effect = None + obj.store.clear( + NotifyMatrix.discovery_base_key, NotifyMatrix.discovery_identity_key) + + assert obj.base_url == 'https://example.com' + assert obj.identity_url == 'https://example.com' + + # Verify cache saved + assert NotifyMatrix.discovery_base_key in obj.store + assert NotifyMatrix.discovery_identity_key in obj.store + + del obj + + @mock.patch('requests.get') @mock.patch('requests.post') def test_plugin_matrix_attachments_api_v2(mock_post, mock_get): @@ -1068,17 +1268,18 @@ def test_plugin_matrix_attachments_api_v2(mock_post, mock_get): # Test our call count assert mock_post.call_count == 5 assert mock_post.call_args_list[0][0][0] == \ - 'https://localhost/_matrix/client/r0/login' + 'https://matrix.example.com/_matrix/client/r0/login' assert mock_post.call_args_list[1][0][0] == \ - 'https://localhost/_matrix/media/r0/upload' + 'https://matrix.example.com/_matrix/media/r0/upload' assert mock_post.call_args_list[2][0][0] == \ - 'https://localhost/_matrix/client/r0/join/%23general%3Alocalhost' + 'https://matrix.example.com/_matrix/client/r0/' \ + 'join/%23general%3Alocalhost' assert mock_post.call_args_list[3][0][0] == \ - 'https://localhost/_matrix/client/r0/rooms/%21abc123%3Alocalhost/' \ - 'send/m.room.message' + 'https://matrix.example.com/_matrix/client/r0' \ + '/rooms/%21abc123%3Alocalhost/send/m.room.message' assert mock_post.call_args_list[4][0][0] == \ - 'https://localhost/_matrix/client/r0/rooms/%21abc123%3Alocalhost/' \ - 'send/m.room.message' + 'https://matrix.example.com/_matrix/client/r0/' \ + 'rooms/%21abc123%3Alocalhost/send/m.room.message' # Attach an unsupported file type; these are skipped attach = AppriseAttachment( @@ -1135,9 +1336,9 @@ def test_plugin_matrix_attachments_api_v2(mock_post, mock_get): # Force a object removal (thus a logout call) del obj - # Instantiate our object + # Instantiate our object (no discovery required) obj = Apprise.instantiate( - 'matrixs://user:pass@localhost/#general?v=2&image=y') + 'matrixs://user:pass@localhost/#general?v=2&discovery=no&image=y') # Reset our object mock_post.reset_mock()