From 3f6ae1aefed645727f8a0386ddfdb30fca69c949 Mon Sep 17 00:00:00 2001 From: Timothy Hobbs Date: Fri, 5 Oct 2018 14:54:22 +0200 Subject: [PATCH 1/5] Transform UserSettings into a normal django model --- helpdesk/forms.py | 41 ++------- .../commands/create_usersettings.py | 3 +- .../migrations/0020_depickle_user_settings.py | 69 ++++++++++++++ helpdesk/models.py | 92 ++++++++++++------- helpdesk/settings.py | 20 ++-- .../templates/helpdesk/user_settings.html | 9 +- helpdesk/tests/helpers.py | 18 ---- helpdesk/tests/test_navigation.py | 12 +-- helpdesk/urls.py | 2 +- helpdesk/views/public.py | 2 +- helpdesk/views/staff.py | 37 +++----- 11 files changed, 172 insertions(+), 133 deletions(-) create mode 100644 helpdesk/migrations/0020_depickle_user_settings.py diff --git a/helpdesk/forms.py b/helpdesk/forms.py index f5d1473c..4192e3e9 100644 --- a/helpdesk/forms.py +++ b/helpdesk/forms.py @@ -19,7 +19,7 @@ from django.utils import timezone from helpdesk.lib import send_templated_mail, safe_template_context, process_attachments from helpdesk.models import (Ticket, Queue, FollowUp, Attachment, IgnoreEmail, TicketCC, - CustomField, TicketCustomFieldValue, TicketDependency) + CustomField, TicketCustomFieldValue, TicketDependency, UserSettings) from helpdesk import settings as helpdesk_settings User = get_user_model() @@ -253,7 +253,7 @@ class AbstractTicketForm(CustomFieldMixin, forms.Form): if ticket.assigned_to and \ ticket.assigned_to != user and \ - ticket.assigned_to.usersettings_helpdesk.settings.get('email_on_ticket_assign', False) and \ + ticket.assigned_to.usersettings_helpdesk.email_on_ticket_assign and \ ticket.assigned_to.email and \ ticket.assigned_to.email not in messages_sent_to: send_templated_mail( @@ -406,40 +406,11 @@ class PublicTicketForm(AbstractTicketForm): return ticket -class UserSettingsForm(forms.Form): - login_view_ticketlist = forms.BooleanField( - label=_('Show Ticket List on Login?'), - help_text=_('Display the ticket list upon login? Otherwise, the dashboard is shown.'), - required=False, - ) +class UserSettingsForm(forms.ModelForm): - email_on_ticket_change = forms.BooleanField( - label=_('E-mail me on ticket change?'), - help_text=_('If you\'re the ticket owner and the ticket is changed via the web by somebody else, do you want to receive an e-mail?'), - required=False, - ) - - email_on_ticket_assign = forms.BooleanField( - label=_('E-mail me when assigned a ticket?'), - help_text=_('If you are assigned a ticket via the web, do you want to receive an e-mail?'), - required=False, - ) - - tickets_per_page = forms.ChoiceField( - label=_('Number of tickets to show per page'), - help_text=_('How many tickets do you want to see on the Ticket List page?'), - required=False, - choices=((10, '10'), (25, '25'), (50, '50'), (100, '100')), - ) - - use_email_as_submitter = forms.BooleanField( - label=_('Use my e-mail address when submitting tickets?'), - help_text=_('When you submit a ticket, do you want to automatically ' - 'use your e-mail address as the submitter address? You ' - 'can type a different e-mail address when entering the ' - 'ticket if needed, this option only changes the default.'), - required=False, - ) + class Meta: + model = UserSettings + exclude = ['user', 'settings_pickled'] class EmailIgnoreForm(forms.ModelForm): diff --git a/helpdesk/management/commands/create_usersettings.py b/helpdesk/management/commands/create_usersettings.py index 46280159..9e5ced07 100644 --- a/helpdesk/management/commands/create_usersettings.py +++ b/helpdesk/management/commands/create_usersettings.py @@ -29,5 +29,4 @@ class Command(BaseCommand): def handle(self, *args, **options): """handle command line""" for u in User.objects.all(): - UserSettings.objects.get_or_create(user=u, - defaults={'settings': DEFAULT_USER_SETTINGS}) + UserSettings.objects.get_or_create(user=u) diff --git a/helpdesk/migrations/0020_depickle_user_settings.py b/helpdesk/migrations/0020_depickle_user_settings.py new file mode 100644 index 00000000..556dcafe --- /dev/null +++ b/helpdesk/migrations/0020_depickle_user_settings.py @@ -0,0 +1,69 @@ +# Generated by Django 2.0.7 on 2018-10-19 14:11 + +from django.db import migrations, models +import helpdesk.models + +def unpickle_settings(settings_pickled): + # return a python dictionary representing the pickled data. + try: + import pickle + except ImportError: + import cPickle as pickle + from helpdesk.lib import b64decode + try: + if six.PY2: + return pickle.loads(b64decode(str(settings_pickled))) + else: + return pickle.loads(b64decode(settings_pickled.encode('utf-8'))) + except Exception: + return {} + +def move_old_values(apps, schema_editor): + UserSettings = apps.get_model("helpdesk", "UserSettings") + db_alias = schema_editor.connection.alias + + for user_settings in UserSettings.objects.using(db_alias).all(): + if user_settings.settings_pickled: + settings_dict = unpickle_settings(user_settings.settings_pickled) + for setting, value in settings_dict.items(): + user_settings.__set_attr__(setting, value) + +class Migration(migrations.Migration): + + dependencies = [ + ('helpdesk', '0019_ticket_secret_key'), + ] + + operations = [ + migrations.AddField( + model_name='usersettings', + name='email_on_ticket_assign', + field=models.BooleanField(default=helpdesk.models.email_on_ticket_assign_default, help_text='If you are assigned a ticket via the web, do you want to receive an e-mail?', verbose_name='E-mail me when assigned a ticket?'), + ), + migrations.AddField( + model_name='usersettings', + name='email_on_ticket_change', + field=models.BooleanField(default=helpdesk.models.email_on_ticket_change_default, help_text="If you're the ticket owner and the ticket is changed via the web by somebody else, do you want to receive an e-mail?", verbose_name='E-mail me on ticket change?'), + ), + migrations.AddField( + model_name='usersettings', + name='login_view_ticketlist', + field=models.BooleanField(default=helpdesk.models.login_view_ticketlist_default, help_text='Display the ticket list upon login? Otherwise, the dashboard is shown.', verbose_name='Show Ticket List on Login?'), + ), + migrations.AddField( + model_name='usersettings', + name='tickets_per_page', + field=models.IntegerField(choices=[(10, '10'), (25, '25'), (50, '50'), (100, '100')], default=helpdesk.models.tickets_per_page_default, help_text='How many tickets do you want to see on the Ticket List page?', verbose_name='Number of tickets to show per page'), + ), + migrations.AddField( + model_name='usersettings', + name='use_email_as_submitter', + field=models.BooleanField(default=helpdesk.models.use_email_as_submitter_default, help_text='When you submit a ticket, do you want to automatically use your e-mail address as the submitter address? You can type a different e-mail address when entering the ticket if needed, this option only changes the default.', verbose_name='Use my e-mail address when submitting tickets?'), + ), + migrations.AlterField( + model_name='usersettings', + name='settings_pickled', + field=models.TextField(blank=True, help_text='DEPRECATED! This is a base64-encoded representation of a pickled Python dictionary. Do not change this field via the admin.', null=True, verbose_name='DEPRECATED! Settings Dictionary DEPRECATED!'), + ), + migrations.RunPython(move_old_values), + ] diff --git a/helpdesk/models.py b/helpdesk/models.py index cb80219d..71d44e9d 100644 --- a/helpdesk/models.py +++ b/helpdesk/models.py @@ -1119,15 +1119,39 @@ class SavedSearch(models.Model): verbose_name_plural = _('Saved searches') +def get_default_setting(setting): + from helpdesk.settings import DEFAULT_USER_SETTINGS + return DEFAULT_USER_SETTINGS[setting] + + +def login_view_ticketlist_default(): + return get_default_setting('login_view_ticketlist') + + +def email_on_ticket_change_default(): + return get_default_setting('email_on_ticket_change') + + +def email_on_ticket_assign_default(): + return get_default_setting('email_on_ticket_assign') + + +def tickets_per_page_default(): + return get_default_setting('tickets_per_page') + + +def use_email_as_submitter_default(): + return get_default_setting('use_email_as_submitter') + + @python_2_unicode_compatible class UserSettings(models.Model): """ A bunch of user-specific settings that we want to be able to define, such as notification preferences and other things that should probably be configurable. - - We should always refer to user.usersettings_helpdesk.settings['setting_name']. """ + PAGE_SIZES = ((10, '10'), (25, '25'), (50, '50'), (100, '100')) user = models.OneToOneField( settings.AUTH_USER_MODEL, @@ -1135,41 +1159,46 @@ class UserSettings(models.Model): related_name="usersettings_helpdesk") settings_pickled = models.TextField( - _('Settings Dictionary'), - help_text=_('This is a base64-encoded representation of a pickled Python dictionary. ' + _('DEPRECATED! Settings Dictionary DEPRECATED!'), + help_text=_('DEPRECATED! This is a base64-encoded representation of a pickled Python dictionary. ' 'Do not change this field via the admin.'), blank=True, null=True, ) - def _set_settings(self, data): - # data should always be a Python dictionary. - try: - import pickle - except ImportError: - import cPickle as pickle - from helpdesk.lib import b64encode - if six.PY2: - self.settings_pickled = b64encode(pickle.dumps(data)) - else: - self.settings_pickled = b64encode(pickle.dumps(data)).decode() + login_view_ticketlist = models.BooleanField( + verbose_name=_('Show Ticket List on Login?'), + help_text=_('Display the ticket list upon login? Otherwise, the dashboard is shown.'), + default=login_view_ticketlist_default, + ) - def _get_settings(self): - # return a python dictionary representing the pickled data. - try: - import pickle - except ImportError: - import cPickle as pickle - from helpdesk.lib import b64decode - try: - if six.PY2: - return pickle.loads(b64decode(str(self.settings_pickled))) - else: - return pickle.loads(b64decode(self.settings_pickled.encode('utf-8'))) - except pickle.UnpicklingError: - return {} + email_on_ticket_change = models.BooleanField( + verbose_name=_('E-mail me on ticket change?'), + help_text=_('If you\'re the ticket owner and the ticket is changed via the web by somebody else, do you want to receive an e-mail?'), + default=email_on_ticket_change_default, + ) - settings = property(_get_settings, _set_settings) + email_on_ticket_assign = models.BooleanField( + verbose_name=_('E-mail me when assigned a ticket?'), + help_text=_('If you are assigned a ticket via the web, do you want to receive an e-mail?'), + default=email_on_ticket_assign_default, + ) + + tickets_per_page = models.IntegerField( + verbose_name=_('Number of tickets to show per page'), + help_text=_('How many tickets do you want to see on the Ticket List page?'), + default=tickets_per_page_default, + choices=PAGE_SIZES, + ) + + use_email_as_submitter = models.BooleanField( + verbose_name=_('Use my e-mail address when submitting tickets?'), + help_text=_('When you submit a ticket, do you want to automatically ' + 'use your e-mail address as the submitter address? You ' + 'can type a different e-mail address when entering the ' + 'ticket if needed, this option only changes the default.'), + default=use_email_as_submitter_default, + ) def __str__(self): return 'Preferences for %s' % self.user @@ -1188,9 +1217,8 @@ def create_usersettings(sender, instance, created, **kwargs): If we end up with users with no UserSettings, then we get horrible 'DoesNotExist: UserSettings matching query does not exist.' errors. """ - from helpdesk.settings import DEFAULT_USER_SETTINGS if created: - UserSettings.objects.create(user=instance, settings=DEFAULT_USER_SETTINGS) + UserSettings.objects.create(user=instance) models.signals.post_save.connect(create_usersettings, sender=settings.AUTH_USER_MODEL) diff --git a/helpdesk/settings.py b/helpdesk/settings.py index be2efbd3..a9009150 100644 --- a/helpdesk/settings.py +++ b/helpdesk/settings.py @@ -6,20 +6,18 @@ import warnings from django.conf import settings from django.core.exceptions import ImproperlyConfigured +DEFAULT_USER_SETTINGS = { + 'login_view_ticketlist': True, + 'email_on_ticket_change': True, + 'email_on_ticket_assign': True, + 'tickets_per_page': 25, + 'use_email_as_submitter': True, +} try: - DEFAULT_USER_SETTINGS = settings.HELPDESK_DEFAULT_SETTINGS + DEFAULT_USER_SETTINGS.update(settings.HELPDESK_DEFAULT_SETTINGS) except AttributeError: - DEFAULT_USER_SETTINGS = None - -if not isinstance(DEFAULT_USER_SETTINGS, dict): - DEFAULT_USER_SETTINGS = { - 'use_email_as_submitter': True, - 'email_on_ticket_assign': True, - 'email_on_ticket_change': True, - 'login_view_ticketlist': True, - 'tickets_per_page': 25 - } + pass HAS_TAG_SUPPORT = False diff --git a/helpdesk/templates/helpdesk/user_settings.html b/helpdesk/templates/helpdesk/user_settings.html index 584f3aa6..3cb8e942 100644 --- a/helpdesk/templates/helpdesk/user_settings.html +++ b/helpdesk/templates/helpdesk/user_settings.html @@ -7,11 +7,14 @@

{% blocktrans %}Use the following options to change the way your helpdesk system works for you. These settings do not impact any other user.{% endblocktrans %}

-
+{% block form_content %} + + {% csrf_token %} {{ form|bootstrap }}
- +
-{% csrf_token %}
+ +{% endblock %} {% endblock %} diff --git a/helpdesk/tests/helpers.py b/helpdesk/tests/helpers.py index 0f456bed..ef2c73ee 100644 --- a/helpdesk/tests/helpers.py +++ b/helpdesk/tests/helpers.py @@ -43,24 +43,6 @@ def reload_urlconf(urlconf=None): clear_url_caches() -def update_user_settings(user, **kwargs): - usersettings = user.usersettings_helpdesk - settings = usersettings.settings - settings.update(kwargs) - usersettings.settings = settings - usersettings.save() - - -def delete_user_settings(user, *args): - usersettings = user.usersettings_helpdesk - settings = usersettings.settings - for setting in args: - if setting in settings: - del settings[setting] - usersettings.settings = settings - usersettings.save() - - def create_ticket(**kwargs): q = kwargs.get('queue', None) if q is None: diff --git a/helpdesk/tests/test_navigation.py b/helpdesk/tests/test_navigation.py index c95225ff..24fcc3fc 100644 --- a/helpdesk/tests/test_navigation.py +++ b/helpdesk/tests/test_navigation.py @@ -6,7 +6,7 @@ from django.test import TestCase from helpdesk import settings as helpdesk_settings from helpdesk.models import Queue -from helpdesk.tests.helpers import (get_staff_user, reload_urlconf, User, update_user_settings, delete_user_settings, create_ticket, print_response) +from helpdesk.tests.helpers import (get_staff_user, reload_urlconf, User, create_ticket, print_response) class KBDisabledTestCase(TestCase): @@ -228,11 +228,8 @@ class HomePageTestCase(TestCase): user = get_staff_user() # login_view_ticketlist is False... - update_user_settings(user, login_view_ticketlist=False) - self.assertUserRedirectedToView(user, 'helpdesk:dashboard') - - # ... or missing - delete_user_settings(user, 'login_view_ticketlist') + user.usersettings_helpdesk.login_view_ticketlist = False + user.usersettings_helpdesk.save() self.assertUserRedirectedToView(user, 'helpdesk:dashboard') def test_no_user_settings_redirect_to_dashboard(self): @@ -246,7 +243,8 @@ class HomePageTestCase(TestCase): def test_redirect_to_ticket_list(self): """Authenticated users are redirected to the ticket list based on their user settings""" user = get_staff_user() - update_user_settings(user, login_view_ticketlist=True) + user.usersettings_helpdesk.login_view_ticketlist = True + user.usersettings_helpdesk.save() self.assertUserRedirectedToView(user, 'helpdesk:list') diff --git a/helpdesk/urls.py b/helpdesk/urls.py index 02d1ff1d..44e23328 100644 --- a/helpdesk/urls.py +++ b/helpdesk/urls.py @@ -127,7 +127,7 @@ urlpatterns = [ name='delete_query'), url(r'^settings/$', - staff.user_settings, + staff.EditUserSettingsView.as_view(), name='user_settings'), url(r'^ignore/$', diff --git a/helpdesk/views/public.py b/helpdesk/views/public.py index 886d9cee..157a31f3 100644 --- a/helpdesk/views/public.py +++ b/helpdesk/views/public.py @@ -49,7 +49,7 @@ class CreateTicketView(FormView): (request.user.is_authenticated and helpdesk_settings.HELPDESK_ALLOW_NON_STAFF_TICKET_UPDATE): try: - if request.user.usersettings_helpdesk.settings.get('login_view_ticketlist', False): + if request.user.usersettings_helpdesk.login_view_ticketlist: return HttpResponseRedirect(reverse('helpdesk:list')) else: return HttpResponseRedirect(reverse('helpdesk:dashboard')) diff --git a/helpdesk/views/staff.py b/helpdesk/views/staff.py index 54961f07..2c7241e0 100644 --- a/helpdesk/views/staff.py +++ b/helpdesk/views/staff.py @@ -15,7 +15,7 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.decorators import user_passes_test from django.contrib.contenttypes.models import ContentType -from django.urls import reverse +from django.urls import reverse, reverse_lazy from django.core.exceptions import ValidationError, PermissionDenied from django.db import connection from django.db.models import Q @@ -26,7 +26,7 @@ from django.utils.translation import ugettext as _ from django.utils.html import escape from django import forms from django.utils import timezone -from django.views.generic.edit import FormView +from django.views.generic.edit import FormView, UpdateView from django.utils import six @@ -44,7 +44,7 @@ from helpdesk.lib import ( ) from helpdesk.models import ( Ticket, Queue, FollowUp, TicketChange, PreSetReply, Attachment, SavedSearch, - IgnoreEmail, TicketCC, TicketDependency, + IgnoreEmail, TicketCC, TicketDependency, UserSettings, ) from helpdesk import settings as helpdesk_settings from helpdesk.views.permissions import MustBeStaffMixin @@ -635,11 +635,9 @@ def update_ticket(request, ticket_id, public=False): if (not reassigned or (reassigned and - ticket.assigned_to.usersettings_helpdesk.settings.get( - 'email_on_ticket_assign', False))) or \ + ticket.assigned_to.usersettings_helpdesk.email_on_ticket_assign)) or \ (not reassigned and - ticket.assigned_to.usersettings_helpdesk.settings.get( - 'email_on_ticket_change', False)): + ticket.assigned_to.usersettings_helpdesk.email_on_ticket_change): send_templated_mail( template_staff, @@ -983,7 +981,7 @@ def ticket_list(request): return render(request, 'helpdesk/ticket_list.html', dict( context, tickets=ticket_qs, - default_tickets_per_page=request.user.usersettings_helpdesk.settings.get('tickets_per_page') or 25, + default_tickets_per_page=request.user.usersettings_helpdesk.tickets_per_page, user_choices=User.objects.filter(is_active=True, is_staff=True), queue_choices=user_queues, status_choices=Ticket.STATUS_CHOICES, @@ -1028,7 +1026,7 @@ class CreateTicketView(MustBeStaffMixin, FormView): def get_initial(self): initial_data = {} request = self.request - if request.user.usersettings_helpdesk.settings.get('use_email_as_submitter', False) and request.user.email: + if request.user.usersettings_helpdesk.use_email_as_submitter and request.user.email: initial_data['submitter_email'] = request.user.email if 'queue' in request.GET: initial_data['queue'] = request.GET['queue'] @@ -1395,21 +1393,14 @@ def delete_saved_query(request, id): delete_saved_query = staff_member_required(delete_saved_query) -@helpdesk_staff_member_required -def user_settings(request): - s = request.user.usersettings_helpdesk - if request.POST: - form = UserSettingsForm(request.POST) - if form.is_valid(): - s.settings = form.cleaned_data - s.save() - else: - form = UserSettingsForm(s.settings) +class EditUserSettingsView(MustBeStaffMixin, UpdateView): + template_name = 'helpdesk/user_settings.html' + form_class = UserSettingsForm + model = UserSettings + success_url = reverse_lazy('helpdesk:dashboard') - return render(request, 'helpdesk/user_settings.html', {'form': form}) - - -user_settings = staff_member_required(user_settings) + def get_object(self): + return UserSettings.objects.get_or_create(user=self.request.user)[0] @helpdesk_superuser_required From 97e23e86f0f6e74216d5a328f0cba17d6f3dabe6 Mon Sep 17 00:00:00 2001 From: Timothy Hobbs Date: Wed, 24 Oct 2018 17:36:01 +0200 Subject: [PATCH 2/5] Small refactor get_email.py --- helpdesk/management/commands/get_email.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/helpdesk/management/commands/get_email.py b/helpdesk/management/commands/get_email.py index 5ae13079..79a575b2 100755 --- a/helpdesk/management/commands/get_email.py +++ b/helpdesk/management/commands/get_email.py @@ -83,18 +83,17 @@ def process_email(quiet=False): allow_email_submission=True): logger = logging.getLogger('django.helpdesk.queue.' + q.slug) - if not q.logging_type or q.logging_type == 'none': + logging_types = { + 'info': logging.INFO, + 'warn': logging.WARN, + 'error': logging.ERROR, + 'crit': logging.CRITICAL, + 'debug': logging.DEBUG, + } + if q.logging_type in logging_types: + logger.setLevel(logging_types[q.logging_type]) + elif not q.logging_type or q.logging_type == 'none': logging.disable(logging.CRITICAL) # disable all messages - elif q.logging_type == 'info': - logger.setLevel(logging.INFO) - elif q.logging_type == 'warn': - logger.setLevel(logging.WARN) - elif q.logging_type == 'error': - logger.setLevel(logging.ERROR) - elif q.logging_type == 'crit': - logger.setLevel(logging.CRITICAL) - elif q.logging_type == 'debug': - logger.setLevel(logging.DEBUG) if quiet: logger.propagate = False # do not propagate to root logger that would log to console logdir = q.logging_dir or '/var/log/helpdesk/' From 9a45d28c959881ef8aa99467610a6248dc52feb8 Mon Sep 17 00:00:00 2001 From: Timothy Hobbs Date: Wed, 24 Oct 2018 18:20:12 +0200 Subject: [PATCH 3/5] More refactors of get_email.py --- helpdesk/email.py | 509 +++++++++++++++++++++ helpdesk/management/commands/get_email.py | 513 +--------------------- helpdesk/tests/test_get_email.py | 36 +- 3 files changed, 528 insertions(+), 530 deletions(-) create mode 100644 helpdesk/email.py diff --git a/helpdesk/email.py b/helpdesk/email.py new file mode 100644 index 00000000..1b0df203 --- /dev/null +++ b/helpdesk/email.py @@ -0,0 +1,509 @@ +#!/usr/bin/python +""" +Django Helpdesk - A Django powered ticket tracker for small enterprise. + +(c) Copyright 2008 Jutda. Copyright 2018 Timothy Hobbs. All Rights Reserved. +See LICENSE for details. + +scripts/get_email.py - Designed to be run from cron, this script checks the + POP and IMAP boxes, or a local mailbox directory, + defined for the queues within a + helpdesk, creating tickets from the new messages (or + adding to existing tickets if needed) +""" + +from datetime import timedelta +import base64 +import binascii +import email +import imaplib +import mimetypes +from os import listdir, unlink +from os.path import isfile, join +import poplib +import re +import socket +import ssl +import sys +from time import ctime + +from bs4 import BeautifulSoup + +from email_reply_parser import EmailReplyParser + +from django.core.files.base import ContentFile +from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.management.base import BaseCommand +from django.db.models import Q +from django.utils.translation import ugettext as _ +from django.utils import encoding, timezone + +from helpdesk import settings +from helpdesk.lib import send_templated_mail, safe_template_context, process_attachments +from helpdesk.models import Queue, Ticket, TicketCC, FollowUp, IgnoreEmail +from django.contrib.auth.models import User + +import logging + + +STRIPPED_SUBJECT_STRINGS = [ + "Re: ", + "Fw: ", + "RE: ", + "FW: ", + "Automatic reply: ", +] + +def process_email(quiet=False): + for q in Queue.objects.filter( + email_box_type__isnull=False, + allow_email_submission=True): + + logger = logging.getLogger('django.helpdesk.queue.' + q.slug) + logging_types = { + 'info': logging.INFO, + 'warn': logging.WARN, + 'error': logging.ERROR, + 'crit': logging.CRITICAL, + 'debug': logging.DEBUG, + } + if q.logging_type in logging_types: + logger.setLevel(logging_types[q.logging_type]) + elif not q.logging_type or q.logging_type == 'none': + logging.disable(logging.CRITICAL) # disable all messages + if quiet: + logger.propagate = False # do not propagate to root logger that would log to console + logdir = q.logging_dir or '/var/log/helpdesk/' + handler = logging.FileHandler(join(logdir, q.slug + '_get_email.log')) + logger.addHandler(handler) + + if not q.email_box_last_check: + q.email_box_last_check = timezone.now() - timedelta(minutes=30) + + queue_time_delta = timedelta(minutes=q.email_box_interval or 0) + + if (q.email_box_last_check + queue_time_delta) < timezone.now(): + process_queue(q, logger=logger) + q.email_box_last_check = timezone.now() + q.save() + + +def pop3_sync(q, logger, server): + server.getwelcome() + server.user(q.email_box_user or settings.QUEUE_EMAIL_BOX_USER) + server.pass_(q.email_box_pass or settings.QUEUE_EMAIL_BOX_PASSWORD) + + messagesInfo = server.list()[1] + logger.info("Received %d messages from POP3 server" % len(messagesInfo)) + + for msgRaw in messagesInfo: + if type(msgRaw) is bytes: + try: + msg = msgRaw.decode("utf-8") + except UnicodeError: + # if couldn't decode easily, just leave it raw + msg = msgRaw + else: + # already a str + msg = msgRaw + msgNum = msg.split(" ")[0] + logger.info("Processing message %s" % msgNum) + + raw_content = server.retr(msgNum)[1] + if type(raw_content[0]) is bytes: + full_message = "\n".join([elm.decode('utf-8') for elm in raw_content]) + else: + full_message = encoding.force_text("\n".join(raw_content), errors='replace') + ticket = ticket_from_message(message=full_message, queue=q, logger=logger) + + if ticket: + server.dele(msgNum) + logger.info("Successfully processed message %s, deleted from POP3 server" % msgNum) + else: + logger.warn("Message %s was not successfully processed, and will be left on POP3 server" % msgNum) + + server.quit() + + +def imap_sync(q, logger, server): + try: + server.login(q.email_box_user or + settings.QUEUE_EMAIL_BOX_USER, + q.email_box_pass or + settings.QUEUE_EMAIL_BOX_PASSWORD) + server.select(q.email_box_imap_folder) + except imaplib.IMAP4.abort: + logger.error("IMAP login failed. Check that the server is accessible and that the username and password are correct.") + server.logout() + sys.exit() + except ssl.SSLError: + logger.error("IMAP login failed due to SSL error. This is often due to a timeout. Please check your connection and try again.") + server.logout() + sys.exit() + + try: + status, data = server.search(None, 'NOT', 'DELETED') + except imaplib.IMAP4.error: + logger.error("IMAP retrieve failed. Is the folder '%s' spelled correctly, and does it exist on the server?" % q.email_box_imap_folder) + if data: + msgnums = data[0].split() + logger.info("Received %d messages from IMAP server" % len(msgnums)) + for num in msgnums: + logger.info("Processing message %s" % num) + status, data = server.fetch(num, '(RFC822)') + full_message = encoding.force_text(data[0][1], errors='replace') + try: + ticket = ticket_from_message(message=full_message, queue=q, logger=logger) + except TypeError: + ticket = None # hotfix. Need to work out WHY. + if ticket: + server.store(num, '+FLAGS', '\\Deleted') + logger.info("Successfully processed message %s, deleted from IMAP server" % num) + else: + logger.warn("Message %s was not successfully processed, and will be left on IMAP server" % num) + + server.expunge() + server.close() + server.logout() + + +def process_queue(q, logger): + logger.info("***** %s: Begin processing mail for django-helpdesk" % ctime()) + + if q.socks_proxy_type and q.socks_proxy_host and q.socks_proxy_port: + try: + import socks + except ImportError: + no_socks_msg = "Queue has been configured with proxy settings, " \ + "but no socks library was installed. Try to " \ + "install PySocks via PyPI." + logger.error(no_socks_msg) + raise ImportError(no_socks_msg) + + proxy_type = { + 'socks4': socks.SOCKS4, + 'socks5': socks.SOCKS5, + }.get(q.socks_proxy_type) + + socks.set_default_proxy(proxy_type=proxy_type, + addr=q.socks_proxy_host, + port=q.socks_proxy_port) + socket.socket = socks.socksocket + + email_box_type = settings.QUEUE_EMAIL_BOX_TYPE or q.email_box_type + + mail_defaults = { + 'pop3': { + 'ssl': { + 'port': 995, + 'init': poplib.POP3_SSL, + }, + 'insecure': { + 'port': 110, + 'init': poplib.POP3, + }, + 'sync': pop3_sync, + }, + 'imap': { + 'ssl': { + 'port': 993, + 'init': imaplib.IMAP4_SSL, + }, + 'insecure': { + 'port': 143, + 'init': imaplib.IMAP4, + }, + 'sync': imap_sync + } + } + if email_box_type in mail_defaults: + encryption = 'insecure' + if q.email_box_ssl or settings.QUEUE_EMAIL_BOX_SSL: + encryption = 'ssl' + if not q.email_box_port: + q.email_box_port = mail_defaults[email_box_type][encryption]['port'] + + server = mail_defaults[email_box_type][encryption]['init']( + q.email_box_host or settings.QUEUE_EMAIL_BOX_HOST, + int(q.email_box_port) + ) + logger.info("Attempting %s server login" % email_box_type.upper()) + mail_defaults[email_box_type]['sync'](q, logger, server) + + elif email_box_type == 'local': + mail_dir = q.email_box_local_dir or '/var/lib/mail/helpdesk/' + mail = [join(mail_dir, f) for f in listdir(mail_dir) if isfile(join(mail_dir, f))] + logger.info("Found %d messages in local mailbox directory" % len(mail)) + + logger.info("Found %d messages in local mailbox directory" % len(mail)) + for i, m in enumerate(mail, 1): + logger.info("Processing message %d" % i) + with open(m, 'r') as f: + full_message = encoding.force_text(f.read(), errors='replace') + ticket = ticket_from_message(message=full_message, queue=q, logger=logger) + if ticket: + logger.info("Successfully processed message %d, ticket/comment created." % i) + try: + unlink(m) # delete message file if ticket was successful + except OSError: + logger.error("Unable to delete message %d." % i) + else: + logger.info("Successfully deleted message %d." % i) + else: + logger.warn("Message %d was not successfully processed, and will be left in local directory" % i) + + +def decodeUnknown(charset, string): + if type(string) is not str: + if not charset: + try: + return str(string, encoding='utf-8', errors='replace') + except UnicodeError: + return str(string, encoding='iso8859-1', errors='replace') + return str(string, encoding=charset, errors='replace') + return string + + +def decode_mail_headers(string): + decoded = email.header.decode_header(string) + return u' '.join([str(msg, encoding=charset, errors='replace') if charset else str(msg) for msg, charset in decoded]) + + +def ticket_from_message(message, queue, logger): + # 'message' must be an RFC822 formatted message. + message = email.message_from_string(message) + subject = message.get('subject', _('Comment from e-mail')) + subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject)) + for affix in STRIPPED_SUBJECT_STRINGS: + subject = subject.replace(affix, "") + subject = subject.strip() + + sender = message.get('from', _('Unknown Sender')) + sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender)) + sender_email = email.utils.parseaddr(sender)[1] + + cc = message.get_all('cc', None) + if cc: + # first, fixup the encoding if necessary + cc = [decode_mail_headers(decodeUnknown(message.get_charset(), x)) for x in cc] + # get_all checks if multiple CC headers, but individual emails may be comma separated too + tempcc = [] + for hdr in cc: + tempcc.extend(hdr.split(',')) + # use a set to ensure no duplicates + cc = set([x.strip() for x in tempcc]) + + for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)): + if ignore.test(sender_email): + if ignore.keep_in_mailbox: + # By returning 'False' the message will be kept in the mailbox, + # and the 'True' will cause the message to be deleted. + return False + return True + + matchobj = re.match(r".*\[" + queue.slug + r"-(?P\d+)\]", subject) + if matchobj: + # This is a reply or forward. + ticket = matchobj.group('id') + logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket)) + else: + logger.info("No tracking ID matched.") + ticket = None + + body = None + counter = 0 + files = [] + + for part in message.walk(): + if part.get_content_maintype() == 'multipart': + continue + + name = part.get_param("name") + if name: + name = email.utils.collapse_rfc2231_value(name) + + if part.get_content_maintype() == 'text' and name is None: + if part.get_content_subtype() == 'plain': + body = EmailReplyParser.parse_reply( + decodeUnknown(part.get_content_charset(), part.get_payload(decode=True)) + ) + # workaround to get unicode text out rather than escaped text + try: + body = body.encode('ascii').decode('unicode_escape') + except UnicodeEncodeError: + body.encode('utf-8') + logger.debug("Discovered plain text MIME part") + else: + files.append( + SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html') + ) + logger.debug("Discovered HTML MIME part") + else: + if not name: + ext = mimetypes.guess_extension(part.get_content_type()) + name = "part-%i%s" % (counter, ext) + payload = part.get_payload() + if isinstance(payload, list): + payload = payload.pop().as_string() + payloadToWrite = payload + # check version of python to ensure use of only the correct error type + non_b64_err = TypeError + try: + logger.debug("Try to base64 decode the attachment payload") + payloadToWrite = base64.decodebytes(payload) + except non_b64_err: + logger.debug("Payload was not base64 encoded, using raw bytes") + payloadToWrite = payload + files.append(SimpleUploadedFile(name, part.get_payload(decode=True), mimetypes.guess_type(name)[0])) + logger.debug("Found MIME attachment %s" % name) + + counter += 1 + + if not body: + mail = BeautifulSoup(part.get_payload(), "lxml") + if ">" in mail.text: + body = mail.find('body') + body = body.text + body = body.encode('ascii', errors='ignore') + else: + body = mail.text + + if ticket: + try: + t = Ticket.objects.get(id=ticket) + except Ticket.DoesNotExist: + logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket)) + ticket = None + else: + logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id)) + if t.status == Ticket.CLOSED_STATUS: + t.status = Ticket.REOPENED_STATUS + t.save() + new = False + + smtp_priority = message.get('priority', '') + smtp_importance = message.get('importance', '') + high_priority_types = {'high', 'important', '1', 'urgent'} + priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3 + + if ticket is None: + if settings.QUEUE_EMAIL_BOX_UPDATE_ONLY: + return None + new = True + t = Ticket.objects.create( + title=subject, + queue=queue, + submitter_email=sender_email, + created=timezone.now(), + description=body, + priority=priority, + ) + logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id)) + + if cc: + # get list of currently CC'd emails + current_cc = TicketCC.objects.filter(ticket=ticket) + current_cc_emails = [x.email for x in current_cc if x.email] + # get emails of any Users CC'd to email, if defined + # (some Users may not have an associated email, e.g, when using LDAP) + current_cc_users = [x.user.email for x in current_cc if x.user and x.user.email] + # ensure submitter, assigned user, queue email not added + other_emails = [queue.email_address] + if t.submitter_email: + other_emails.append(t.submitter_email) + if t.assigned_to: + other_emails.append(t.assigned_to.email) + current_cc = set(current_cc_emails + current_cc_users + other_emails) + # first, add any User not previously CC'd (as identified by User's email) + all_users = User.objects.all() + all_user_emails = set([x.email for x in all_users]) + users_not_currently_ccd = all_user_emails.difference(set(current_cc)) + users_to_cc = cc.intersection(users_not_currently_ccd) + for user in users_to_cc: + tcc = TicketCC.objects.create( + ticket=t, + user=User.objects.get(email=user), + can_view=True, + can_update=False + ) + tcc.save() + # then add remaining emails alphabetically, makes testing easy + new_cc = cc.difference(current_cc).difference(all_user_emails) + new_cc = sorted(list(new_cc)) + for ccemail in new_cc: + tcc = TicketCC.objects.create( + ticket=t, + email=ccemail.replace('\n', ' ').replace('\r', ' '), + can_view=True, + can_update=False + ) + tcc.save() + + f = FollowUp( + ticket=t, + title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}), + date=timezone.now(), + public=True, + comment=body, + ) + + if t.status == Ticket.REOPENED_STATUS: + f.new_status = Ticket.REOPENED_STATUS + f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}) + + f.save() + logger.debug("Created new FollowUp for Ticket") + + logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)) + + attached = process_attachments(f, files) + for att_file in attached: + logger.info("Attachment '%s' (with size %s) successfully added to ticket from email." % (att_file[0], att_file[1].size)) + + context = safe_template_context(t) + + if new: + if sender_email: + send_templated_mail( + 'newticket_submitter', + context, + recipients=sender_email, + sender=queue.from_address, + fail_silently=True, + ) + if queue.new_ticket_cc: + send_templated_mail( + 'newticket_cc', + context, + recipients=queue.new_ticket_cc, + sender=queue.from_address, + fail_silently=True, + ) + if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc: + send_templated_mail( + 'newticket_cc', + context, + recipients=queue.updated_ticket_cc, + sender=queue.from_address, + fail_silently=True, + ) + else: + context.update(comment=f.comment) + if t.assigned_to: + send_templated_mail( + 'updated_owner', + context, + recipients=t.assigned_to.email, + sender=queue.from_address, + fail_silently=True, + ) + if queue.updated_ticket_cc: + send_templated_mail( + 'updated_cc', + context, + recipients=queue.updated_ticket_cc, + sender=queue.from_address, + fail_silently=True, + ) + + return t diff --git a/helpdesk/management/commands/get_email.py b/helpdesk/management/commands/get_email.py index 79a575b2..ce344bd5 100755 --- a/helpdesk/management/commands/get_email.py +++ b/helpdesk/management/commands/get_email.py @@ -10,49 +10,9 @@ scripts/get_email.py - Designed to be run from cron, this script checks the helpdesk, creating tickets from the new messages (or adding to existing tickets if needed) """ -from __future__ import unicode_literals - -from datetime import timedelta -import base64 -import binascii -import email -import imaplib -import mimetypes -from os import listdir, unlink -from os.path import isfile, join -import poplib -import re -import socket -import ssl -import sys -from time import ctime - -from bs4 import BeautifulSoup - -from email_reply_parser import EmailReplyParser - -from django.core.files.base import ContentFile -from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management.base import BaseCommand -from django.db.models import Q -from django.utils.translation import ugettext as _ -from django.utils import encoding, six, timezone -from helpdesk import settings -from helpdesk.lib import send_templated_mail, safe_template_context, process_attachments -from helpdesk.models import Queue, Ticket, TicketCC, FollowUp, IgnoreEmail -from django.contrib.auth.models import User - -import logging - - -STRIPPED_SUBJECT_STRINGS = [ - "Re: ", - "Fw: ", - "RE: ", - "FW: ", - "Automatic reply: ", -] +from helpdesk.email import process_email class Command(BaseCommand): @@ -77,476 +37,5 @@ class Command(BaseCommand): process_email(quiet=quiet) -def process_email(quiet=False): - for q in Queue.objects.filter( - email_box_type__isnull=False, - allow_email_submission=True): - - logger = logging.getLogger('django.helpdesk.queue.' + q.slug) - logging_types = { - 'info': logging.INFO, - 'warn': logging.WARN, - 'error': logging.ERROR, - 'crit': logging.CRITICAL, - 'debug': logging.DEBUG, - } - if q.logging_type in logging_types: - logger.setLevel(logging_types[q.logging_type]) - elif not q.logging_type or q.logging_type == 'none': - logging.disable(logging.CRITICAL) # disable all messages - if quiet: - logger.propagate = False # do not propagate to root logger that would log to console - logdir = q.logging_dir or '/var/log/helpdesk/' - handler = logging.FileHandler(join(logdir, q.slug + '_get_email.log')) - logger.addHandler(handler) - - if not q.email_box_last_check: - q.email_box_last_check = timezone.now() - timedelta(minutes=30) - - queue_time_delta = timedelta(minutes=q.email_box_interval or 0) - - if (q.email_box_last_check + queue_time_delta) < timezone.now(): - process_queue(q, logger=logger) - q.email_box_last_check = timezone.now() - q.save() - - -def process_queue(q, logger): - logger.info("***** %s: Begin processing mail for django-helpdesk" % ctime()) - - if q.socks_proxy_type and q.socks_proxy_host and q.socks_proxy_port: - try: - import socks - except ImportError: - no_socks_msg = "Queue has been configured with proxy settings, " \ - "but no socks library was installed. Try to " \ - "install PySocks via PyPI." - logger.error(no_socks_msg) - raise ImportError(no_socks_msg) - - proxy_type = { - 'socks4': socks.SOCKS4, - 'socks5': socks.SOCKS5, - }.get(q.socks_proxy_type) - - socks.set_default_proxy(proxy_type=proxy_type, - addr=q.socks_proxy_host, - port=q.socks_proxy_port) - socket.socket = socks.socksocket - elif six.PY2: - socket.socket = socket._socketobject - - email_box_type = settings.QUEUE_EMAIL_BOX_TYPE or q.email_box_type - - if email_box_type == 'pop3': - if q.email_box_ssl or settings.QUEUE_EMAIL_BOX_SSL: - if not q.email_box_port: - q.email_box_port = 995 - server = poplib.POP3_SSL(q.email_box_host or - settings.QUEUE_EMAIL_BOX_HOST, - int(q.email_box_port)) - else: - if not q.email_box_port: - q.email_box_port = 110 - server = poplib.POP3(q.email_box_host or - settings.QUEUE_EMAIL_BOX_HOST, - int(q.email_box_port)) - - logger.info("Attempting POP3 server login") - - server.getwelcome() - server.user(q.email_box_user or settings.QUEUE_EMAIL_BOX_USER) - server.pass_(q.email_box_pass or settings.QUEUE_EMAIL_BOX_PASSWORD) - - messagesInfo = server.list()[1] - logger.info("Received %d messages from POP3 server" % len(messagesInfo)) - - for msgRaw in messagesInfo: - if six.PY3 and type(msgRaw) is bytes: - # in py3, msgRaw may be a bytes object, decode to str - try: - msg = msgRaw.decode("utf-8") - except UnicodeError: - # if couldn't decode easily, just leave it raw - msg = msgRaw - else: - # already a str - msg = msgRaw - msgNum = msg.split(" ")[0] - logger.info("Processing message %s" % msgNum) - - if six.PY2: - full_message = encoding.force_text("\n".join(server.retr(msgNum)[1]), errors='replace') - else: - raw_content = server.retr(msgNum)[1] - if type(raw_content[0]) is bytes: - full_message = "\n".join([elm.decode('utf-8') for elm in raw_content]) - else: - full_message = encoding.force_text("\n".join(raw_content), errors='replace') - ticket = ticket_from_message(message=full_message, queue=q, logger=logger) - - if ticket: - server.dele(msgNum) - logger.info("Successfully processed message %s, deleted from POP3 server" % msgNum) - else: - logger.warn("Message %s was not successfully processed, and will be left on POP3 server" % msgNum) - - server.quit() - - elif email_box_type == 'imap': - if q.email_box_ssl or settings.QUEUE_EMAIL_BOX_SSL: - if not q.email_box_port: - q.email_box_port = 993 - server = imaplib.IMAP4_SSL(q.email_box_host or - settings.QUEUE_EMAIL_BOX_HOST, - int(q.email_box_port)) - else: - if not q.email_box_port: - q.email_box_port = 143 - server = imaplib.IMAP4(q.email_box_host or - settings.QUEUE_EMAIL_BOX_HOST, - int(q.email_box_port)) - - logger.info("Attempting IMAP server login") - - try: - server.login(q.email_box_user or - settings.QUEUE_EMAIL_BOX_USER, - q.email_box_pass or - settings.QUEUE_EMAIL_BOX_PASSWORD) - server.select(q.email_box_imap_folder) - except imaplib.IMAP4.abort: - logger.error("IMAP login failed. Check that the server is accessible and that the username and password are correct.") - server.logout() - sys.exit() - except ssl.SSLError: - logger.error("IMAP login failed due to SSL error. This is often due to a timeout. Please check your connection and try again.") - server.logout() - sys.exit() - - try: - status, data = server.search(None, 'NOT', 'DELETED') - except imaplib.IMAP4.error: - logger.error("IMAP retrieve failed. Is the folder '%s' spelled correctly, and does it exist on the server?" % q.email_box_imap_folder) - if data: - msgnums = data[0].split() - logger.info("Received %d messages from IMAP server" % len(msgnums)) - for num in msgnums: - logger.info("Processing message %s" % num) - status, data = server.fetch(num, '(RFC822)') - full_message = encoding.force_text(data[0][1], errors='replace') - try: - ticket = ticket_from_message(message=full_message, queue=q, logger=logger) - except TypeError: - ticket = None # hotfix. Need to work out WHY. - if ticket: - server.store(num, '+FLAGS', '\\Deleted') - logger.info("Successfully processed message %s, deleted from IMAP server" % num) - else: - logger.warn("Message %s was not successfully processed, and will be left on IMAP server" % num) - - server.expunge() - server.close() - server.logout() - - elif email_box_type == 'local': - mail_dir = q.email_box_local_dir or '/var/lib/mail/helpdesk/' - mail = [join(mail_dir, f) for f in listdir(mail_dir) if isfile(join(mail_dir, f))] - logger.info("Found %d messages in local mailbox directory" % len(mail)) - - logger.info("Found %d messages in local mailbox directory" % len(mail)) - for i, m in enumerate(mail, 1): - logger.info("Processing message %d" % i) - with open(m, 'r') as f: - full_message = encoding.force_text(f.read(), errors='replace') - ticket = ticket_from_message(message=full_message, queue=q, logger=logger) - if ticket: - logger.info("Successfully processed message %d, ticket/comment created." % i) - try: - unlink(m) # delete message file if ticket was successful - except OSError: - logger.error("Unable to delete message %d." % i) - else: - logger.info("Successfully deleted message %d." % i) - else: - logger.warn("Message %d was not successfully processed, and will be left in local directory" % i) - - -def decodeUnknown(charset, string): - if six.PY2: - if not charset: - try: - return string.decode('utf-8', 'replace') - except UnicodeError: - return string.decode('iso8859-1', 'replace') - return unicode(string, charset) - elif six.PY3: - if type(string) is not str: - if not charset: - try: - return str(string, encoding='utf-8', errors='replace') - except UnicodeError: - return str(string, encoding='iso8859-1', errors='replace') - return str(string, encoding=charset, errors='replace') - return string - - -def decode_mail_headers(string): - decoded = email.header.decode_header(string) if six.PY3 else email.header.decode_header(string.encode('utf-8')) - if six.PY2: - return u' '.join([unicode(msg, charset or 'utf-8') for msg, charset in decoded]) - elif six.PY3: - return u' '.join([str(msg, encoding=charset, errors='replace') if charset else str(msg) for msg, charset in decoded]) - - -def ticket_from_message(message, queue, logger): - # 'message' must be an RFC822 formatted message. - message = email.message_from_string(message) if six.PY3 else email.message_from_string(message.encode('utf-8')) - subject = message.get('subject', _('Comment from e-mail')) - subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject)) - for affix in STRIPPED_SUBJECT_STRINGS: - subject = subject.replace(affix, "") - subject = subject.strip() - - sender = message.get('from', _('Unknown Sender')) - sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender)) - sender_email = email.utils.parseaddr(sender)[1] - - cc = message.get_all('cc', None) - if cc: - # first, fixup the encoding if necessary - cc = [decode_mail_headers(decodeUnknown(message.get_charset(), x)) for x in cc] - # get_all checks if multiple CC headers, but individual emails may be comma separated too - tempcc = [] - for hdr in cc: - tempcc.extend(hdr.split(',')) - # use a set to ensure no duplicates - cc = set([x.strip() for x in tempcc]) - - for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)): - if ignore.test(sender_email): - if ignore.keep_in_mailbox: - # By returning 'False' the message will be kept in the mailbox, - # and the 'True' will cause the message to be deleted. - return False - return True - - matchobj = re.match(r".*\[" + queue.slug + r"-(?P\d+)\]", subject) - if matchobj: - # This is a reply or forward. - ticket = matchobj.group('id') - logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket)) - else: - logger.info("No tracking ID matched.") - ticket = None - - body = None - counter = 0 - files = [] - - for part in message.walk(): - if part.get_content_maintype() == 'multipart': - continue - - name = part.get_param("name") - if name: - name = email.utils.collapse_rfc2231_value(name) - - if part.get_content_maintype() == 'text' and name is None: - if part.get_content_subtype() == 'plain': - body = EmailReplyParser.parse_reply( - decodeUnknown(part.get_content_charset(), part.get_payload(decode=True)) - ) - # workaround to get unicode text out rather than escaped text - try: - body = body.encode('ascii').decode('unicode_escape') - except UnicodeEncodeError: - body.encode('utf-8') - logger.debug("Discovered plain text MIME part") - else: - files.append( - SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html') - ) - logger.debug("Discovered HTML MIME part") - else: - if not name: - ext = mimetypes.guess_extension(part.get_content_type()) - name = "part-%i%s" % (counter, ext) - payload = part.get_payload() - if isinstance(payload, list): - payload = payload.pop().as_string() - payloadToWrite = payload - # check version of python to ensure use of only the correct error type - if six.PY2: - non_b64_err = binascii.Error - else: - non_b64_err = TypeError - try: - logger.debug("Try to base64 decode the attachment payload") - if six.PY2: - payloadToWrite = base64.decodestring(payload) - else: - payloadToWrite = base64.decodebytes(payload) - except non_b64_err: - logger.debug("Payload was not base64 encoded, using raw bytes") - payloadToWrite = payload - files.append(SimpleUploadedFile(name, part.get_payload(decode=True), mimetypes.guess_type(name)[0])) - logger.debug("Found MIME attachment %s" % name) - - counter += 1 - - if not body: - mail = BeautifulSoup(part.get_payload(), "lxml") - if ">" in mail.text: - body = mail.find('body') - body = body.text - body = body.encode('ascii', errors='ignore') - else: - body = mail.text - - if ticket: - try: - t = Ticket.objects.get(id=ticket) - except Ticket.DoesNotExist: - logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket)) - ticket = None - else: - logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id)) - if t.status == Ticket.CLOSED_STATUS: - t.status = Ticket.REOPENED_STATUS - t.save() - new = False - - smtp_priority = message.get('priority', '') - smtp_importance = message.get('importance', '') - high_priority_types = {'high', 'important', '1', 'urgent'} - priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3 - - if ticket is None: - if settings.QUEUE_EMAIL_BOX_UPDATE_ONLY: - return None - new = True - t = Ticket.objects.create( - title=subject, - queue=queue, - submitter_email=sender_email, - created=timezone.now(), - description=body, - priority=priority, - ) - logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id)) - - if cc: - # get list of currently CC'd emails - current_cc = TicketCC.objects.filter(ticket=ticket) - current_cc_emails = [x.email for x in current_cc if x.email] - # get emails of any Users CC'd to email, if defined - # (some Users may not have an associated email, e.g, when using LDAP) - current_cc_users = [x.user.email for x in current_cc if x.user and x.user.email] - # ensure submitter, assigned user, queue email not added - other_emails = [queue.email_address] - if t.submitter_email: - other_emails.append(t.submitter_email) - if t.assigned_to: - other_emails.append(t.assigned_to.email) - current_cc = set(current_cc_emails + current_cc_users + other_emails) - # first, add any User not previously CC'd (as identified by User's email) - all_users = User.objects.all() - all_user_emails = set([x.email for x in all_users]) - users_not_currently_ccd = all_user_emails.difference(set(current_cc)) - users_to_cc = cc.intersection(users_not_currently_ccd) - for user in users_to_cc: - tcc = TicketCC.objects.create( - ticket=t, - user=User.objects.get(email=user), - can_view=True, - can_update=False - ) - tcc.save() - # then add remaining emails alphabetically, makes testing easy - new_cc = cc.difference(current_cc).difference(all_user_emails) - new_cc = sorted(list(new_cc)) - for ccemail in new_cc: - tcc = TicketCC.objects.create( - ticket=t, - email=ccemail.replace('\n', ' ').replace('\r', ' '), - can_view=True, - can_update=False - ) - tcc.save() - - f = FollowUp( - ticket=t, - title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}), - date=timezone.now(), - public=True, - comment=body, - ) - - if t.status == Ticket.REOPENED_STATUS: - f.new_status = Ticket.REOPENED_STATUS - f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}) - - f.save() - logger.debug("Created new FollowUp for Ticket") - - if six.PY2: - logger.info(("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace')) - elif six.PY3: - logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)) - - attached = process_attachments(f, files) - for att_file in attached: - logger.info("Attachment '%s' (with size %s) successfully added to ticket from email." % (att_file[0], att_file[1].size)) - - context = safe_template_context(t) - - if new: - if sender_email: - send_templated_mail( - 'newticket_submitter', - context, - recipients=sender_email, - sender=queue.from_address, - fail_silently=True, - ) - if queue.new_ticket_cc: - send_templated_mail( - 'newticket_cc', - context, - recipients=queue.new_ticket_cc, - sender=queue.from_address, - fail_silently=True, - ) - if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc: - send_templated_mail( - 'newticket_cc', - context, - recipients=queue.updated_ticket_cc, - sender=queue.from_address, - fail_silently=True, - ) - else: - context.update(comment=f.comment) - if t.assigned_to: - send_templated_mail( - 'updated_owner', - context, - recipients=t.assigned_to.email, - sender=queue.from_address, - fail_silently=True, - ) - if queue.updated_ticket_cc: - send_templated_mail( - 'updated_cc', - context, - recipients=queue.updated_ticket_cc, - sender=queue.from_address, - fail_silently=True, - ) - - return t - - if __name__ == '__main__': process_email() diff --git a/helpdesk/tests/test_get_email.py b/helpdesk/tests/test_get_email.py index 583eaddd..fb97708c 100644 --- a/helpdesk/tests/test_get_email.py +++ b/helpdesk/tests/test_get_email.py @@ -98,8 +98,8 @@ class GetEmailParametricTemplate(object): else: # Test local email reading if self.method == 'local': - with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \ - mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \ + with mock.patch('helpdesk.email.listdir') as mocked_listdir, \ + mock.patch('helpdesk.email.isfile') as mocked_isfile, \ mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)): mocked_isfile.return_value = True mocked_listdir.return_value = ['filename1', 'filename2'] @@ -120,7 +120,7 @@ class GetEmailParametricTemplate(object): mocked_poplib_server = mock.Mock() mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list) mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib: + with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib: mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server) call_command('get_email') @@ -136,7 +136,7 @@ class GetEmailParametricTemplate(object): # we ignore the second arg as the data item/mime-part is constant (RFC822) mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib: + with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib: mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server) call_command('get_email') @@ -171,8 +171,8 @@ class GetEmailParametricTemplate(object): else: # Test local email reading if self.method == 'local': - with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \ - mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \ + with mock.patch('helpdesk.email.listdir') as mocked_listdir, \ + mock.patch('helpdesk.email.isfile') as mocked_isfile, \ mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)): mocked_isfile.return_value = True mocked_listdir.return_value = ['filename1', 'filename2'] @@ -193,7 +193,7 @@ class GetEmailParametricTemplate(object): mocked_poplib_server = mock.Mock() mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list) mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib: + with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib: mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server) call_command('get_email') @@ -209,7 +209,7 @@ class GetEmailParametricTemplate(object): # we ignore the second arg as the data item/mime-part is constant (RFC822) mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib: + with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib: mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server) call_command('get_email') @@ -284,8 +284,8 @@ class GetEmailParametricTemplate(object): else: # Test local email reading if self.method == 'local': - with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \ - mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \ + with mock.patch('helpdesk.email.listdir') as mocked_listdir, \ + mock.patch('helpdesk.email.isfile') as mocked_isfile, \ mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=msg.as_string())): mocked_isfile.return_value = True mocked_listdir.return_value = ['filename1', 'filename2'] @@ -306,7 +306,7 @@ class GetEmailParametricTemplate(object): mocked_poplib_server = mock.Mock() mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list) mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib: + with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib: mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server) call_command('get_email') @@ -322,7 +322,7 @@ class GetEmailParametricTemplate(object): # we ignore the second arg as the data item/mime-part is constant (RFC822) mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib: + with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib: mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server) call_command('get_email') @@ -531,8 +531,8 @@ a9eiiQ+3V1v+7wWHXCzq else: # Test local email reading if self.method == 'local': - with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \ - mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \ + with mock.patch('helpdesk.email.listdir') as mocked_listdir, \ + mock.patch('helpdesk.email.isfile') as mocked_isfile, \ mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)): mocked_isfile.return_value = True mocked_listdir.return_value = ['filename1'] @@ -551,7 +551,7 @@ a9eiiQ+3V1v+7wWHXCzq mocked_poplib_server = mock.Mock() mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list) mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails['1']) - with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib: + with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib: mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server) call_command('get_email') @@ -566,7 +566,7 @@ a9eiiQ+3V1v+7wWHXCzq # we ignore the second arg as the data item/mime-part is constant (RFC822) mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib: + with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib: mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server) call_command('get_email') @@ -701,8 +701,8 @@ class GetEmailCCHandling(TestCase): test_email = "To: queue@example.com\nCc: " + test_email_cc_one + ", " + test_email_cc_one + ", " + test_email_cc_two + ", " + test_email_cc_three + "\nCC: " + test_email_cc_one + ", " + test_email_cc_three + ", " + test_email_cc_four + ", " + ticket_user_emails + "\nFrom: " + test_email_from + "\nSubject: " + test_email_subject + "\n\n" + test_email_body test_mail_len = len(test_email) - with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \ - mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \ + with mock.patch('helpdesk.email.listdir') as mocked_listdir, \ + mock.patch('helpdesk.email.isfile') as mocked_isfile, \ mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)): mocked_isfile.return_value = True From 6c37d73d4e95ce8dea55b3966084454288cf98f3 Mon Sep 17 00:00:00 2001 From: Timothy Hobbs Date: Wed, 31 Oct 2018 16:24:57 +0100 Subject: [PATCH 4/5] DRY out email sending code and normalize behavior This refactor removes duplicated logic for deciding whom the messages get sent to. It also normalizes behavior ensuring that all CCed addresses are sent to in all cases that CCed individuals should be notified. --- helpdesk/email.py | 54 ++---- helpdesk/forms.py | 62 ++----- helpdesk/lib.py | 124 +------------ .../management/commands/escalate_tickets.py | 34 +--- helpdesk/models.py | 50 +++++ helpdesk/templated_email.py | 114 ++++++++++++ helpdesk/views/staff.py | 172 ++++++------------ 7 files changed, 249 insertions(+), 361 deletions(-) create mode 100644 helpdesk/templated_email.py diff --git a/helpdesk/email.py b/helpdesk/email.py index 1b0df203..3a95d198 100644 --- a/helpdesk/email.py +++ b/helpdesk/email.py @@ -39,7 +39,7 @@ from django.utils.translation import ugettext as _ from django.utils import encoding, timezone from helpdesk import settings -from helpdesk.lib import send_templated_mail, safe_template_context, process_attachments +from helpdesk.lib import safe_template_context, process_attachments from helpdesk.models import Queue, Ticket, TicketCC, FollowUp, IgnoreEmail from django.contrib.auth.models import User @@ -54,6 +54,7 @@ STRIPPED_SUBJECT_STRINGS = [ "Automatic reply: ", ] + def process_email(quiet=False): for q in Queue.objects.filter( email_box_type__isnull=False, @@ -463,47 +464,18 @@ def ticket_from_message(message, queue, logger): context = safe_template_context(t) if new: - if sender_email: - send_templated_mail( - 'newticket_submitter', - context, - recipients=sender_email, - sender=queue.from_address, - fail_silently=True, - ) - if queue.new_ticket_cc: - send_templated_mail( - 'newticket_cc', - context, - recipients=queue.new_ticket_cc, - sender=queue.from_address, - fail_silently=True, - ) - if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc: - send_templated_mail( - 'newticket_cc', - context, - recipients=queue.updated_ticket_cc, - sender=queue.from_address, - fail_silently=True, - ) + t.send( + {'submitter': ('newticket_submitter', context), + 'new_ticket_cc': ('newticket_cc', context), + 'ticket_cc': ('newticket_cc', context)}, + fail_silently=True, + ) else: context.update(comment=f.comment) - if t.assigned_to: - send_templated_mail( - 'updated_owner', - context, - recipients=t.assigned_to.email, - sender=queue.from_address, - fail_silently=True, - ) - if queue.updated_ticket_cc: - send_templated_mail( - 'updated_cc', - context, - recipients=queue.updated_ticket_cc, - sender=queue.from_address, - fail_silently=True, - ) + t.send( + {'assigned_to': ('updated_owner', context), + 'ticket_cc': ('updated_cc', context)}, + fail_silently=True, + ) return t diff --git a/helpdesk/forms.py b/helpdesk/forms.py index 4192e3e9..777587ea 100644 --- a/helpdesk/forms.py +++ b/helpdesk/forms.py @@ -17,7 +17,7 @@ from django.utils.translation import ugettext_lazy as _ from django.contrib.auth import get_user_model from django.utils import timezone -from helpdesk.lib import send_templated_mail, safe_template_context, process_attachments +from helpdesk.lib import safe_template_context, process_attachments from helpdesk.models import (Ticket, Queue, FollowUp, Attachment, IgnoreEmail, TicketCC, CustomField, TicketCustomFieldValue, TicketDependency, UserSettings) from helpdesk import settings as helpdesk_settings @@ -238,56 +238,16 @@ class AbstractTicketForm(CustomFieldMixin, forms.Form): context = safe_template_context(ticket) context['comment'] = followup.comment - messages_sent_to = [] - - if ticket.submitter_email: - send_templated_mail( - 'newticket_submitter', - context, - recipients=ticket.submitter_email, - sender=queue.from_address, - fail_silently=True, - files=files, - ) - messages_sent_to.append(ticket.submitter_email) - - if ticket.assigned_to and \ - ticket.assigned_to != user and \ - ticket.assigned_to.usersettings_helpdesk.email_on_ticket_assign and \ - ticket.assigned_to.email and \ - ticket.assigned_to.email not in messages_sent_to: - send_templated_mail( - 'assigned_owner', - context, - recipients=ticket.assigned_to.email, - sender=queue.from_address, - fail_silently=True, - files=files, - ) - messages_sent_to.append(ticket.assigned_to.email) - - if queue.new_ticket_cc and queue.new_ticket_cc not in messages_sent_to: - send_templated_mail( - 'newticket_cc', - context, - recipients=queue.new_ticket_cc, - sender=queue.from_address, - fail_silently=True, - files=files, - ) - messages_sent_to.append(queue.new_ticket_cc) - - if queue.updated_ticket_cc and \ - queue.updated_ticket_cc != queue.new_ticket_cc and \ - queue.updated_ticket_cc not in messages_sent_to: - send_templated_mail( - 'newticket_cc', - context, - recipients=queue.updated_ticket_cc, - sender=queue.from_address, - fail_silently=True, - files=files, - ) + roles = {'submitter': ('newticket_submitter', context), + 'new_ticket_cc': ('newticket_cc', context), + 'ticket_cc': ('newticket_cc', context)} + if ticket.assigned_to and ticket.assigned_to.usersettings_helpdesk.email_on_ticket_assign: + roles['assigned_to'] = ('assigned_owner', context) + ticket.send( + roles, + fail_silently=True, + files=files, + ) class TicketForm(AbstractTicketForm): diff --git a/helpdesk/lib.py b/helpdesk/lib.py index 0692213a..b5564b98 100644 --- a/helpdesk/lib.py +++ b/helpdesk/lib.py @@ -9,142 +9,22 @@ lib.py - Common functions (eg multipart e-mail) import logging import mimetypes import os -from smtplib import SMTPException from django.conf import settings from django.db.models import Q -from django.utils import six from django.utils.encoding import smart_text from django.utils.safestring import mark_safe from helpdesk.models import Attachment, EmailTemplate -import six - from model_utils import Choices -if six.PY3: - from base64 import encodebytes as b64encode - from base64 import decodebytes as b64decode -else: - from base64 import urlsafe_b64encode as b64encode - from base64 import urlsafe_b64decode as b64decode +from base64 import encodebytes as b64encode +from base64 import decodebytes as b64decode logger = logging.getLogger('helpdesk') -def send_templated_mail(template_name, - context, - recipients, - sender=None, - bcc=None, - fail_silently=False, - files=None): - """ - send_templated_mail() is a wrapper around Django's e-mail routines that - allows us to easily send multipart (text/plain & text/html) e-mails using - templates that are stored in the database. This lets the admin provide - both a text and a HTML template for each message. - - template_name is the slug of the template to use for this message (see - models.EmailTemplate) - - context is a dictionary to be used when rendering the template - - recipients can be either a string, eg 'a@b.com', or a list of strings. - - sender should contain a string, eg 'My Site '. If you leave it - blank, it'll use settings.DEFAULT_FROM_EMAIL as a fallback. - - bcc is an optional list of addresses that will receive this message as a - blind carbon copy. - - fail_silently is passed to Django's mail routine. Set to 'True' to ignore - any errors at send time. - - files can be a list of tuples. Each tuple should be a filename to attach, - along with the File objects to be read. files can be blank. - - """ - from django.core.mail import EmailMultiAlternatives - from django.template import engines - from_string = engines['django'].from_string - - from helpdesk.models import EmailTemplate - from helpdesk.settings import HELPDESK_EMAIL_SUBJECT_TEMPLATE, \ - HELPDESK_EMAIL_FALLBACK_LOCALE - - locale = context['queue'].get('locale') or HELPDESK_EMAIL_FALLBACK_LOCALE - - try: - t = EmailTemplate.objects.get(template_name__iexact=template_name, locale=locale) - except EmailTemplate.DoesNotExist: - try: - t = EmailTemplate.objects.get(template_name__iexact=template_name, locale__isnull=True) - except EmailTemplate.DoesNotExist: - logger.warning('template "%s" does not exist, no mail sent', template_name) - return # just ignore if template doesn't exist - - subject_part = from_string( - HELPDESK_EMAIL_SUBJECT_TEMPLATE % { - "subject": t.subject - }).render(context).replace('\n', '').replace('\r', '') - - footer_file = os.path.join('helpdesk', locale, 'email_text_footer.txt') - - text_part = from_string( - "%s{%% include '%s' %%}" % (t.plain_text, footer_file) - ).render(context) - - email_html_base_file = os.path.join('helpdesk', locale, 'email_html_base.html') - # keep new lines in html emails - if 'comment' in context: - context['comment'] = mark_safe(context['comment'].replace('\r\n', '
')) - - html_part = from_string( - "{%% extends '%s' %%}{%% block title %%}" - "%s" - "{%% endblock %%}{%% block content %%}%s{%% endblock %%}" % - (email_html_base_file, t.heading, t.html) - ).render(context) - - if isinstance(recipients, str): - if recipients.find(','): - recipients = recipients.split(',') - elif type(recipients) != list: - recipients = [recipients] - - msg = EmailMultiAlternatives(subject_part, text_part, - sender or settings.DEFAULT_FROM_EMAIL, - recipients, bcc=bcc) - msg.attach_alternative(html_part, "text/html") - - if files: - for filename, filefield in files: - mime = mimetypes.guess_type(filename) - if mime[0] is not None and mime[0] == "text/plain": - with open(filefield.path, 'r') as attachedfile: - content = attachedfile.read() - msg.attach(filename, content) - else: - if six.PY3: - msg.attach_file(filefield.path) - else: - with open(filefield.path, 'rb') as attachedfile: - content = attachedfile.read() - msg.attach(filename, content) - - logger.debug('Sending email to: {!r}'.format(recipients)) - - try: - return msg.send() - except SMTPException as e: - logger.exception('SMTPException raised while sending email to {}'.format(recipients)) - if not fail_silently: - raise e - return 0 - - def query_to_dict(results, descriptions): """ Replacement method for cursor.dictfetchall() as that method no longer diff --git a/helpdesk/management/commands/escalate_tickets.py b/helpdesk/management/commands/escalate_tickets.py index b2788762..88d26e31 100644 --- a/helpdesk/management/commands/escalate_tickets.py +++ b/helpdesk/management/commands/escalate_tickets.py @@ -24,7 +24,7 @@ except ImportError: from datetime import datetime as timezone from helpdesk.models import Queue, Ticket, FollowUp, EscalationExclusion, TicketChange -from helpdesk.lib import send_templated_mail, safe_template_context +from helpdesk.lib import safe_template_context class Command(BaseCommand): @@ -107,32 +107,12 @@ def escalate_tickets(queues, verbose): context = safe_template_context(t) - if t.submitter_email: - send_templated_mail( - 'escalated_submitter', - context, - recipients=t.submitter_email, - sender=t.queue.from_address, - fail_silently=True, - ) - - if t.queue.updated_ticket_cc: - send_templated_mail( - 'escalated_cc', - context, - recipients=t.queue.updated_ticket_cc, - sender=t.queue.from_address, - fail_silently=True, - ) - - if t.assigned_to: - send_templated_mail( - 'escalated_owner', - context, - recipients=t.assigned_to.email, - sender=t.queue.from_address, - fail_silently=True, - ) + t.send( + {'submitter': ('escalated_submitter', context), + 'ticket_cc': ('escalated_cc', context), + 'assigned_to': ('escalated_owner', context)} + fail_silently=True, + ) if verbose: print(" - Esclating %s from %s>%s" % ( diff --git a/helpdesk/models.py b/helpdesk/models.py index 01ca72d5..fd425bb9 100644 --- a/helpdesk/models.py +++ b/helpdesk/models.py @@ -23,6 +23,8 @@ import re import six import uuid +from .templated_email import send_templated_mail + @python_2_unicode_compatible class Queue(models.Model): @@ -491,6 +493,54 @@ class Ticket(models.Model): default=mk_secret, ) + def send(self, roles, dont_send_to=None, **kwargs): + """ + Send notifications to everyone interested in this ticket. + + The the roles argument is a dictionary mapping from roles to (template, context) pairs. + If a role is not present in the dictionary, users of that type will not recieve the notification. + + The following roles exist: + + - 'submitter' + - 'new_ticket_cc' + - 'ticket_cc' + - 'assigned_to' + + Here is an example roles dictionary: + + { + 'submitter': (template_name, context), + 'assigned_to': (template_name2, context), + } + + **kwargs are passed to send_templated_mail defined in templated_mail.py + + returns the set of email addresses the notification was delivered to. + + """ + recipients = set() + + if dont_send_to is not None: + recipients.update(dont_send_to) + + def should_receive(email): + return email and email not in recipients + + def send(role, recipient): + if recipient and recipient not in recipients and role in roles: + template, context = roles[role] + send_templated_mail(template, context, recipient, sender=self.queue.from_address, **kwargs) + recipients.add(recipient) + send('submitter', self.submitter_email) + send('new_ticket_cc', self.queue.new_ticket_cc) + if self.assigned_to and self.assigned_to.usersettings_helpdesk.email_on_ticket_assign: + send('assigned_to', self.assigned_to.email) + send('ticket_cc', self.queue.updated_ticket_cc) + for cc in self.ticketcc_set.all(): + send('ticket_cc', cc.email_address) + return recipients + def _get_assigned_to(self): """ Custom property to allow us to easily print 'Unassigned' if a ticket has no owner, or the users name if it's assigned. If the user diff --git a/helpdesk/templated_email.py b/helpdesk/templated_email.py new file mode 100644 index 00000000..ab327bd6 --- /dev/null +++ b/helpdesk/templated_email.py @@ -0,0 +1,114 @@ +import os +import mimetypes +import logging +from smtplib import SMTPException + +from django.utils.safestring import mark_safe + +logger = logging.getLogger('helpdesk') + + +def send_templated_mail(template_name, + context, + recipients, + sender=None, + bcc=None, + fail_silently=False, + files=None): + """ + send_templated_mail() is a wrapper around Django's e-mail routines that + allows us to easily send multipart (text/plain & text/html) e-mails using + templates that are stored in the database. This lets the admin provide + both a text and a HTML template for each message. + + template_name is the slug of the template to use for this message (see + models.EmailTemplate) + + context is a dictionary to be used when rendering the template + + recipients can be either a string, eg 'a@b.com', or a list of strings. + + sender should contain a string, eg 'My Site '. If you leave it + blank, it'll use settings.DEFAULT_FROM_EMAIL as a fallback. + + bcc is an optional list of addresses that will receive this message as a + blind carbon copy. + + fail_silently is passed to Django's mail routine. Set to 'True' to ignore + any errors at send time. + + files can be a list of tuples. Each tuple should be a filename to attach, + along with the File objects to be read. files can be blank. + + """ + from django.core.mail import EmailMultiAlternatives + from django.template import engines + from_string = engines['django'].from_string + + from helpdesk.models import EmailTemplate + from helpdesk.settings import HELPDESK_EMAIL_SUBJECT_TEMPLATE, \ + HELPDESK_EMAIL_FALLBACK_LOCALE + + locale = context['queue'].get('locale') or HELPDESK_EMAIL_FALLBACK_LOCALE + + try: + t = EmailTemplate.objects.get(template_name__iexact=template_name, locale=locale) + except EmailTemplate.DoesNotExist: + try: + t = EmailTemplate.objects.get(template_name__iexact=template_name, locale__isnull=True) + except EmailTemplate.DoesNotExist: + logger.warning('template "%s" does not exist, no mail sent', template_name) + return # just ignore if template doesn't exist + + subject_part = from_string( + HELPDESK_EMAIL_SUBJECT_TEMPLATE % { + "subject": t.subject + }).render(context).replace('\n', '').replace('\r', '') + + footer_file = os.path.join('helpdesk', locale, 'email_text_footer.txt') + + text_part = from_string( + "%s{%% include '%s' %%}" % (t.plain_text, footer_file) + ).render(context) + + email_html_base_file = os.path.join('helpdesk', locale, 'email_html_base.html') + # keep new lines in html emails + if 'comment' in context: + context['comment'] = mark_safe(context['comment'].replace('\r\n', '
')) + + html_part = from_string( + "{%% extends '%s' %%}{%% block title %%}" + "%s" + "{%% endblock %%}{%% block content %%}%s{%% endblock %%}" % + (email_html_base_file, t.heading, t.html) + ).render(context) + + if isinstance(recipients, str): + if recipients.find(','): + recipients = recipients.split(',') + elif type(recipients) != list: + recipients = [recipients] + + msg = EmailMultiAlternatives(subject_part, text_part, + sender or settings.DEFAULT_FROM_EMAIL, + recipients, bcc=bcc) + msg.attach_alternative(html_part, "text/html") + + if files: + for filename, filefield in files: + mime = mimetypes.guess_type(filename) + if mime[0] is not None and mime[0] == "text/plain": + with open(filefield.path, 'r') as attachedfile: + content = attachedfile.read() + msg.attach(filename, content) + else: + msg.attach_file(filefield.path) + logger.debug('Sending email to: {!r}'.format(recipients)) + + try: + return msg.send() + except SMTPException as e: + logger.exception('SMTPException raised while sending email to {}'.format(recipients)) + if not fail_silently: + raise e + return 0 diff --git a/helpdesk/views/staff.py b/helpdesk/views/staff.py index ec8b729a..df986c1c 100644 --- a/helpdesk/views/staff.py +++ b/helpdesk/views/staff.py @@ -47,7 +47,7 @@ from helpdesk.forms import ( TicketCCEmailForm, TicketCCUserForm, EditFollowUpForm, TicketDependencyForm ) from helpdesk.lib import ( - send_templated_mail, query_to_dict, apply_query, safe_template_context, + query_to_dict, apply_query, safe_template_context, process_attachments, queue_template_context, ) from helpdesk.models import ( @@ -578,8 +578,6 @@ def update_ticket(request, ticket_id, public=False): if new_status == Ticket.RESOLVED_STATUS or ticket.resolution is None: ticket.resolution = comment - messages_sent_to = [] - # ticket might have changed above, so we re-instantiate context with the # (possibly) updated ticket. context = safe_template_context(ticket) @@ -588,6 +586,11 @@ def update_ticket(request, ticket_id, public=False): comment=f.comment, ) + messages_sent_to = set() + try: + messages_sent_to.add(request.user.email) + except AttributeError: + pass if public and (f.comment or ( f.new_status in (Ticket.RESOLVED_STATUS, Ticket.CLOSED_STATUS))): @@ -598,83 +601,44 @@ def update_ticket(request, ticket_id, public=False): else: template = 'updated_' - template_suffix = 'submitter' + roles = { + 'submitter': (template + 'submitter', context), + 'ticket_cc': (template + 'cc', context), + 'assigned_to': (template + 'cc', context), + } + messages_sent_to.update(ticket.send(roles, dont_send_to=messages_sent_to, fail_silently=True, files=files,)) - if ticket.submitter_email: - send_templated_mail( - template + template_suffix, - context, - recipients=ticket.submitter_email, - sender=ticket.queue.from_address, - fail_silently=True, - files=files, - ) - messages_sent_to.append(ticket.submitter_email) + if reassigned: + template_staff = 'assigned_owner' + elif f.new_status == Ticket.RESOLVED_STATUS: + template_staff = 'resolved_owner' + elif f.new_status == Ticket.CLOSED_STATUS: + template_staff = 'closed_owner' + else: + template_staff = 'updated_owner' - template_suffix = 'cc' + messages_sent_to.update(ticket.send( + {'assigned_to': (template_staff, context)}, + dont_send_to=messages_sent_to, + fail_silently=True, + files=files, + )) - for cc in ticket.ticketcc_set.all(): - if cc.email_address not in messages_sent_to: - send_templated_mail( - template + template_suffix, - context, - recipients=cc.email_address, - sender=ticket.queue.from_address, - fail_silently=True, - files=files, - ) - messages_sent_to.append(cc.email_address) + if reassigned: + template_cc = 'assigned_cc' + elif f.new_status == Ticket.RESOLVED_STATUS: + template_cc = 'resolved_cc' + elif f.new_status == Ticket.CLOSED_STATUS: + template_cc = 'closed_cc' + else: + template_cc = 'updated_cc' - if ticket.assigned_to and \ - request.user != ticket.assigned_to and \ - ticket.assigned_to.email and \ - ticket.assigned_to.email not in messages_sent_to: - # We only send e-mails to staff members if the ticket is updated by - # another user. The actual template varies, depending on what has been - # changed. - if reassigned: - template_staff = 'assigned_owner' - elif f.new_status == Ticket.RESOLVED_STATUS: - template_staff = 'resolved_owner' - elif f.new_status == Ticket.CLOSED_STATUS: - template_staff = 'closed_owner' - else: - template_staff = 'updated_owner' - - if (not reassigned or - (reassigned and - ticket.assigned_to.usersettings_helpdesk.email_on_ticket_assign)) or \ - (not reassigned and - ticket.assigned_to.usersettings_helpdesk.email_on_ticket_change): - - send_templated_mail( - template_staff, - context, - recipients=ticket.assigned_to.email, - sender=ticket.queue.from_address, - fail_silently=True, - files=files, - ) - messages_sent_to.append(ticket.assigned_to.email) - - if ticket.queue.updated_ticket_cc and ticket.queue.updated_ticket_cc not in messages_sent_to: - if reassigned: - template_cc = 'assigned_cc' - elif f.new_status == Ticket.RESOLVED_STATUS: - template_cc = 'resolved_cc' - elif f.new_status == Ticket.CLOSED_STATUS: - template_cc = 'closed_cc' - else: - template_cc = 'updated_cc' - - send_templated_mail( - template_cc, - context, - recipients=ticket.queue.updated_ticket_cc, - sender=ticket.queue.from_address, - fail_silently=True, - files=files, - ) + messages_sent_to.update(ticket.send( + {'ticket_cc': (template_cc, context)}, + dont_send_to=messages_sent_to, + fail_silently=True, + files=files, + )) ticket.save() @@ -760,51 +724,19 @@ def mass_update(request): context.update(resolution=t.resolution, queue=queue_template_context(t.queue)) - messages_sent_to = [] + messages_sent_to = set() + try: + messages_sent_to.add(request.user.email) + except AttributeError: + pass - if t.submitter_email: - send_templated_mail( - 'closed_submitter', - context, - recipients=t.submitter_email, - sender=t.queue.from_address, - fail_silently=True, - ) - messages_sent_to.append(t.submitter_email) - - for cc in t.ticketcc_set.all(): - if cc.email_address not in messages_sent_to: - send_templated_mail( - 'closed_submitter', - context, - recipients=cc.email_address, - sender=t.queue.from_address, - fail_silently=True, - ) - messages_sent_to.append(cc.email_address) - - if t.assigned_to and \ - request.user != t.assigned_to and \ - t.assigned_to.email and \ - t.assigned_to.email not in messages_sent_to: - send_templated_mail( - 'closed_owner', - context, - recipients=t.assigned_to.email, - sender=t.queue.from_address, - fail_silently=True, - ) - messages_sent_to.append(t.assigned_to.email) - - if t.queue.updated_ticket_cc and \ - t.queue.updated_ticket_cc not in messages_sent_to: - send_templated_mail( - 'closed_cc', - context, - recipients=t.queue.updated_ticket_cc, - sender=t.queue.from_address, - fail_silently=True, - ) + messages_sent_to.update(t.send( + {'submitter': ('closed_submitter', context), + 'ticket_cc': ('closed_cc', context), + 'assigned_to': ('closded_owner', context)}, + dont_send_to=messages_sent_to, + fail_silently=True, + )) elif action == 'delete': t.delete() From 09d8f8c488f6781d7641f45b187881eb424c501b Mon Sep 17 00:00:00 2001 From: Timothy Hobbs Date: Thu, 1 Nov 2018 19:30:34 +0100 Subject: [PATCH 5/5] Try to obey UserSettings when sending emails --- helpdesk/models.py | 2 +- helpdesk/views/staff.py | 27 +++++++++++++++++---------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/helpdesk/models.py b/helpdesk/models.py index fd425bb9..db22bd81 100644 --- a/helpdesk/models.py +++ b/helpdesk/models.py @@ -534,7 +534,7 @@ class Ticket(models.Model): recipients.add(recipient) send('submitter', self.submitter_email) send('new_ticket_cc', self.queue.new_ticket_cc) - if self.assigned_to and self.assigned_to.usersettings_helpdesk.email_on_ticket_assign: + if self.assigned_to: send('assigned_to', self.assigned_to.email) send('ticket_cc', self.queue.updated_ticket_cc) for cc in self.ticketcc_set.all(): diff --git a/helpdesk/views/staff.py b/helpdesk/views/staff.py index df986c1c..759f661b 100644 --- a/helpdesk/views/staff.py +++ b/helpdesk/views/staff.py @@ -604,8 +604,9 @@ def update_ticket(request, ticket_id, public=False): roles = { 'submitter': (template + 'submitter', context), 'ticket_cc': (template + 'cc', context), - 'assigned_to': (template + 'cc', context), } + if ticket.assigned_to and ticket.assigned_to.usersettings_helpdesk.email_on_ticket_change: + roles['assigned_to'] = (template + 'cc', context) messages_sent_to.update(ticket.send(roles, dont_send_to=messages_sent_to, fail_silently=True, files=files,)) if reassigned: @@ -617,12 +618,13 @@ def update_ticket(request, ticket_id, public=False): else: template_staff = 'updated_owner' - messages_sent_to.update(ticket.send( - {'assigned_to': (template_staff, context)}, - dont_send_to=messages_sent_to, - fail_silently=True, - files=files, - )) + if ticket.assigned_to and (ticket.assigned_to.usersettings_helpdesk.email_on_ticket_change or (reassigned and ticket.assigned_to.usersettings_helpdesk.email_on_ticket_assigned)): + messages_sent_to.update(ticket.send( + {'assigned_to': (template_staff, context)}, + dont_send_to=messages_sent_to, + fail_silently=True, + files=files, + )) if reassigned: template_cc = 'assigned_cc' @@ -730,10 +732,15 @@ def mass_update(request): except AttributeError: pass + roles = { + 'submitter': ('closed_submitter', context), + 'ticket_cc': ('closed_cc', context), + } + if ticket.assigned_to and ticket.assigned_to.usersettings_helpdesk.email_on_ticket_change: + roles['assigned_to'] = ('closed_owner', context), + messages_sent_to.update(t.send( - {'submitter': ('closed_submitter', context), - 'ticket_cc': ('closed_cc', context), - 'assigned_to': ('closded_owner', context)}, + roles, dont_send_to=messages_sent_to, fail_silently=True, ))