Merge pull request #660 from auto-mat/email-refactor

Email refactor

helpdesk/email.py

@@ -0,0 +1,481 @@
+#!/usr/bin/python
+"""
+Django Helpdesk - A Django powered ticket tracker for small enterprise.
+
+(c) Copyright 2008 Jutda. Copyright 2018 Timothy Hobbs. All Rights Reserved.
+See LICENSE for details.
+
+scripts/get_email.py - Designed to be run from cron, this script checks the
+                       POP and IMAP boxes, or a local mailbox directory,
+                       defined for the queues within a
+                       helpdesk, creating tickets from the new messages (or
+                       adding to existing tickets if needed)
+"""
+
+from datetime import timedelta
+import base64
+import binascii
+import email
+import imaplib
+import mimetypes
+from os import listdir, unlink
+from os.path import isfile, join
+import poplib
+import re
+import socket
+import ssl
+import sys
+from time import ctime
+
+from bs4 import BeautifulSoup
+
+from email_reply_parser import EmailReplyParser
+
+from django.core.files.base import ContentFile
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+from django.utils.translation import ugettext as _
+from django.utils import encoding, timezone
+
+from helpdesk import settings
+from helpdesk.lib import safe_template_context, process_attachments
+from helpdesk.models import Queue, Ticket, TicketCC, FollowUp, IgnoreEmail
+from django.contrib.auth.models import User
+
+import logging
+
+
+STRIPPED_SUBJECT_STRINGS = [
+    "Re: ",
+    "Fw: ",
+    "RE: ",
+    "FW: ",
+    "Automatic reply: ",
+]
+
+
+def process_email(quiet=False):
+    for q in Queue.objects.filter(
+            email_box_type__isnull=False,
+            allow_email_submission=True):
+
+        logger = logging.getLogger('django.helpdesk.queue.' + q.slug)
+        logging_types = {
+            'info': logging.INFO,
+            'warn': logging.WARN,
+            'error': logging.ERROR,
+            'crit': logging.CRITICAL,
+            'debug': logging.DEBUG,
+        }
+        if q.logging_type in logging_types:
+            logger.setLevel(logging_types[q.logging_type])
+        elif not q.logging_type or q.logging_type == 'none':
+            logging.disable(logging.CRITICAL)  # disable all messages
+        if quiet:
+            logger.propagate = False  # do not propagate to root logger that would log to console
+        logdir = q.logging_dir or '/var/log/helpdesk/'
+        handler = logging.FileHandler(join(logdir, q.slug + '_get_email.log'))
+        logger.addHandler(handler)
+
+        if not q.email_box_last_check:
+            q.email_box_last_check = timezone.now() - timedelta(minutes=30)
+
+        queue_time_delta = timedelta(minutes=q.email_box_interval or 0)
+
+        if (q.email_box_last_check + queue_time_delta) < timezone.now():
+            process_queue(q, logger=logger)
+            q.email_box_last_check = timezone.now()
+            q.save()
+
+
+def pop3_sync(q, logger, server):
+    server.getwelcome()
+    server.user(q.email_box_user or settings.QUEUE_EMAIL_BOX_USER)
+    server.pass_(q.email_box_pass or settings.QUEUE_EMAIL_BOX_PASSWORD)
+
+    messagesInfo = server.list()[1]
+    logger.info("Received %d messages from POP3 server" % len(messagesInfo))
+
+    for msgRaw in messagesInfo:
+        if type(msgRaw) is bytes:
+            try:
+                msg = msgRaw.decode("utf-8")
+            except UnicodeError:
+                # if couldn't decode easily, just leave it raw
+                msg = msgRaw
+        else:
+            # already a str
+            msg = msgRaw
+        msgNum = msg.split(" ")[0]
+        logger.info("Processing message %s" % msgNum)
+
+        raw_content = server.retr(msgNum)[1]
+        if type(raw_content[0]) is bytes:
+            full_message = "\n".join([elm.decode('utf-8') for elm in raw_content])
+        else:
+            full_message = encoding.force_text("\n".join(raw_content), errors='replace')
+        ticket = ticket_from_message(message=full_message, queue=q, logger=logger)
+
+        if ticket:
+            server.dele(msgNum)
+            logger.info("Successfully processed message %s, deleted from POP3 server" % msgNum)
+        else:
+            logger.warn("Message %s was not successfully processed, and will be left on POP3 server" % msgNum)
+
+    server.quit()
+
+
+def imap_sync(q, logger, server):
+    try:
+        server.login(q.email_box_user or
+                     settings.QUEUE_EMAIL_BOX_USER,
+                     q.email_box_pass or
+                     settings.QUEUE_EMAIL_BOX_PASSWORD)
+        server.select(q.email_box_imap_folder)
+    except imaplib.IMAP4.abort:
+        logger.error("IMAP login failed. Check that the server is accessible and that the username and password are correct.")
+        server.logout()
+        sys.exit()
+    except ssl.SSLError:
+        logger.error("IMAP login failed due to SSL error. This is often due to a timeout. Please check your connection and try again.")
+        server.logout()
+        sys.exit()
+
+    try:
+        status, data = server.search(None, 'NOT', 'DELETED')
+    except imaplib.IMAP4.error:
+        logger.error("IMAP retrieve failed. Is the folder '%s' spelled correctly, and does it exist on the server?" % q.email_box_imap_folder)
+    if data:
+        msgnums = data[0].split()
+        logger.info("Received %d messages from IMAP server" % len(msgnums))
+        for num in msgnums:
+            logger.info("Processing message %s" % num)
+            status, data = server.fetch(num, '(RFC822)')
+            full_message = encoding.force_text(data[0][1], errors='replace')
+            try:
+                ticket = ticket_from_message(message=full_message, queue=q, logger=logger)
+            except TypeError:
+                ticket = None  # hotfix. Need to work out WHY.
+            if ticket:
+                server.store(num, '+FLAGS', '\\Deleted')
+                logger.info("Successfully processed message %s, deleted from IMAP server" % num)
+            else:
+                logger.warn("Message %s was not successfully processed, and will be left on IMAP server" % num)
+
+    server.expunge()
+    server.close()
+    server.logout()
+
+
+def process_queue(q, logger):
+    logger.info("***** %s: Begin processing mail for django-helpdesk" % ctime())
+
+    if q.socks_proxy_type and q.socks_proxy_host and q.socks_proxy_port:
+        try:
+            import socks
+        except ImportError:
+            no_socks_msg = "Queue has been configured with proxy settings, " \
+                           "but no socks library was installed. Try to " \
+                           "install PySocks via PyPI."
+            logger.error(no_socks_msg)
+            raise ImportError(no_socks_msg)
+
+        proxy_type = {
+            'socks4': socks.SOCKS4,
+            'socks5': socks.SOCKS5,
+        }.get(q.socks_proxy_type)
+
+        socks.set_default_proxy(proxy_type=proxy_type,
+                                addr=q.socks_proxy_host,
+                                port=q.socks_proxy_port)
+        socket.socket = socks.socksocket
+
+    email_box_type = settings.QUEUE_EMAIL_BOX_TYPE or q.email_box_type
+
+    mail_defaults = {
+        'pop3': {
+            'ssl': {
+                'port': 995,
+                'init': poplib.POP3_SSL,
+            },
+            'insecure': {
+                'port': 110,
+                'init': poplib.POP3,
+            },
+            'sync': pop3_sync,
+        },
+        'imap': {
+            'ssl': {
+                'port': 993,
+                'init': imaplib.IMAP4_SSL,
+            },
+            'insecure': {
+                'port': 143,
+                'init': imaplib.IMAP4,
+            },
+            'sync': imap_sync
+        }
+    }
+    if email_box_type in mail_defaults:
+        encryption = 'insecure'
+        if q.email_box_ssl or settings.QUEUE_EMAIL_BOX_SSL:
+            encryption = 'ssl'
+        if not q.email_box_port:
+            q.email_box_port = mail_defaults[email_box_type][encryption]['port']
+
+        server = mail_defaults[email_box_type][encryption]['init'](
+            q.email_box_host or settings.QUEUE_EMAIL_BOX_HOST,
+            int(q.email_box_port)
+        )
+        logger.info("Attempting %s server login" % email_box_type.upper())
+        mail_defaults[email_box_type]['sync'](q, logger, server)
+
+    elif email_box_type == 'local':
+        mail_dir = q.email_box_local_dir or '/var/lib/mail/helpdesk/'
+        mail = [join(mail_dir, f) for f in listdir(mail_dir) if isfile(join(mail_dir, f))]
+        logger.info("Found %d messages in local mailbox directory" % len(mail))
+
+        logger.info("Found %d messages in local mailbox directory" % len(mail))
+        for i, m in enumerate(mail, 1):
+            logger.info("Processing message %d" % i)
+            with open(m, 'r') as f:
+                full_message = encoding.force_text(f.read(), errors='replace')
+                ticket = ticket_from_message(message=full_message, queue=q, logger=logger)
+            if ticket:
+                logger.info("Successfully processed message %d, ticket/comment created." % i)
+                try:
+                    unlink(m)  # delete message file if ticket was successful
+                except OSError:
+                    logger.error("Unable to delete message %d." % i)
+                else:
+                    logger.info("Successfully deleted message %d." % i)
+            else:
+                logger.warn("Message %d was not successfully processed, and will be left in local directory" % i)
+
+
+def decodeUnknown(charset, string):
+    if type(string) is not str:
+        if not charset:
+            try:
+                return str(string, encoding='utf-8', errors='replace')
+            except UnicodeError:
+                return str(string, encoding='iso8859-1', errors='replace')
+        return str(string, encoding=charset, errors='replace')
+    return string
+
+
+def decode_mail_headers(string):
+    decoded = email.header.decode_header(string)
+    return u' '.join([str(msg, encoding=charset, errors='replace') if charset else str(msg) for msg, charset in decoded])
+
+
+def ticket_from_message(message, queue, logger):
+    # 'message' must be an RFC822 formatted message.
+    message = email.message_from_string(message)
+    subject = message.get('subject', _('Comment from e-mail'))
+    subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject))
+    for affix in STRIPPED_SUBJECT_STRINGS:
+        subject = subject.replace(affix, "")
+    subject = subject.strip()
+
+    sender = message.get('from', _('Unknown Sender'))
+    sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
+    sender_email = email.utils.parseaddr(sender)[1]
+
+    cc = message.get_all('cc', None)
+    if cc:
+        # first, fixup the encoding if necessary
+        cc = [decode_mail_headers(decodeUnknown(message.get_charset(), x)) for x in cc]
+        # get_all checks if multiple CC headers, but individual emails may be comma separated too
+        tempcc = []
+        for hdr in cc:
+            tempcc.extend(hdr.split(','))
+        # use a set to ensure no duplicates
+        cc = set([x.strip() for x in tempcc])
+
+    for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)):
+        if ignore.test(sender_email):
+            if ignore.keep_in_mailbox:
+                # By returning 'False' the message will be kept in the mailbox,
+                # and the 'True' will cause the message to be deleted.
+                return False
+            return True
+
+    matchobj = re.match(r".*\[" + queue.slug + r"-(?P<id>\d+)\]", subject)
+    if matchobj:
+        # This is a reply or forward.
+        ticket = matchobj.group('id')
+        logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket))
+    else:
+        logger.info("No tracking ID matched.")
+        ticket = None
+
+    body = None
+    counter = 0
+    files = []
+
+    for part in message.walk():
+        if part.get_content_maintype() == 'multipart':
+            continue
+
+        name = part.get_param("name")
+        if name:
+            name = email.utils.collapse_rfc2231_value(name)
+
+        if part.get_content_maintype() == 'text' and name is None:
+            if part.get_content_subtype() == 'plain':
+                body = EmailReplyParser.parse_reply(
+                    decodeUnknown(part.get_content_charset(), part.get_payload(decode=True))
+                )
+                # workaround to get unicode text out rather than escaped text
+                try:
+                    body = body.encode('ascii').decode('unicode_escape')
+                except UnicodeEncodeError:
+                    body.encode('utf-8')
+                logger.debug("Discovered plain text MIME part")
+            else:
+                files.append(
+                    SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html')
+                )
+                logger.debug("Discovered HTML MIME part")
+        else:
+            if not name:
+                ext = mimetypes.guess_extension(part.get_content_type())
+                name = "part-%i%s" % (counter, ext)
+            payload = part.get_payload()
+            if isinstance(payload, list):
+                payload = payload.pop().as_string()
+            payloadToWrite = payload
+            # check version of python to ensure use of only the correct error type
+            non_b64_err = TypeError
+            try:
+                logger.debug("Try to base64 decode the attachment payload")
+                payloadToWrite = base64.decodebytes(payload)
+            except non_b64_err:
+                logger.debug("Payload was not base64 encoded, using raw bytes")
+                payloadToWrite = payload
+            files.append(SimpleUploadedFile(name, part.get_payload(decode=True), mimetypes.guess_type(name)[0]))
+            logger.debug("Found MIME attachment %s" % name)
+
+        counter += 1
+
+    if not body:
+        mail = BeautifulSoup(part.get_payload(), "lxml")
+        if ">" in mail.text:
+            body = mail.find('body')
+            body = body.text
+            body = body.encode('ascii', errors='ignore')
+        else:
+            body = mail.text
+
+    if ticket:
+        try:
+            t = Ticket.objects.get(id=ticket)
+        except Ticket.DoesNotExist:
+            logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket))
+            ticket = None
+        else:
+            logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id))
+            if t.status == Ticket.CLOSED_STATUS:
+                t.status = Ticket.REOPENED_STATUS
+                t.save()
+            new = False
+
+    smtp_priority = message.get('priority', '')
+    smtp_importance = message.get('importance', '')
+    high_priority_types = {'high', 'important', '1', 'urgent'}
+    priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3
+
+    if ticket is None:
+        if settings.QUEUE_EMAIL_BOX_UPDATE_ONLY:
+            return None
+        new = True
+        t = Ticket.objects.create(
+            title=subject,
+            queue=queue,
+            submitter_email=sender_email,
+            created=timezone.now(),
+            description=body,
+            priority=priority,
+        )
+        logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id))
+
+    if cc:
+        # get list of currently CC'd emails
+        current_cc = TicketCC.objects.filter(ticket=ticket)
+        current_cc_emails = [x.email for x in current_cc if x.email]
+        # get emails of any Users CC'd to email, if defined
+        # (some Users may not have an associated email, e.g, when using LDAP)
+        current_cc_users = [x.user.email for x in current_cc if x.user and x.user.email]
+        # ensure submitter, assigned user, queue email not added
+        other_emails = [queue.email_address]
+        if t.submitter_email:
+            other_emails.append(t.submitter_email)
+        if t.assigned_to:
+            other_emails.append(t.assigned_to.email)
+        current_cc = set(current_cc_emails + current_cc_users + other_emails)
+        # first, add any User not previously CC'd (as identified by User's email)
+        all_users = User.objects.all()
+        all_user_emails = set([x.email for x in all_users])
+        users_not_currently_ccd = all_user_emails.difference(set(current_cc))
+        users_to_cc = cc.intersection(users_not_currently_ccd)
+        for user in users_to_cc:
+            tcc = TicketCC.objects.create(
+                ticket=t,
+                user=User.objects.get(email=user),
+                can_view=True,
+                can_update=False
+            )
+            tcc.save()
+        # then add remaining emails alphabetically, makes testing easy
+        new_cc = cc.difference(current_cc).difference(all_user_emails)
+        new_cc = sorted(list(new_cc))
+        for ccemail in new_cc:
+            tcc = TicketCC.objects.create(
+                ticket=t,
+                email=ccemail.replace('\n', ' ').replace('\r', ' '),
+                can_view=True,
+                can_update=False
+            )
+            tcc.save()
+
+    f = FollowUp(
+        ticket=t,
+        title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}),
+        date=timezone.now(),
+        public=True,
+        comment=body,
+    )
+
+    if t.status == Ticket.REOPENED_STATUS:
+        f.new_status = Ticket.REOPENED_STATUS
+        f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email})
+
+    f.save()
+    logger.debug("Created new FollowUp for Ticket")
+
+    logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,))
+
+    attached = process_attachments(f, files)
+    for att_file in attached:
+        logger.info("Attachment '%s' (with size %s) successfully added to ticket from email." % (att_file[0], att_file[1].size))
+
+    context = safe_template_context(t)
+
+    if new:
+        t.send(
+            {'submitter': ('newticket_submitter', context),
+             'new_ticket_cc': ('newticket_cc', context),
+             'ticket_cc': ('newticket_cc', context)},
+            fail_silently=True,
+        )
+    else:
+        context.update(comment=f.comment)
+        t.send(
+            {'assigned_to': ('updated_owner', context),
+             'ticket_cc': ('updated_cc', context)},
+            fail_silently=True,
+        )
+
+    return t

helpdesk/forms.py

@@ -17,7 +17,7 @@ from django.utils.translation import ugettext_lazy as _
 from django.contrib.auth import get_user_model
 from django.utils import timezone
 
-from helpdesk.lib import send_templated_mail, safe_template_context, process_attachments
+from helpdesk.lib import safe_template_context, process_attachments
 from helpdesk.models import (Ticket, Queue, FollowUp, Attachment, IgnoreEmail, TicketCC,
                              CustomField, TicketCustomFieldValue, TicketDependency, UserSettings)
 from helpdesk import settings as helpdesk_settings
@@ -238,56 +238,16 @@ class AbstractTicketForm(CustomFieldMixin, forms.Form):
         context = safe_template_context(ticket)
         context['comment'] = followup.comment
 
-        messages_sent_to = []
+        roles = {'submitter': ('newticket_submitter', context),
-
+                 'new_ticket_cc': ('newticket_cc', context),
-        if ticket.submitter_email:
+                 'ticket_cc': ('newticket_cc', context)}
-            send_templated_mail(
+        if ticket.assigned_to and ticket.assigned_to.usersettings_helpdesk.email_on_ticket_assign:
-                'newticket_submitter',
+            roles['assigned_to'] = ('assigned_owner', context)
-                context,
+        ticket.send(
-                recipients=ticket.submitter_email,
+            roles,
-                sender=queue.from_address,
+            fail_silently=True,
-                fail_silently=True,
+            files=files,
-                files=files,
+        )
-            )
-            messages_sent_to.append(ticket.submitter_email)
-
-        if ticket.assigned_to and \
-                ticket.assigned_to != user and \
-                ticket.assigned_to.usersettings_helpdesk.email_on_ticket_assign and \
-                ticket.assigned_to.email and \
-                ticket.assigned_to.email not in messages_sent_to:
-            send_templated_mail(
-                'assigned_owner',
-                context,
-                recipients=ticket.assigned_to.email,
-                sender=queue.from_address,
-                fail_silently=True,
-                files=files,
-            )
-            messages_sent_to.append(ticket.assigned_to.email)
-
-        if queue.new_ticket_cc and queue.new_ticket_cc not in messages_sent_to:
-            send_templated_mail(
-                'newticket_cc',
-                context,
-                recipients=queue.new_ticket_cc,
-                sender=queue.from_address,
-                fail_silently=True,
-                files=files,
-            )
-            messages_sent_to.append(queue.new_ticket_cc)
-
-        if queue.updated_ticket_cc and \
-                queue.updated_ticket_cc != queue.new_ticket_cc and \
-                queue.updated_ticket_cc not in messages_sent_to:
-            send_templated_mail(
-                'newticket_cc',
-                context,
-                recipients=queue.updated_ticket_cc,
-                sender=queue.from_address,
-                fail_silently=True,
-                files=files,
-            )
 
 
 class TicketForm(AbstractTicketForm):

helpdesk/lib.py

@@ -9,142 +9,22 @@ lib.py - Common functions (eg multipart e-mail)
 import logging
 import mimetypes
 import os
-from smtplib import SMTPException
 
 from django.conf import settings
 from django.db.models import Q
-from django.utils import six
 from django.utils.encoding import smart_text
 from django.utils.safestring import mark_safe
 
 from helpdesk.models import Attachment, EmailTemplate
 
-import six
-
 from model_utils import Choices
 
-if six.PY3:
+from base64 import encodebytes as b64encode
-    from base64 import encodebytes as b64encode
+from base64 import decodebytes as b64decode
-    from base64 import decodebytes as b64decode
-else:
-    from base64 import urlsafe_b64encode as b64encode
-    from base64 import urlsafe_b64decode as b64decode
 
 logger = logging.getLogger('helpdesk')
 
 
-def send_templated_mail(template_name,
-                        context,
-                        recipients,
-                        sender=None,
-                        bcc=None,
-                        fail_silently=False,
-                        files=None):
-    """
-    send_templated_mail() is a wrapper around Django's e-mail routines that
-    allows us to easily send multipart (text/plain & text/html) e-mails using
-    templates that are stored in the database. This lets the admin provide
-    both a text and a HTML template for each message.
-
-    template_name is the slug of the template to use for this message (see
-        models.EmailTemplate)
-
-    context is a dictionary to be used when rendering the template
-
-    recipients can be either a string, eg 'a@b.com', or a list of strings.
-
-    sender should contain a string, eg 'My Site <me@z.com>'. If you leave it
-        blank, it'll use settings.DEFAULT_FROM_EMAIL as a fallback.
-
-    bcc is an optional list of addresses that will receive this message as a
-        blind carbon copy.
-
-    fail_silently is passed to Django's mail routine. Set to 'True' to ignore
-        any errors at send time.
-
-    files can be a list of tuples. Each tuple should be a filename to attach,
-        along with the File objects to be read. files can be blank.
-
-    """
-    from django.core.mail import EmailMultiAlternatives
-    from django.template import engines
-    from_string = engines['django'].from_string
-
-    from helpdesk.models import EmailTemplate
-    from helpdesk.settings import HELPDESK_EMAIL_SUBJECT_TEMPLATE, \
-        HELPDESK_EMAIL_FALLBACK_LOCALE
-
-    locale = context['queue'].get('locale') or HELPDESK_EMAIL_FALLBACK_LOCALE
-
-    try:
-        t = EmailTemplate.objects.get(template_name__iexact=template_name, locale=locale)
-    except EmailTemplate.DoesNotExist:
-        try:
-            t = EmailTemplate.objects.get(template_name__iexact=template_name, locale__isnull=True)
-        except EmailTemplate.DoesNotExist:
-            logger.warning('template "%s" does not exist, no mail sent', template_name)
-            return  # just ignore if template doesn't exist
-
-    subject_part = from_string(
-        HELPDESK_EMAIL_SUBJECT_TEMPLATE % {
-            "subject": t.subject
-        }).render(context).replace('\n', '').replace('\r', '')
-
-    footer_file = os.path.join('helpdesk', locale, 'email_text_footer.txt')
-
-    text_part = from_string(
-        "%s{%% include '%s' %%}" % (t.plain_text, footer_file)
-    ).render(context)
-
-    email_html_base_file = os.path.join('helpdesk', locale, 'email_html_base.html')
-    # keep new lines in html emails
-    if 'comment' in context:
-        context['comment'] = mark_safe(context['comment'].replace('\r\n', '<br>'))
-
-    html_part = from_string(
-        "{%% extends '%s' %%}{%% block title %%}"
-        "%s"
-        "{%% endblock %%}{%% block content %%}%s{%% endblock %%}" %
-        (email_html_base_file, t.heading, t.html)
-    ).render(context)
-
-    if isinstance(recipients, str):
-        if recipients.find(','):
-            recipients = recipients.split(',')
-    elif type(recipients) != list:
-        recipients = [recipients]
-
-    msg = EmailMultiAlternatives(subject_part, text_part,
-                                 sender or settings.DEFAULT_FROM_EMAIL,
-                                 recipients, bcc=bcc)
-    msg.attach_alternative(html_part, "text/html")
-
-    if files:
-        for filename, filefield in files:
-            mime = mimetypes.guess_type(filename)
-            if mime[0] is not None and mime[0] == "text/plain":
-                with open(filefield.path, 'r') as attachedfile:
-                    content = attachedfile.read()
-                    msg.attach(filename, content)
-            else:
-                if six.PY3:
-                    msg.attach_file(filefield.path)
-                else:
-                    with open(filefield.path, 'rb') as attachedfile:
-                        content = attachedfile.read()
-                        msg.attach(filename, content)
-
-    logger.debug('Sending email to: {!r}'.format(recipients))
-
-    try:
-        return msg.send()
-    except SMTPException as e:
-        logger.exception('SMTPException raised while sending email to {}'.format(recipients))
-        if not fail_silently:
-            raise e
-        return 0
-
-
 def query_to_dict(results, descriptions):
     """
     Replacement method for cursor.dictfetchall() as that method no longer

helpdesk/management/commands/escalate_tickets.py

@@ -24,7 +24,7 @@ except ImportError:
     from datetime import datetime as timezone
 
 from helpdesk.models import Queue, Ticket, FollowUp, EscalationExclusion, TicketChange
-from helpdesk.lib import send_templated_mail, safe_template_context
+from helpdesk.lib import safe_template_context
 
 
 class Command(BaseCommand):
@@ -107,32 +107,12 @@ def escalate_tickets(queues, verbose):
 
             context = safe_template_context(t)
 
-            if t.submitter_email:
+            t.send(
-                send_templated_mail(
+                {'submitter': ('escalated_submitter', context),
-                    'escalated_submitter',
+                 'ticket_cc': ('escalated_cc', context),
-                    context,
+                 'assigned_to': ('escalated_owner', context)}
-                    recipients=t.submitter_email,
+                fail_silently=True,
-                    sender=t.queue.from_address,
+            )
-                    fail_silently=True,
-                )
-
-            if t.queue.updated_ticket_cc:
-                send_templated_mail(
-                    'escalated_cc',
-                    context,
-                    recipients=t.queue.updated_ticket_cc,
-                    sender=t.queue.from_address,
-                    fail_silently=True,
-                )
-
-            if t.assigned_to:
-                send_templated_mail(
-                    'escalated_owner',
-                    context,
-                    recipients=t.assigned_to.email,
-                    sender=t.queue.from_address,
-                    fail_silently=True,
-                )
 
             if verbose:
                 print("  - Esclating %s from %s>%s" % (

helpdesk/management/commands/get_email.py

@@ -10,49 +10,9 @@ scripts/get_email.py - Designed to be run from cron, this script checks the
                        helpdesk, creating tickets from the new messages (or
                        adding to existing tickets if needed)
 """
-from __future__ import unicode_literals
-
-from datetime import timedelta
-import base64
-import binascii
-import email
-import imaplib
-import mimetypes
-from os import listdir, unlink
-from os.path import isfile, join
-import poplib
-import re
-import socket
-import ssl
-import sys
-from time import ctime
-
-from bs4 import BeautifulSoup
-
-from email_reply_parser import EmailReplyParser
-
-from django.core.files.base import ContentFile
-from django.core.files.uploadedfile import SimpleUploadedFile
 from django.core.management.base import BaseCommand
-from django.db.models import Q
-from django.utils.translation import ugettext as _
-from django.utils import encoding, six, timezone
 
-from helpdesk import settings
+from helpdesk.email import process_email
-from helpdesk.lib import send_templated_mail, safe_template_context, process_attachments
-from helpdesk.models import Queue, Ticket, TicketCC, FollowUp, IgnoreEmail
-from django.contrib.auth.models import User
-
-import logging
-
-
-STRIPPED_SUBJECT_STRINGS = [
-    "Re: ",
-    "Fw: ",
-    "RE: ",
-    "FW: ",
-    "Automatic reply: ",
-]
 
 
 class Command(BaseCommand):
@@ -77,477 +37,5 @@ class Command(BaseCommand):
         process_email(quiet=quiet)
 
 
-def process_email(quiet=False):
-    for q in Queue.objects.filter(
-            email_box_type__isnull=False,
-            allow_email_submission=True):
-
-        logger = logging.getLogger('django.helpdesk.queue.' + q.slug)
-        if not q.logging_type or q.logging_type == 'none':
-            logging.disable(logging.CRITICAL)  # disable all messages
-        elif q.logging_type == 'info':
-            logger.setLevel(logging.INFO)
-        elif q.logging_type == 'warn':
-            logger.setLevel(logging.WARN)
-        elif q.logging_type == 'error':
-            logger.setLevel(logging.ERROR)
-        elif q.logging_type == 'crit':
-            logger.setLevel(logging.CRITICAL)
-        elif q.logging_type == 'debug':
-            logger.setLevel(logging.DEBUG)
-        if quiet:
-            logger.propagate = False  # do not propagate to root logger that would log to console
-        logdir = q.logging_dir or '/var/log/helpdesk/'
-        handler = logging.FileHandler(join(logdir, q.slug + '_get_email.log'))
-        logger.addHandler(handler)
-
-        if not q.email_box_last_check:
-            q.email_box_last_check = timezone.now() - timedelta(minutes=30)
-
-        queue_time_delta = timedelta(minutes=q.email_box_interval or 0)
-
-        if (q.email_box_last_check + queue_time_delta) < timezone.now():
-            process_queue(q, logger=logger)
-            q.email_box_last_check = timezone.now()
-            q.save()
-
-
-def process_queue(q, logger):
-    logger.info("***** %s: Begin processing mail for django-helpdesk" % ctime())
-
-    if q.socks_proxy_type and q.socks_proxy_host and q.socks_proxy_port:
-        try:
-            import socks
-        except ImportError:
-            no_socks_msg = "Queue has been configured with proxy settings, " \
-                           "but no socks library was installed. Try to " \
-                           "install PySocks via PyPI."
-            logger.error(no_socks_msg)
-            raise ImportError(no_socks_msg)
-
-        proxy_type = {
-            'socks4': socks.SOCKS4,
-            'socks5': socks.SOCKS5,
-        }.get(q.socks_proxy_type)
-
-        socks.set_default_proxy(proxy_type=proxy_type,
-                                addr=q.socks_proxy_host,
-                                port=q.socks_proxy_port)
-        socket.socket = socks.socksocket
-    elif six.PY2:
-        socket.socket = socket._socketobject
-
-    email_box_type = settings.QUEUE_EMAIL_BOX_TYPE or q.email_box_type
-
-    if email_box_type == 'pop3':
-        if q.email_box_ssl or settings.QUEUE_EMAIL_BOX_SSL:
-            if not q.email_box_port:
-                q.email_box_port = 995
-            server = poplib.POP3_SSL(q.email_box_host or
-                                     settings.QUEUE_EMAIL_BOX_HOST,
-                                     int(q.email_box_port))
-        else:
-            if not q.email_box_port:
-                q.email_box_port = 110
-            server = poplib.POP3(q.email_box_host or
-                                 settings.QUEUE_EMAIL_BOX_HOST,
-                                 int(q.email_box_port))
-
-        logger.info("Attempting POP3 server login")
-
-        server.getwelcome()
-        server.user(q.email_box_user or settings.QUEUE_EMAIL_BOX_USER)
-        server.pass_(q.email_box_pass or settings.QUEUE_EMAIL_BOX_PASSWORD)
-
-        messagesInfo = server.list()[1]
-        logger.info("Received %d messages from POP3 server" % len(messagesInfo))
-
-        for msgRaw in messagesInfo:
-            if six.PY3 and type(msgRaw) is bytes:
-                # in py3, msgRaw may be a bytes object, decode to str
-                try:
-                    msg = msgRaw.decode("utf-8")
-                except UnicodeError:
-                    # if couldn't decode easily, just leave it raw
-                    msg = msgRaw
-            else:
-                # already a str
-                msg = msgRaw
-            msgNum = msg.split(" ")[0]
-            logger.info("Processing message %s" % msgNum)
-
-            if six.PY2:
-                full_message = encoding.force_text("\n".join(server.retr(msgNum)[1]), errors='replace')
-            else:
-                raw_content = server.retr(msgNum)[1]
-                if type(raw_content[0]) is bytes:
-                    full_message = "\n".join([elm.decode('utf-8') for elm in raw_content])
-                else:
-                    full_message = encoding.force_text("\n".join(raw_content), errors='replace')
-            ticket = ticket_from_message(message=full_message, queue=q, logger=logger)
-
-            if ticket:
-                server.dele(msgNum)
-                logger.info("Successfully processed message %s, deleted from POP3 server" % msgNum)
-            else:
-                logger.warn("Message %s was not successfully processed, and will be left on POP3 server" % msgNum)
-
-        server.quit()
-
-    elif email_box_type == 'imap':
-        if q.email_box_ssl or settings.QUEUE_EMAIL_BOX_SSL:
-            if not q.email_box_port:
-                q.email_box_port = 993
-            server = imaplib.IMAP4_SSL(q.email_box_host or
-                                       settings.QUEUE_EMAIL_BOX_HOST,
-                                       int(q.email_box_port))
-        else:
-            if not q.email_box_port:
-                q.email_box_port = 143
-            server = imaplib.IMAP4(q.email_box_host or
-                                   settings.QUEUE_EMAIL_BOX_HOST,
-                                   int(q.email_box_port))
-
-        logger.info("Attempting IMAP server login")
-
-        try:
-            server.login(q.email_box_user or
-                         settings.QUEUE_EMAIL_BOX_USER,
-                         q.email_box_pass or
-                         settings.QUEUE_EMAIL_BOX_PASSWORD)
-            server.select(q.email_box_imap_folder)
-        except imaplib.IMAP4.abort:
-            logger.error("IMAP login failed. Check that the server is accessible and that the username and password are correct.")
-            server.logout()
-            sys.exit()
-        except ssl.SSLError:
-            logger.error("IMAP login failed due to SSL error. This is often due to a timeout. Please check your connection and try again.")
-            server.logout()
-            sys.exit()
-
-        try:
-            status, data = server.search(None, 'NOT', 'DELETED')
-        except imaplib.IMAP4.error:
-            logger.error("IMAP retrieve failed. Is the folder '%s' spelled correctly, and does it exist on the server?" % q.email_box_imap_folder)
-        if data:
-            msgnums = data[0].split()
-            logger.info("Received %d messages from IMAP server" % len(msgnums))
-            for num in msgnums:
-                logger.info("Processing message %s" % num)
-                status, data = server.fetch(num, '(RFC822)')
-                full_message = encoding.force_text(data[0][1], errors='replace')
-                try:
-                    ticket = ticket_from_message(message=full_message, queue=q, logger=logger)
-                except TypeError:
-                    ticket = None  # hotfix. Need to work out WHY.
-                if ticket:
-                    server.store(num, '+FLAGS', '\\Deleted')
-                    logger.info("Successfully processed message %s, deleted from IMAP server" % num)
-                else:
-                    logger.warn("Message %s was not successfully processed, and will be left on IMAP server" % num)
-
-        server.expunge()
-        server.close()
-        server.logout()
-
-    elif email_box_type == 'local':
-        mail_dir = q.email_box_local_dir or '/var/lib/mail/helpdesk/'
-        mail = [join(mail_dir, f) for f in listdir(mail_dir) if isfile(join(mail_dir, f))]
-        logger.info("Found %d messages in local mailbox directory" % len(mail))
-
-        logger.info("Found %d messages in local mailbox directory" % len(mail))
-        for i, m in enumerate(mail, 1):
-            logger.info("Processing message %d" % i)
-            with open(m, 'r') as f:
-                full_message = encoding.force_text(f.read(), errors='replace')
-                ticket = ticket_from_message(message=full_message, queue=q, logger=logger)
-            if ticket:
-                logger.info("Successfully processed message %d, ticket/comment created." % i)
-                try:
-                    unlink(m)  # delete message file if ticket was successful
-                except OSError:
-                    logger.error("Unable to delete message %d." % i)
-                else:
-                    logger.info("Successfully deleted message %d." % i)
-            else:
-                logger.warn("Message %d was not successfully processed, and will be left in local directory" % i)
-
-
-def decodeUnknown(charset, string):
-    if six.PY2:
-        if not charset:
-            try:
-                return string.decode('utf-8', 'replace')
-            except UnicodeError:
-                return string.decode('iso8859-1', 'replace')
-        return unicode(string, charset)
-    elif six.PY3:
-        if type(string) is not str:
-            if not charset:
-                try:
-                    return str(string, encoding='utf-8', errors='replace')
-                except UnicodeError:
-                    return str(string, encoding='iso8859-1', errors='replace')
-            return str(string, encoding=charset, errors='replace')
-        return string
-
-
-def decode_mail_headers(string):
-    decoded = email.header.decode_header(string) if six.PY3 else email.header.decode_header(string.encode('utf-8'))
-    if six.PY2:
-        return u' '.join([unicode(msg, charset or 'utf-8') for msg, charset in decoded])
-    elif six.PY3:
-        return u' '.join([str(msg, encoding=charset, errors='replace') if charset else str(msg) for msg, charset in decoded])
-
-
-def ticket_from_message(message, queue, logger):
-    # 'message' must be an RFC822 formatted message.
-    message = email.message_from_string(message) if six.PY3 else email.message_from_string(message.encode('utf-8'))
-    subject = message.get('subject', _('Comment from e-mail'))
-    subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject))
-    for affix in STRIPPED_SUBJECT_STRINGS:
-        subject = subject.replace(affix, "")
-    subject = subject.strip()
-
-    sender = message.get('from', _('Unknown Sender'))
-    sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender))
-    sender_email = email.utils.parseaddr(sender)[1]
-
-    cc = message.get_all('cc', None)
-    if cc:
-        # first, fixup the encoding if necessary
-        cc = [decode_mail_headers(decodeUnknown(message.get_charset(), x)) for x in cc]
-        # get_all checks if multiple CC headers, but individual emails may be comma separated too
-        tempcc = []
-        for hdr in cc:
-            tempcc.extend(hdr.split(','))
-        # use a set to ensure no duplicates
-        cc = set([x.strip() for x in tempcc])
-
-    for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)):
-        if ignore.test(sender_email):
-            if ignore.keep_in_mailbox:
-                # By returning 'False' the message will be kept in the mailbox,
-                # and the 'True' will cause the message to be deleted.
-                return False
-            return True
-
-    matchobj = re.match(r".*\[" + queue.slug + r"-(?P<id>\d+)\]", subject)
-    if matchobj:
-        # This is a reply or forward.
-        ticket = matchobj.group('id')
-        logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket))
-    else:
-        logger.info("No tracking ID matched.")
-        ticket = None
-
-    body = None
-    counter = 0
-    files = []
-
-    for part in message.walk():
-        if part.get_content_maintype() == 'multipart':
-            continue
-
-        name = part.get_param("name")
-        if name:
-            name = email.utils.collapse_rfc2231_value(name)
-
-        if part.get_content_maintype() == 'text' and name is None:
-            if part.get_content_subtype() == 'plain':
-                body = EmailReplyParser.parse_reply(
-                    decodeUnknown(part.get_content_charset(), part.get_payload(decode=True))
-                )
-                # workaround to get unicode text out rather than escaped text
-                try:
-                    body = body.encode('ascii').decode('unicode_escape')
-                except UnicodeEncodeError:
-                    body.encode('utf-8')
-                logger.debug("Discovered plain text MIME part")
-            else:
-                files.append(
-                    SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html')
-                )
-                logger.debug("Discovered HTML MIME part")
-        else:
-            if not name:
-                ext = mimetypes.guess_extension(part.get_content_type())
-                name = "part-%i%s" % (counter, ext)
-            payload = part.get_payload()
-            if isinstance(payload, list):
-                payload = payload.pop().as_string()
-            payloadToWrite = payload
-            # check version of python to ensure use of only the correct error type
-            if six.PY2:
-                non_b64_err = binascii.Error
-            else:
-                non_b64_err = TypeError
-            try:
-                logger.debug("Try to base64 decode the attachment payload")
-                if six.PY2:
-                    payloadToWrite = base64.decodestring(payload)
-                else:
-                    payloadToWrite = base64.decodebytes(payload)
-            except non_b64_err:
-                logger.debug("Payload was not base64 encoded, using raw bytes")
-                payloadToWrite = payload
-            files.append(SimpleUploadedFile(name, part.get_payload(decode=True), mimetypes.guess_type(name)[0]))
-            logger.debug("Found MIME attachment %s" % name)
-
-        counter += 1
-
-    if not body:
-        mail = BeautifulSoup(part.get_payload(), "lxml")
-        if ">" in mail.text:
-            body = mail.find('body')
-            body = body.text
-            body = body.encode('ascii', errors='ignore')
-        else:
-            body = mail.text
-
-    if ticket:
-        try:
-            t = Ticket.objects.get(id=ticket)
-        except Ticket.DoesNotExist:
-            logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket))
-            ticket = None
-        else:
-            logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id))
-            if t.status == Ticket.CLOSED_STATUS:
-                t.status = Ticket.REOPENED_STATUS
-                t.save()
-            new = False
-
-    smtp_priority = message.get('priority', '')
-    smtp_importance = message.get('importance', '')
-    high_priority_types = {'high', 'important', '1', 'urgent'}
-    priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3
-
-    if ticket is None:
-        if settings.QUEUE_EMAIL_BOX_UPDATE_ONLY:
-            return None
-        new = True
-        t = Ticket.objects.create(
-            title=subject,
-            queue=queue,
-            submitter_email=sender_email,
-            created=timezone.now(),
-            description=body,
-            priority=priority,
-        )
-        logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id))
-
-    if cc:
-        # get list of currently CC'd emails
-        current_cc = TicketCC.objects.filter(ticket=ticket)
-        current_cc_emails = [x.email for x in current_cc if x.email]
-        # get emails of any Users CC'd to email, if defined
-        # (some Users may not have an associated email, e.g, when using LDAP)
-        current_cc_users = [x.user.email for x in current_cc if x.user and x.user.email]
-        # ensure submitter, assigned user, queue email not added
-        other_emails = [queue.email_address]
-        if t.submitter_email:
-            other_emails.append(t.submitter_email)
-        if t.assigned_to:
-            other_emails.append(t.assigned_to.email)
-        current_cc = set(current_cc_emails + current_cc_users + other_emails)
-        # first, add any User not previously CC'd (as identified by User's email)
-        all_users = User.objects.all()
-        all_user_emails = set([x.email for x in all_users])
-        users_not_currently_ccd = all_user_emails.difference(set(current_cc))
-        users_to_cc = cc.intersection(users_not_currently_ccd)
-        for user in users_to_cc:
-            tcc = TicketCC.objects.create(
-                ticket=t,
-                user=User.objects.get(email=user),
-                can_view=True,
-                can_update=False
-            )
-            tcc.save()
-        # then add remaining emails alphabetically, makes testing easy
-        new_cc = cc.difference(current_cc).difference(all_user_emails)
-        new_cc = sorted(list(new_cc))
-        for ccemail in new_cc:
-            tcc = TicketCC.objects.create(
-                ticket=t,
-                email=ccemail.replace('\n', ' ').replace('\r', ' '),
-                can_view=True,
-                can_update=False
-            )
-            tcc.save()
-
-    f = FollowUp(
-        ticket=t,
-        title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}),
-        date=timezone.now(),
-        public=True,
-        comment=body,
-    )
-
-    if t.status == Ticket.REOPENED_STATUS:
-        f.new_status = Ticket.REOPENED_STATUS
-        f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email})
-
-    f.save()
-    logger.debug("Created new FollowUp for Ticket")
-
-    if six.PY2:
-        logger.info(("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace'))
-    elif six.PY3:
-        logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,))
-
-    attached = process_attachments(f, files)
-    for att_file in attached:
-        logger.info("Attachment '%s' (with size %s) successfully added to ticket from email." % (att_file[0], att_file[1].size))
-
-    context = safe_template_context(t)
-
-    if new:
-        if sender_email:
-            send_templated_mail(
-                'newticket_submitter',
-                context,
-                recipients=sender_email,
-                sender=queue.from_address,
-                fail_silently=True,
-            )
-        if queue.new_ticket_cc:
-            send_templated_mail(
-                'newticket_cc',
-                context,
-                recipients=queue.new_ticket_cc,
-                sender=queue.from_address,
-                fail_silently=True,
-            )
-        if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc:
-            send_templated_mail(
-                'newticket_cc',
-                context,
-                recipients=queue.updated_ticket_cc,
-                sender=queue.from_address,
-                fail_silently=True,
-            )
-    else:
-        context.update(comment=f.comment)
-        if t.assigned_to:
-            send_templated_mail(
-                'updated_owner',
-                context,
-                recipients=t.assigned_to.email,
-                sender=queue.from_address,
-                fail_silently=True,
-            )
-        if queue.updated_ticket_cc:
-            send_templated_mail(
-                'updated_cc',
-                context,
-                recipients=queue.updated_ticket_cc,
-                sender=queue.from_address,
-                fail_silently=True,
-            )
-
-    return t
-
-
 if __name__ == '__main__':
     process_email()

helpdesk/models.py

@@ -23,6 +23,8 @@ import re
 import six
 import uuid
 
+from .templated_email import send_templated_mail
+
 
 @python_2_unicode_compatible
 class Queue(models.Model):
@@ -491,6 +493,54 @@ class Ticket(models.Model):
         default=mk_secret,
     )
 
+    def send(self, roles, dont_send_to=None, **kwargs):
+        """
+        Send notifications to everyone interested in this ticket.
+
+        The the roles argument is a dictionary mapping from roles to (template, context) pairs.
+        If a role is not present in the dictionary, users of that type will not recieve the notification.
+
+        The following roles exist:
+
+          - 'submitter'
+          - 'new_ticket_cc'
+          - 'ticket_cc'
+          - 'assigned_to'
+
+        Here is an example roles dictionary:
+
+        {
+            'submitter': (template_name, context),
+            'assigned_to': (template_name2, context),
+        }
+
+        **kwargs are passed to send_templated_mail defined in templated_mail.py
+
+        returns the set of email addresses the notification was delivered to.
+
+        """
+        recipients = set()
+
+        if dont_send_to is not None:
+            recipients.update(dont_send_to)
+
+        def should_receive(email):
+            return email and email not in recipients
+
+        def send(role, recipient):
+            if recipient and recipient not in recipients and role in roles:
+                template, context = roles[role]
+                send_templated_mail(template, context, recipient, sender=self.queue.from_address, **kwargs)
+                recipients.add(recipient)
+        send('submitter', self.submitter_email)
+        send('new_ticket_cc', self.queue.new_ticket_cc)
+        if self.assigned_to and self.assigned_to.usersettings_helpdesk.email_on_ticket_assign:
+            send('assigned_to', self.assigned_to.email)
+        send('ticket_cc', self.queue.updated_ticket_cc)
+        for cc in self.ticketcc_set.all():
+            send('ticket_cc', cc.email_address)
+        return recipients
+
     def _get_assigned_to(self):
         """ Custom property to allow us to easily print 'Unassigned' if a
         ticket has no owner, or the users name if it's assigned. If the user

helpdesk/templated_email.py

@@ -0,0 +1,114 @@
+import os
+import mimetypes
+import logging
+from smtplib import SMTPException
+
+from django.utils.safestring import mark_safe
+
+logger = logging.getLogger('helpdesk')
+
+
+def send_templated_mail(template_name,
+                        context,
+                        recipients,
+                        sender=None,
+                        bcc=None,
+                        fail_silently=False,
+                        files=None):
+    """
+    send_templated_mail() is a wrapper around Django's e-mail routines that
+    allows us to easily send multipart (text/plain & text/html) e-mails using
+    templates that are stored in the database. This lets the admin provide
+    both a text and a HTML template for each message.
+
+    template_name is the slug of the template to use for this message (see
+        models.EmailTemplate)
+
+    context is a dictionary to be used when rendering the template
+
+    recipients can be either a string, eg 'a@b.com', or a list of strings.
+
+    sender should contain a string, eg 'My Site <me@z.com>'. If you leave it
+        blank, it'll use settings.DEFAULT_FROM_EMAIL as a fallback.
+
+    bcc is an optional list of addresses that will receive this message as a
+        blind carbon copy.
+
+    fail_silently is passed to Django's mail routine. Set to 'True' to ignore
+        any errors at send time.
+
+    files can be a list of tuples. Each tuple should be a filename to attach,
+        along with the File objects to be read. files can be blank.
+
+    """
+    from django.core.mail import EmailMultiAlternatives
+    from django.template import engines
+    from_string = engines['django'].from_string
+
+    from helpdesk.models import EmailTemplate
+    from helpdesk.settings import HELPDESK_EMAIL_SUBJECT_TEMPLATE, \
+        HELPDESK_EMAIL_FALLBACK_LOCALE
+
+    locale = context['queue'].get('locale') or HELPDESK_EMAIL_FALLBACK_LOCALE
+
+    try:
+        t = EmailTemplate.objects.get(template_name__iexact=template_name, locale=locale)
+    except EmailTemplate.DoesNotExist:
+        try:
+            t = EmailTemplate.objects.get(template_name__iexact=template_name, locale__isnull=True)
+        except EmailTemplate.DoesNotExist:
+            logger.warning('template "%s" does not exist, no mail sent', template_name)
+            return  # just ignore if template doesn't exist
+
+    subject_part = from_string(
+        HELPDESK_EMAIL_SUBJECT_TEMPLATE % {
+            "subject": t.subject
+        }).render(context).replace('\n', '').replace('\r', '')
+
+    footer_file = os.path.join('helpdesk', locale, 'email_text_footer.txt')
+
+    text_part = from_string(
+        "%s{%% include '%s' %%}" % (t.plain_text, footer_file)
+    ).render(context)
+
+    email_html_base_file = os.path.join('helpdesk', locale, 'email_html_base.html')
+    # keep new lines in html emails
+    if 'comment' in context:
+        context['comment'] = mark_safe(context['comment'].replace('\r\n', '<br>'))
+
+    html_part = from_string(
+        "{%% extends '%s' %%}{%% block title %%}"
+        "%s"
+        "{%% endblock %%}{%% block content %%}%s{%% endblock %%}" %
+        (email_html_base_file, t.heading, t.html)
+    ).render(context)
+
+    if isinstance(recipients, str):
+        if recipients.find(','):
+            recipients = recipients.split(',')
+    elif type(recipients) != list:
+        recipients = [recipients]
+
+    msg = EmailMultiAlternatives(subject_part, text_part,
+                                 sender or settings.DEFAULT_FROM_EMAIL,
+                                 recipients, bcc=bcc)
+    msg.attach_alternative(html_part, "text/html")
+
+    if files:
+        for filename, filefield in files:
+            mime = mimetypes.guess_type(filename)
+            if mime[0] is not None and mime[0] == "text/plain":
+                with open(filefield.path, 'r') as attachedfile:
+                    content = attachedfile.read()
+                    msg.attach(filename, content)
+            else:
+                msg.attach_file(filefield.path)
+    logger.debug('Sending email to: {!r}'.format(recipients))
+
+    try:
+        return msg.send()
+    except SMTPException as e:
+        logger.exception('SMTPException raised while sending email to {}'.format(recipients))
+        if not fail_silently:
+            raise e
+        return 0

helpdesk/tests/test_get_email.py

@@ -98,8 +98,8 @@ class GetEmailParametricTemplate(object):
         else:
             # Test local email reading
             if self.method == 'local':
-                with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \
+                with mock.patch('helpdesk.email.listdir') as mocked_listdir, \
-                        mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \
+                        mock.patch('helpdesk.email.isfile') as mocked_isfile, \
                         mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)):
                     mocked_isfile.return_value = True
                     mocked_listdir.return_value = ['filename1', 'filename2']
@@ -120,7 +120,7 @@ class GetEmailParametricTemplate(object):
                 mocked_poplib_server = mock.Mock()
                 mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list)
                 mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails[x])
-                with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib:
+                with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib:
                     mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server)
                     call_command('get_email')
 
@@ -136,7 +136,7 @@ class GetEmailParametricTemplate(object):
 
                 # we ignore the second arg as the data item/mime-part is constant (RFC822)
                 mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x])
-                with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib:
+                with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib:
                     mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server)
                     call_command('get_email')
 
@@ -171,8 +171,8 @@ class GetEmailParametricTemplate(object):
         else:
             # Test local email reading
             if self.method == 'local':
-                with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \
+                with mock.patch('helpdesk.email.listdir') as mocked_listdir, \
-                        mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \
+                        mock.patch('helpdesk.email.isfile') as mocked_isfile, \
                         mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)):
                     mocked_isfile.return_value = True
                     mocked_listdir.return_value = ['filename1', 'filename2']
@@ -193,7 +193,7 @@ class GetEmailParametricTemplate(object):
                 mocked_poplib_server = mock.Mock()
                 mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list)
                 mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails[x])
-                with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib:
+                with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib:
                     mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server)
                     call_command('get_email')
 
@@ -209,7 +209,7 @@ class GetEmailParametricTemplate(object):
 
                 # we ignore the second arg as the data item/mime-part is constant (RFC822)
                 mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x])
-                with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib:
+                with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib:
                     mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server)
                     call_command('get_email')
 
@@ -284,8 +284,8 @@ class GetEmailParametricTemplate(object):
         else:
             # Test local email reading
             if self.method == 'local':
-                with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \
+                with mock.patch('helpdesk.email.listdir') as mocked_listdir, \
-                        mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \
+                        mock.patch('helpdesk.email.isfile') as mocked_isfile, \
                         mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=msg.as_string())):
                     mocked_isfile.return_value = True
                     mocked_listdir.return_value = ['filename1', 'filename2']
@@ -306,7 +306,7 @@ class GetEmailParametricTemplate(object):
                 mocked_poplib_server = mock.Mock()
                 mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list)
                 mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails[x])
-                with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib:
+                with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib:
                     mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server)
                     call_command('get_email')
 
@@ -322,7 +322,7 @@ class GetEmailParametricTemplate(object):
 
                 # we ignore the second arg as the data item/mime-part is constant (RFC822)
                 mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x])
-                with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib:
+                with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib:
                     mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server)
                     call_command('get_email')
 
@@ -531,8 +531,8 @@ a9eiiQ+3V1v+7wWHXCzq
         else:
             # Test local email reading
             if self.method == 'local':
-                with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \
+                with mock.patch('helpdesk.email.listdir') as mocked_listdir, \
-                        mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \
+                        mock.patch('helpdesk.email.isfile') as mocked_isfile, \
                         mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)):
                     mocked_isfile.return_value = True
                     mocked_listdir.return_value = ['filename1']
@@ -551,7 +551,7 @@ a9eiiQ+3V1v+7wWHXCzq
                 mocked_poplib_server = mock.Mock()
                 mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list)
                 mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails['1'])
-                with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib:
+                with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib:
                     mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server)
                     call_command('get_email')
 
@@ -566,7 +566,7 @@ a9eiiQ+3V1v+7wWHXCzq
 
                 # we ignore the second arg as the data item/mime-part is constant (RFC822)
                 mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x])
-                with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib:
+                with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib:
                     mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server)
                     call_command('get_email')
 
@@ -701,8 +701,8 @@ class GetEmailCCHandling(TestCase):
         test_email = "To: queue@example.com\nCc: " + test_email_cc_one + ", " + test_email_cc_one + ", " + test_email_cc_two + ", " + test_email_cc_three + "\nCC: " + test_email_cc_one + ", " + test_email_cc_three + ", " + test_email_cc_four + ", " + ticket_user_emails + "\nFrom: " + test_email_from + "\nSubject: " + test_email_subject + "\n\n" + test_email_body
         test_mail_len = len(test_email)
 
-        with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \
+        with mock.patch('helpdesk.email.listdir') as mocked_listdir, \
-                mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \
+                mock.patch('helpdesk.email.isfile') as mocked_isfile, \
                 mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)):
 
             mocked_isfile.return_value = True

helpdesk/views/staff.py

@@ -47,7 +47,7 @@ from helpdesk.forms import (
     TicketCCEmailForm, TicketCCUserForm, EditFollowUpForm, TicketDependencyForm
 )
 from helpdesk.lib import (
-    send_templated_mail, query_to_dict, apply_query, safe_template_context,
+    query_to_dict, apply_query, safe_template_context,
     process_attachments, queue_template_context,
 )
 from helpdesk.models import (
@@ -578,8 +578,6 @@ def update_ticket(request, ticket_id, public=False):
         if new_status == Ticket.RESOLVED_STATUS or ticket.resolution is None:
             ticket.resolution = comment
 
-    messages_sent_to = []
-
     # ticket might have changed above, so we re-instantiate context with the
     # (possibly) updated ticket.
     context = safe_template_context(ticket)
@@ -588,6 +586,11 @@ def update_ticket(request, ticket_id, public=False):
         comment=f.comment,
     )
 
+    messages_sent_to = set()
+    try:
+        messages_sent_to.add(request.user.email)
+    except AttributeError:
+        pass
     if public and (f.comment or (
         f.new_status in (Ticket.RESOLVED_STATUS,
                          Ticket.CLOSED_STATUS))):
@@ -598,83 +601,44 @@ def update_ticket(request, ticket_id, public=False):
         else:
             template = 'updated_'
 
-        template_suffix = 'submitter'
+        roles = {
+            'submitter': (template + 'submitter', context),
+            'ticket_cc': (template + 'cc', context),
+            'assigned_to': (template + 'cc', context),
+        }
+        messages_sent_to.update(ticket.send(roles, dont_send_to=messages_sent_to, fail_silently=True, files=files,))
 
-        if ticket.submitter_email:
+    if reassigned:
-            send_templated_mail(
+        template_staff = 'assigned_owner'
-                template + template_suffix,
+    elif f.new_status == Ticket.RESOLVED_STATUS:
-                context,
+        template_staff = 'resolved_owner'
-                recipients=ticket.submitter_email,
+    elif f.new_status == Ticket.CLOSED_STATUS:
-                sender=ticket.queue.from_address,
+        template_staff = 'closed_owner'
-                fail_silently=True,
+    else:
-                files=files,
+        template_staff = 'updated_owner'
-            )
-            messages_sent_to.append(ticket.submitter_email)
 
-        template_suffix = 'cc'
+    messages_sent_to.update(ticket.send(
+        {'assigned_to': (template_staff, context)},
+        dont_send_to=messages_sent_to,
+        fail_silently=True,
+        files=files,
+    ))
 
-        for cc in ticket.ticketcc_set.all():
+    if reassigned:
-            if cc.email_address not in messages_sent_to:
+        template_cc = 'assigned_cc'
-                send_templated_mail(
+    elif f.new_status == Ticket.RESOLVED_STATUS:
-                    template + template_suffix,
+        template_cc = 'resolved_cc'
-                    context,
+    elif f.new_status == Ticket.CLOSED_STATUS:
-                    recipients=cc.email_address,
+        template_cc = 'closed_cc'
-                    sender=ticket.queue.from_address,
+    else:
-                    fail_silently=True,
+        template_cc = 'updated_cc'
-                    files=files,
-                )
-                messages_sent_to.append(cc.email_address)
 
-    if ticket.assigned_to and \
+    messages_sent_to.update(ticket.send(
-            request.user != ticket.assigned_to and \
+        {'ticket_cc': (template_cc, context)},
-            ticket.assigned_to.email and \
+        dont_send_to=messages_sent_to,
-            ticket.assigned_to.email not in messages_sent_to:
+        fail_silently=True,
-        # We only send e-mails to staff members if the ticket is updated by
+        files=files,
-        # another user. The actual template varies, depending on what has been
+    ))
-        # changed.
-        if reassigned:
-            template_staff = 'assigned_owner'
-        elif f.new_status == Ticket.RESOLVED_STATUS:
-            template_staff = 'resolved_owner'
-        elif f.new_status == Ticket.CLOSED_STATUS:
-            template_staff = 'closed_owner'
-        else:
-            template_staff = 'updated_owner'
-
-        if (not reassigned or
-                (reassigned and
-                    ticket.assigned_to.usersettings_helpdesk.email_on_ticket_assign)) or \
-            (not reassigned and
-                ticket.assigned_to.usersettings_helpdesk.email_on_ticket_change):
-
-            send_templated_mail(
-                template_staff,
-                context,
-                recipients=ticket.assigned_to.email,
-                sender=ticket.queue.from_address,
-                fail_silently=True,
-                files=files,
-            )
-            messages_sent_to.append(ticket.assigned_to.email)
-
-    if ticket.queue.updated_ticket_cc and ticket.queue.updated_ticket_cc not in messages_sent_to:
-        if reassigned:
-            template_cc = 'assigned_cc'
-        elif f.new_status == Ticket.RESOLVED_STATUS:
-            template_cc = 'resolved_cc'
-        elif f.new_status == Ticket.CLOSED_STATUS:
-            template_cc = 'closed_cc'
-        else:
-            template_cc = 'updated_cc'
-
-        send_templated_mail(
-            template_cc,
-            context,
-            recipients=ticket.queue.updated_ticket_cc,
-            sender=ticket.queue.from_address,
-            fail_silently=True,
-            files=files,
-        )
 
     ticket.save()
 
@@ -760,51 +724,19 @@ def mass_update(request):
             context.update(resolution=t.resolution,
                            queue=queue_template_context(t.queue))
 
-            messages_sent_to = []
+            messages_sent_to = set()
+            try:
+                messages_sent_to.add(request.user.email)
+            except AttributeError:
+                pass
 
-            if t.submitter_email:
+            messages_sent_to.update(t.send(
-                send_templated_mail(
+                {'submitter': ('closed_submitter', context),
-                    'closed_submitter',
+                 'ticket_cc': ('closed_cc', context),
-                    context,
+                 'assigned_to': ('closded_owner', context)},
-                    recipients=t.submitter_email,
+                dont_send_to=messages_sent_to,
-                    sender=t.queue.from_address,
+                fail_silently=True,
-                    fail_silently=True,
+            ))
-                )
-                messages_sent_to.append(t.submitter_email)
-
-            for cc in t.ticketcc_set.all():
-                if cc.email_address not in messages_sent_to:
-                    send_templated_mail(
-                        'closed_submitter',
-                        context,
-                        recipients=cc.email_address,
-                        sender=t.queue.from_address,
-                        fail_silently=True,
-                    )
-                    messages_sent_to.append(cc.email_address)
-
-            if t.assigned_to and \
-                    request.user != t.assigned_to and \
-                    t.assigned_to.email and \
-                    t.assigned_to.email not in messages_sent_to:
-                send_templated_mail(
-                    'closed_owner',
-                    context,
-                    recipients=t.assigned_to.email,
-                    sender=t.queue.from_address,
-                    fail_silently=True,
-                )
-                messages_sent_to.append(t.assigned_to.email)
-
-            if t.queue.updated_ticket_cc and \
-                    t.queue.updated_ticket_cc not in messages_sent_to:
-                send_templated_mail(
-                    'closed_cc',
-                    context,
-                    recipients=t.queue.updated_ticket_cc,
-                    sender=t.queue.from_address,
-                    fail_silently=True,
-                )
 
         elif action == 'delete':
             t.delete()