From 9a45d28c959881ef8aa99467610a6248dc52feb8 Mon Sep 17 00:00:00 2001 From: Timothy Hobbs Date: Wed, 24 Oct 2018 18:20:12 +0200 Subject: [PATCH] More refactors of get_email.py --- helpdesk/email.py | 509 +++++++++++++++++++++ helpdesk/management/commands/get_email.py | 513 +--------------------- helpdesk/tests/test_get_email.py | 36 +- 3 files changed, 528 insertions(+), 530 deletions(-) create mode 100644 helpdesk/email.py diff --git a/helpdesk/email.py b/helpdesk/email.py new file mode 100644 index 00000000..1b0df203 --- /dev/null +++ b/helpdesk/email.py @@ -0,0 +1,509 @@ +#!/usr/bin/python +""" +Django Helpdesk - A Django powered ticket tracker for small enterprise. + +(c) Copyright 2008 Jutda. Copyright 2018 Timothy Hobbs. All Rights Reserved. +See LICENSE for details. + +scripts/get_email.py - Designed to be run from cron, this script checks the + POP and IMAP boxes, or a local mailbox directory, + defined for the queues within a + helpdesk, creating tickets from the new messages (or + adding to existing tickets if needed) +""" + +from datetime import timedelta +import base64 +import binascii +import email +import imaplib +import mimetypes +from os import listdir, unlink +from os.path import isfile, join +import poplib +import re +import socket +import ssl +import sys +from time import ctime + +from bs4 import BeautifulSoup + +from email_reply_parser import EmailReplyParser + +from django.core.files.base import ContentFile +from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.management.base import BaseCommand +from django.db.models import Q +from django.utils.translation import ugettext as _ +from django.utils import encoding, timezone + +from helpdesk import settings +from helpdesk.lib import send_templated_mail, safe_template_context, process_attachments +from helpdesk.models import Queue, Ticket, TicketCC, FollowUp, IgnoreEmail +from django.contrib.auth.models import User + +import logging + + +STRIPPED_SUBJECT_STRINGS = [ + "Re: ", + "Fw: ", + "RE: ", + "FW: ", + "Automatic reply: ", +] + +def process_email(quiet=False): + for q in Queue.objects.filter( + email_box_type__isnull=False, + allow_email_submission=True): + + logger = logging.getLogger('django.helpdesk.queue.' + q.slug) + logging_types = { + 'info': logging.INFO, + 'warn': logging.WARN, + 'error': logging.ERROR, + 'crit': logging.CRITICAL, + 'debug': logging.DEBUG, + } + if q.logging_type in logging_types: + logger.setLevel(logging_types[q.logging_type]) + elif not q.logging_type or q.logging_type == 'none': + logging.disable(logging.CRITICAL) # disable all messages + if quiet: + logger.propagate = False # do not propagate to root logger that would log to console + logdir = q.logging_dir or '/var/log/helpdesk/' + handler = logging.FileHandler(join(logdir, q.slug + '_get_email.log')) + logger.addHandler(handler) + + if not q.email_box_last_check: + q.email_box_last_check = timezone.now() - timedelta(minutes=30) + + queue_time_delta = timedelta(minutes=q.email_box_interval or 0) + + if (q.email_box_last_check + queue_time_delta) < timezone.now(): + process_queue(q, logger=logger) + q.email_box_last_check = timezone.now() + q.save() + + +def pop3_sync(q, logger, server): + server.getwelcome() + server.user(q.email_box_user or settings.QUEUE_EMAIL_BOX_USER) + server.pass_(q.email_box_pass or settings.QUEUE_EMAIL_BOX_PASSWORD) + + messagesInfo = server.list()[1] + logger.info("Received %d messages from POP3 server" % len(messagesInfo)) + + for msgRaw in messagesInfo: + if type(msgRaw) is bytes: + try: + msg = msgRaw.decode("utf-8") + except UnicodeError: + # if couldn't decode easily, just leave it raw + msg = msgRaw + else: + # already a str + msg = msgRaw + msgNum = msg.split(" ")[0] + logger.info("Processing message %s" % msgNum) + + raw_content = server.retr(msgNum)[1] + if type(raw_content[0]) is bytes: + full_message = "\n".join([elm.decode('utf-8') for elm in raw_content]) + else: + full_message = encoding.force_text("\n".join(raw_content), errors='replace') + ticket = ticket_from_message(message=full_message, queue=q, logger=logger) + + if ticket: + server.dele(msgNum) + logger.info("Successfully processed message %s, deleted from POP3 server" % msgNum) + else: + logger.warn("Message %s was not successfully processed, and will be left on POP3 server" % msgNum) + + server.quit() + + +def imap_sync(q, logger, server): + try: + server.login(q.email_box_user or + settings.QUEUE_EMAIL_BOX_USER, + q.email_box_pass or + settings.QUEUE_EMAIL_BOX_PASSWORD) + server.select(q.email_box_imap_folder) + except imaplib.IMAP4.abort: + logger.error("IMAP login failed. Check that the server is accessible and that the username and password are correct.") + server.logout() + sys.exit() + except ssl.SSLError: + logger.error("IMAP login failed due to SSL error. This is often due to a timeout. Please check your connection and try again.") + server.logout() + sys.exit() + + try: + status, data = server.search(None, 'NOT', 'DELETED') + except imaplib.IMAP4.error: + logger.error("IMAP retrieve failed. Is the folder '%s' spelled correctly, and does it exist on the server?" % q.email_box_imap_folder) + if data: + msgnums = data[0].split() + logger.info("Received %d messages from IMAP server" % len(msgnums)) + for num in msgnums: + logger.info("Processing message %s" % num) + status, data = server.fetch(num, '(RFC822)') + full_message = encoding.force_text(data[0][1], errors='replace') + try: + ticket = ticket_from_message(message=full_message, queue=q, logger=logger) + except TypeError: + ticket = None # hotfix. Need to work out WHY. + if ticket: + server.store(num, '+FLAGS', '\\Deleted') + logger.info("Successfully processed message %s, deleted from IMAP server" % num) + else: + logger.warn("Message %s was not successfully processed, and will be left on IMAP server" % num) + + server.expunge() + server.close() + server.logout() + + +def process_queue(q, logger): + logger.info("***** %s: Begin processing mail for django-helpdesk" % ctime()) + + if q.socks_proxy_type and q.socks_proxy_host and q.socks_proxy_port: + try: + import socks + except ImportError: + no_socks_msg = "Queue has been configured with proxy settings, " \ + "but no socks library was installed. Try to " \ + "install PySocks via PyPI." + logger.error(no_socks_msg) + raise ImportError(no_socks_msg) + + proxy_type = { + 'socks4': socks.SOCKS4, + 'socks5': socks.SOCKS5, + }.get(q.socks_proxy_type) + + socks.set_default_proxy(proxy_type=proxy_type, + addr=q.socks_proxy_host, + port=q.socks_proxy_port) + socket.socket = socks.socksocket + + email_box_type = settings.QUEUE_EMAIL_BOX_TYPE or q.email_box_type + + mail_defaults = { + 'pop3': { + 'ssl': { + 'port': 995, + 'init': poplib.POP3_SSL, + }, + 'insecure': { + 'port': 110, + 'init': poplib.POP3, + }, + 'sync': pop3_sync, + }, + 'imap': { + 'ssl': { + 'port': 993, + 'init': imaplib.IMAP4_SSL, + }, + 'insecure': { + 'port': 143, + 'init': imaplib.IMAP4, + }, + 'sync': imap_sync + } + } + if email_box_type in mail_defaults: + encryption = 'insecure' + if q.email_box_ssl or settings.QUEUE_EMAIL_BOX_SSL: + encryption = 'ssl' + if not q.email_box_port: + q.email_box_port = mail_defaults[email_box_type][encryption]['port'] + + server = mail_defaults[email_box_type][encryption]['init']( + q.email_box_host or settings.QUEUE_EMAIL_BOX_HOST, + int(q.email_box_port) + ) + logger.info("Attempting %s server login" % email_box_type.upper()) + mail_defaults[email_box_type]['sync'](q, logger, server) + + elif email_box_type == 'local': + mail_dir = q.email_box_local_dir or '/var/lib/mail/helpdesk/' + mail = [join(mail_dir, f) for f in listdir(mail_dir) if isfile(join(mail_dir, f))] + logger.info("Found %d messages in local mailbox directory" % len(mail)) + + logger.info("Found %d messages in local mailbox directory" % len(mail)) + for i, m in enumerate(mail, 1): + logger.info("Processing message %d" % i) + with open(m, 'r') as f: + full_message = encoding.force_text(f.read(), errors='replace') + ticket = ticket_from_message(message=full_message, queue=q, logger=logger) + if ticket: + logger.info("Successfully processed message %d, ticket/comment created." % i) + try: + unlink(m) # delete message file if ticket was successful + except OSError: + logger.error("Unable to delete message %d." % i) + else: + logger.info("Successfully deleted message %d." % i) + else: + logger.warn("Message %d was not successfully processed, and will be left in local directory" % i) + + +def decodeUnknown(charset, string): + if type(string) is not str: + if not charset: + try: + return str(string, encoding='utf-8', errors='replace') + except UnicodeError: + return str(string, encoding='iso8859-1', errors='replace') + return str(string, encoding=charset, errors='replace') + return string + + +def decode_mail_headers(string): + decoded = email.header.decode_header(string) + return u' '.join([str(msg, encoding=charset, errors='replace') if charset else str(msg) for msg, charset in decoded]) + + +def ticket_from_message(message, queue, logger): + # 'message' must be an RFC822 formatted message. + message = email.message_from_string(message) + subject = message.get('subject', _('Comment from e-mail')) + subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject)) + for affix in STRIPPED_SUBJECT_STRINGS: + subject = subject.replace(affix, "") + subject = subject.strip() + + sender = message.get('from', _('Unknown Sender')) + sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender)) + sender_email = email.utils.parseaddr(sender)[1] + + cc = message.get_all('cc', None) + if cc: + # first, fixup the encoding if necessary + cc = [decode_mail_headers(decodeUnknown(message.get_charset(), x)) for x in cc] + # get_all checks if multiple CC headers, but individual emails may be comma separated too + tempcc = [] + for hdr in cc: + tempcc.extend(hdr.split(',')) + # use a set to ensure no duplicates + cc = set([x.strip() for x in tempcc]) + + for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)): + if ignore.test(sender_email): + if ignore.keep_in_mailbox: + # By returning 'False' the message will be kept in the mailbox, + # and the 'True' will cause the message to be deleted. + return False + return True + + matchobj = re.match(r".*\[" + queue.slug + r"-(?P\d+)\]", subject) + if matchobj: + # This is a reply or forward. + ticket = matchobj.group('id') + logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket)) + else: + logger.info("No tracking ID matched.") + ticket = None + + body = None + counter = 0 + files = [] + + for part in message.walk(): + if part.get_content_maintype() == 'multipart': + continue + + name = part.get_param("name") + if name: + name = email.utils.collapse_rfc2231_value(name) + + if part.get_content_maintype() == 'text' and name is None: + if part.get_content_subtype() == 'plain': + body = EmailReplyParser.parse_reply( + decodeUnknown(part.get_content_charset(), part.get_payload(decode=True)) + ) + # workaround to get unicode text out rather than escaped text + try: + body = body.encode('ascii').decode('unicode_escape') + except UnicodeEncodeError: + body.encode('utf-8') + logger.debug("Discovered plain text MIME part") + else: + files.append( + SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html') + ) + logger.debug("Discovered HTML MIME part") + else: + if not name: + ext = mimetypes.guess_extension(part.get_content_type()) + name = "part-%i%s" % (counter, ext) + payload = part.get_payload() + if isinstance(payload, list): + payload = payload.pop().as_string() + payloadToWrite = payload + # check version of python to ensure use of only the correct error type + non_b64_err = TypeError + try: + logger.debug("Try to base64 decode the attachment payload") + payloadToWrite = base64.decodebytes(payload) + except non_b64_err: + logger.debug("Payload was not base64 encoded, using raw bytes") + payloadToWrite = payload + files.append(SimpleUploadedFile(name, part.get_payload(decode=True), mimetypes.guess_type(name)[0])) + logger.debug("Found MIME attachment %s" % name) + + counter += 1 + + if not body: + mail = BeautifulSoup(part.get_payload(), "lxml") + if ">" in mail.text: + body = mail.find('body') + body = body.text + body = body.encode('ascii', errors='ignore') + else: + body = mail.text + + if ticket: + try: + t = Ticket.objects.get(id=ticket) + except Ticket.DoesNotExist: + logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket)) + ticket = None + else: + logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id)) + if t.status == Ticket.CLOSED_STATUS: + t.status = Ticket.REOPENED_STATUS + t.save() + new = False + + smtp_priority = message.get('priority', '') + smtp_importance = message.get('importance', '') + high_priority_types = {'high', 'important', '1', 'urgent'} + priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3 + + if ticket is None: + if settings.QUEUE_EMAIL_BOX_UPDATE_ONLY: + return None + new = True + t = Ticket.objects.create( + title=subject, + queue=queue, + submitter_email=sender_email, + created=timezone.now(), + description=body, + priority=priority, + ) + logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id)) + + if cc: + # get list of currently CC'd emails + current_cc = TicketCC.objects.filter(ticket=ticket) + current_cc_emails = [x.email for x in current_cc if x.email] + # get emails of any Users CC'd to email, if defined + # (some Users may not have an associated email, e.g, when using LDAP) + current_cc_users = [x.user.email for x in current_cc if x.user and x.user.email] + # ensure submitter, assigned user, queue email not added + other_emails = [queue.email_address] + if t.submitter_email: + other_emails.append(t.submitter_email) + if t.assigned_to: + other_emails.append(t.assigned_to.email) + current_cc = set(current_cc_emails + current_cc_users + other_emails) + # first, add any User not previously CC'd (as identified by User's email) + all_users = User.objects.all() + all_user_emails = set([x.email for x in all_users]) + users_not_currently_ccd = all_user_emails.difference(set(current_cc)) + users_to_cc = cc.intersection(users_not_currently_ccd) + for user in users_to_cc: + tcc = TicketCC.objects.create( + ticket=t, + user=User.objects.get(email=user), + can_view=True, + can_update=False + ) + tcc.save() + # then add remaining emails alphabetically, makes testing easy + new_cc = cc.difference(current_cc).difference(all_user_emails) + new_cc = sorted(list(new_cc)) + for ccemail in new_cc: + tcc = TicketCC.objects.create( + ticket=t, + email=ccemail.replace('\n', ' ').replace('\r', ' '), + can_view=True, + can_update=False + ) + tcc.save() + + f = FollowUp( + ticket=t, + title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}), + date=timezone.now(), + public=True, + comment=body, + ) + + if t.status == Ticket.REOPENED_STATUS: + f.new_status = Ticket.REOPENED_STATUS + f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}) + + f.save() + logger.debug("Created new FollowUp for Ticket") + + logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)) + + attached = process_attachments(f, files) + for att_file in attached: + logger.info("Attachment '%s' (with size %s) successfully added to ticket from email." % (att_file[0], att_file[1].size)) + + context = safe_template_context(t) + + if new: + if sender_email: + send_templated_mail( + 'newticket_submitter', + context, + recipients=sender_email, + sender=queue.from_address, + fail_silently=True, + ) + if queue.new_ticket_cc: + send_templated_mail( + 'newticket_cc', + context, + recipients=queue.new_ticket_cc, + sender=queue.from_address, + fail_silently=True, + ) + if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc: + send_templated_mail( + 'newticket_cc', + context, + recipients=queue.updated_ticket_cc, + sender=queue.from_address, + fail_silently=True, + ) + else: + context.update(comment=f.comment) + if t.assigned_to: + send_templated_mail( + 'updated_owner', + context, + recipients=t.assigned_to.email, + sender=queue.from_address, + fail_silently=True, + ) + if queue.updated_ticket_cc: + send_templated_mail( + 'updated_cc', + context, + recipients=queue.updated_ticket_cc, + sender=queue.from_address, + fail_silently=True, + ) + + return t diff --git a/helpdesk/management/commands/get_email.py b/helpdesk/management/commands/get_email.py index 79a575b2..ce344bd5 100755 --- a/helpdesk/management/commands/get_email.py +++ b/helpdesk/management/commands/get_email.py @@ -10,49 +10,9 @@ scripts/get_email.py - Designed to be run from cron, this script checks the helpdesk, creating tickets from the new messages (or adding to existing tickets if needed) """ -from __future__ import unicode_literals - -from datetime import timedelta -import base64 -import binascii -import email -import imaplib -import mimetypes -from os import listdir, unlink -from os.path import isfile, join -import poplib -import re -import socket -import ssl -import sys -from time import ctime - -from bs4 import BeautifulSoup - -from email_reply_parser import EmailReplyParser - -from django.core.files.base import ContentFile -from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management.base import BaseCommand -from django.db.models import Q -from django.utils.translation import ugettext as _ -from django.utils import encoding, six, timezone -from helpdesk import settings -from helpdesk.lib import send_templated_mail, safe_template_context, process_attachments -from helpdesk.models import Queue, Ticket, TicketCC, FollowUp, IgnoreEmail -from django.contrib.auth.models import User - -import logging - - -STRIPPED_SUBJECT_STRINGS = [ - "Re: ", - "Fw: ", - "RE: ", - "FW: ", - "Automatic reply: ", -] +from helpdesk.email import process_email class Command(BaseCommand): @@ -77,476 +37,5 @@ class Command(BaseCommand): process_email(quiet=quiet) -def process_email(quiet=False): - for q in Queue.objects.filter( - email_box_type__isnull=False, - allow_email_submission=True): - - logger = logging.getLogger('django.helpdesk.queue.' + q.slug) - logging_types = { - 'info': logging.INFO, - 'warn': logging.WARN, - 'error': logging.ERROR, - 'crit': logging.CRITICAL, - 'debug': logging.DEBUG, - } - if q.logging_type in logging_types: - logger.setLevel(logging_types[q.logging_type]) - elif not q.logging_type or q.logging_type == 'none': - logging.disable(logging.CRITICAL) # disable all messages - if quiet: - logger.propagate = False # do not propagate to root logger that would log to console - logdir = q.logging_dir or '/var/log/helpdesk/' - handler = logging.FileHandler(join(logdir, q.slug + '_get_email.log')) - logger.addHandler(handler) - - if not q.email_box_last_check: - q.email_box_last_check = timezone.now() - timedelta(minutes=30) - - queue_time_delta = timedelta(minutes=q.email_box_interval or 0) - - if (q.email_box_last_check + queue_time_delta) < timezone.now(): - process_queue(q, logger=logger) - q.email_box_last_check = timezone.now() - q.save() - - -def process_queue(q, logger): - logger.info("***** %s: Begin processing mail for django-helpdesk" % ctime()) - - if q.socks_proxy_type and q.socks_proxy_host and q.socks_proxy_port: - try: - import socks - except ImportError: - no_socks_msg = "Queue has been configured with proxy settings, " \ - "but no socks library was installed. Try to " \ - "install PySocks via PyPI." - logger.error(no_socks_msg) - raise ImportError(no_socks_msg) - - proxy_type = { - 'socks4': socks.SOCKS4, - 'socks5': socks.SOCKS5, - }.get(q.socks_proxy_type) - - socks.set_default_proxy(proxy_type=proxy_type, - addr=q.socks_proxy_host, - port=q.socks_proxy_port) - socket.socket = socks.socksocket - elif six.PY2: - socket.socket = socket._socketobject - - email_box_type = settings.QUEUE_EMAIL_BOX_TYPE or q.email_box_type - - if email_box_type == 'pop3': - if q.email_box_ssl or settings.QUEUE_EMAIL_BOX_SSL: - if not q.email_box_port: - q.email_box_port = 995 - server = poplib.POP3_SSL(q.email_box_host or - settings.QUEUE_EMAIL_BOX_HOST, - int(q.email_box_port)) - else: - if not q.email_box_port: - q.email_box_port = 110 - server = poplib.POP3(q.email_box_host or - settings.QUEUE_EMAIL_BOX_HOST, - int(q.email_box_port)) - - logger.info("Attempting POP3 server login") - - server.getwelcome() - server.user(q.email_box_user or settings.QUEUE_EMAIL_BOX_USER) - server.pass_(q.email_box_pass or settings.QUEUE_EMAIL_BOX_PASSWORD) - - messagesInfo = server.list()[1] - logger.info("Received %d messages from POP3 server" % len(messagesInfo)) - - for msgRaw in messagesInfo: - if six.PY3 and type(msgRaw) is bytes: - # in py3, msgRaw may be a bytes object, decode to str - try: - msg = msgRaw.decode("utf-8") - except UnicodeError: - # if couldn't decode easily, just leave it raw - msg = msgRaw - else: - # already a str - msg = msgRaw - msgNum = msg.split(" ")[0] - logger.info("Processing message %s" % msgNum) - - if six.PY2: - full_message = encoding.force_text("\n".join(server.retr(msgNum)[1]), errors='replace') - else: - raw_content = server.retr(msgNum)[1] - if type(raw_content[0]) is bytes: - full_message = "\n".join([elm.decode('utf-8') for elm in raw_content]) - else: - full_message = encoding.force_text("\n".join(raw_content), errors='replace') - ticket = ticket_from_message(message=full_message, queue=q, logger=logger) - - if ticket: - server.dele(msgNum) - logger.info("Successfully processed message %s, deleted from POP3 server" % msgNum) - else: - logger.warn("Message %s was not successfully processed, and will be left on POP3 server" % msgNum) - - server.quit() - - elif email_box_type == 'imap': - if q.email_box_ssl or settings.QUEUE_EMAIL_BOX_SSL: - if not q.email_box_port: - q.email_box_port = 993 - server = imaplib.IMAP4_SSL(q.email_box_host or - settings.QUEUE_EMAIL_BOX_HOST, - int(q.email_box_port)) - else: - if not q.email_box_port: - q.email_box_port = 143 - server = imaplib.IMAP4(q.email_box_host or - settings.QUEUE_EMAIL_BOX_HOST, - int(q.email_box_port)) - - logger.info("Attempting IMAP server login") - - try: - server.login(q.email_box_user or - settings.QUEUE_EMAIL_BOX_USER, - q.email_box_pass or - settings.QUEUE_EMAIL_BOX_PASSWORD) - server.select(q.email_box_imap_folder) - except imaplib.IMAP4.abort: - logger.error("IMAP login failed. Check that the server is accessible and that the username and password are correct.") - server.logout() - sys.exit() - except ssl.SSLError: - logger.error("IMAP login failed due to SSL error. This is often due to a timeout. Please check your connection and try again.") - server.logout() - sys.exit() - - try: - status, data = server.search(None, 'NOT', 'DELETED') - except imaplib.IMAP4.error: - logger.error("IMAP retrieve failed. Is the folder '%s' spelled correctly, and does it exist on the server?" % q.email_box_imap_folder) - if data: - msgnums = data[0].split() - logger.info("Received %d messages from IMAP server" % len(msgnums)) - for num in msgnums: - logger.info("Processing message %s" % num) - status, data = server.fetch(num, '(RFC822)') - full_message = encoding.force_text(data[0][1], errors='replace') - try: - ticket = ticket_from_message(message=full_message, queue=q, logger=logger) - except TypeError: - ticket = None # hotfix. Need to work out WHY. - if ticket: - server.store(num, '+FLAGS', '\\Deleted') - logger.info("Successfully processed message %s, deleted from IMAP server" % num) - else: - logger.warn("Message %s was not successfully processed, and will be left on IMAP server" % num) - - server.expunge() - server.close() - server.logout() - - elif email_box_type == 'local': - mail_dir = q.email_box_local_dir or '/var/lib/mail/helpdesk/' - mail = [join(mail_dir, f) for f in listdir(mail_dir) if isfile(join(mail_dir, f))] - logger.info("Found %d messages in local mailbox directory" % len(mail)) - - logger.info("Found %d messages in local mailbox directory" % len(mail)) - for i, m in enumerate(mail, 1): - logger.info("Processing message %d" % i) - with open(m, 'r') as f: - full_message = encoding.force_text(f.read(), errors='replace') - ticket = ticket_from_message(message=full_message, queue=q, logger=logger) - if ticket: - logger.info("Successfully processed message %d, ticket/comment created." % i) - try: - unlink(m) # delete message file if ticket was successful - except OSError: - logger.error("Unable to delete message %d." % i) - else: - logger.info("Successfully deleted message %d." % i) - else: - logger.warn("Message %d was not successfully processed, and will be left in local directory" % i) - - -def decodeUnknown(charset, string): - if six.PY2: - if not charset: - try: - return string.decode('utf-8', 'replace') - except UnicodeError: - return string.decode('iso8859-1', 'replace') - return unicode(string, charset) - elif six.PY3: - if type(string) is not str: - if not charset: - try: - return str(string, encoding='utf-8', errors='replace') - except UnicodeError: - return str(string, encoding='iso8859-1', errors='replace') - return str(string, encoding=charset, errors='replace') - return string - - -def decode_mail_headers(string): - decoded = email.header.decode_header(string) if six.PY3 else email.header.decode_header(string.encode('utf-8')) - if six.PY2: - return u' '.join([unicode(msg, charset or 'utf-8') for msg, charset in decoded]) - elif six.PY3: - return u' '.join([str(msg, encoding=charset, errors='replace') if charset else str(msg) for msg, charset in decoded]) - - -def ticket_from_message(message, queue, logger): - # 'message' must be an RFC822 formatted message. - message = email.message_from_string(message) if six.PY3 else email.message_from_string(message.encode('utf-8')) - subject = message.get('subject', _('Comment from e-mail')) - subject = decode_mail_headers(decodeUnknown(message.get_charset(), subject)) - for affix in STRIPPED_SUBJECT_STRINGS: - subject = subject.replace(affix, "") - subject = subject.strip() - - sender = message.get('from', _('Unknown Sender')) - sender = decode_mail_headers(decodeUnknown(message.get_charset(), sender)) - sender_email = email.utils.parseaddr(sender)[1] - - cc = message.get_all('cc', None) - if cc: - # first, fixup the encoding if necessary - cc = [decode_mail_headers(decodeUnknown(message.get_charset(), x)) for x in cc] - # get_all checks if multiple CC headers, but individual emails may be comma separated too - tempcc = [] - for hdr in cc: - tempcc.extend(hdr.split(',')) - # use a set to ensure no duplicates - cc = set([x.strip() for x in tempcc]) - - for ignore in IgnoreEmail.objects.filter(Q(queues=queue) | Q(queues__isnull=True)): - if ignore.test(sender_email): - if ignore.keep_in_mailbox: - # By returning 'False' the message will be kept in the mailbox, - # and the 'True' will cause the message to be deleted. - return False - return True - - matchobj = re.match(r".*\[" + queue.slug + r"-(?P\d+)\]", subject) - if matchobj: - # This is a reply or forward. - ticket = matchobj.group('id') - logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket)) - else: - logger.info("No tracking ID matched.") - ticket = None - - body = None - counter = 0 - files = [] - - for part in message.walk(): - if part.get_content_maintype() == 'multipart': - continue - - name = part.get_param("name") - if name: - name = email.utils.collapse_rfc2231_value(name) - - if part.get_content_maintype() == 'text' and name is None: - if part.get_content_subtype() == 'plain': - body = EmailReplyParser.parse_reply( - decodeUnknown(part.get_content_charset(), part.get_payload(decode=True)) - ) - # workaround to get unicode text out rather than escaped text - try: - body = body.encode('ascii').decode('unicode_escape') - except UnicodeEncodeError: - body.encode('utf-8') - logger.debug("Discovered plain text MIME part") - else: - files.append( - SimpleUploadedFile(_("email_html_body.html"), encoding.smart_bytes(part.get_payload()), 'text/html') - ) - logger.debug("Discovered HTML MIME part") - else: - if not name: - ext = mimetypes.guess_extension(part.get_content_type()) - name = "part-%i%s" % (counter, ext) - payload = part.get_payload() - if isinstance(payload, list): - payload = payload.pop().as_string() - payloadToWrite = payload - # check version of python to ensure use of only the correct error type - if six.PY2: - non_b64_err = binascii.Error - else: - non_b64_err = TypeError - try: - logger.debug("Try to base64 decode the attachment payload") - if six.PY2: - payloadToWrite = base64.decodestring(payload) - else: - payloadToWrite = base64.decodebytes(payload) - except non_b64_err: - logger.debug("Payload was not base64 encoded, using raw bytes") - payloadToWrite = payload - files.append(SimpleUploadedFile(name, part.get_payload(decode=True), mimetypes.guess_type(name)[0])) - logger.debug("Found MIME attachment %s" % name) - - counter += 1 - - if not body: - mail = BeautifulSoup(part.get_payload(), "lxml") - if ">" in mail.text: - body = mail.find('body') - body = body.text - body = body.encode('ascii', errors='ignore') - else: - body = mail.text - - if ticket: - try: - t = Ticket.objects.get(id=ticket) - except Ticket.DoesNotExist: - logger.info("Tracking ID %s-%s not associated with existing ticket. Creating new ticket." % (queue.slug, ticket)) - ticket = None - else: - logger.info("Found existing ticket with Tracking ID %s-%s" % (t.queue.slug, t.id)) - if t.status == Ticket.CLOSED_STATUS: - t.status = Ticket.REOPENED_STATUS - t.save() - new = False - - smtp_priority = message.get('priority', '') - smtp_importance = message.get('importance', '') - high_priority_types = {'high', 'important', '1', 'urgent'} - priority = 2 if high_priority_types & {smtp_priority, smtp_importance} else 3 - - if ticket is None: - if settings.QUEUE_EMAIL_BOX_UPDATE_ONLY: - return None - new = True - t = Ticket.objects.create( - title=subject, - queue=queue, - submitter_email=sender_email, - created=timezone.now(), - description=body, - priority=priority, - ) - logger.debug("Created new ticket %s-%s" % (t.queue.slug, t.id)) - - if cc: - # get list of currently CC'd emails - current_cc = TicketCC.objects.filter(ticket=ticket) - current_cc_emails = [x.email for x in current_cc if x.email] - # get emails of any Users CC'd to email, if defined - # (some Users may not have an associated email, e.g, when using LDAP) - current_cc_users = [x.user.email for x in current_cc if x.user and x.user.email] - # ensure submitter, assigned user, queue email not added - other_emails = [queue.email_address] - if t.submitter_email: - other_emails.append(t.submitter_email) - if t.assigned_to: - other_emails.append(t.assigned_to.email) - current_cc = set(current_cc_emails + current_cc_users + other_emails) - # first, add any User not previously CC'd (as identified by User's email) - all_users = User.objects.all() - all_user_emails = set([x.email for x in all_users]) - users_not_currently_ccd = all_user_emails.difference(set(current_cc)) - users_to_cc = cc.intersection(users_not_currently_ccd) - for user in users_to_cc: - tcc = TicketCC.objects.create( - ticket=t, - user=User.objects.get(email=user), - can_view=True, - can_update=False - ) - tcc.save() - # then add remaining emails alphabetically, makes testing easy - new_cc = cc.difference(current_cc).difference(all_user_emails) - new_cc = sorted(list(new_cc)) - for ccemail in new_cc: - tcc = TicketCC.objects.create( - ticket=t, - email=ccemail.replace('\n', ' ').replace('\r', ' '), - can_view=True, - can_update=False - ) - tcc.save() - - f = FollowUp( - ticket=t, - title=_('E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}), - date=timezone.now(), - public=True, - comment=body, - ) - - if t.status == Ticket.REOPENED_STATUS: - f.new_status = Ticket.REOPENED_STATUS - f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}) - - f.save() - logger.debug("Created new FollowUp for Ticket") - - if six.PY2: - logger.info(("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace')) - elif six.PY3: - logger.info("[%s-%s] %s" % (t.queue.slug, t.id, t.title,)) - - attached = process_attachments(f, files) - for att_file in attached: - logger.info("Attachment '%s' (with size %s) successfully added to ticket from email." % (att_file[0], att_file[1].size)) - - context = safe_template_context(t) - - if new: - if sender_email: - send_templated_mail( - 'newticket_submitter', - context, - recipients=sender_email, - sender=queue.from_address, - fail_silently=True, - ) - if queue.new_ticket_cc: - send_templated_mail( - 'newticket_cc', - context, - recipients=queue.new_ticket_cc, - sender=queue.from_address, - fail_silently=True, - ) - if queue.updated_ticket_cc and queue.updated_ticket_cc != queue.new_ticket_cc: - send_templated_mail( - 'newticket_cc', - context, - recipients=queue.updated_ticket_cc, - sender=queue.from_address, - fail_silently=True, - ) - else: - context.update(comment=f.comment) - if t.assigned_to: - send_templated_mail( - 'updated_owner', - context, - recipients=t.assigned_to.email, - sender=queue.from_address, - fail_silently=True, - ) - if queue.updated_ticket_cc: - send_templated_mail( - 'updated_cc', - context, - recipients=queue.updated_ticket_cc, - sender=queue.from_address, - fail_silently=True, - ) - - return t - - if __name__ == '__main__': process_email() diff --git a/helpdesk/tests/test_get_email.py b/helpdesk/tests/test_get_email.py index 583eaddd..fb97708c 100644 --- a/helpdesk/tests/test_get_email.py +++ b/helpdesk/tests/test_get_email.py @@ -98,8 +98,8 @@ class GetEmailParametricTemplate(object): else: # Test local email reading if self.method == 'local': - with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \ - mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \ + with mock.patch('helpdesk.email.listdir') as mocked_listdir, \ + mock.patch('helpdesk.email.isfile') as mocked_isfile, \ mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)): mocked_isfile.return_value = True mocked_listdir.return_value = ['filename1', 'filename2'] @@ -120,7 +120,7 @@ class GetEmailParametricTemplate(object): mocked_poplib_server = mock.Mock() mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list) mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib: + with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib: mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server) call_command('get_email') @@ -136,7 +136,7 @@ class GetEmailParametricTemplate(object): # we ignore the second arg as the data item/mime-part is constant (RFC822) mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib: + with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib: mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server) call_command('get_email') @@ -171,8 +171,8 @@ class GetEmailParametricTemplate(object): else: # Test local email reading if self.method == 'local': - with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \ - mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \ + with mock.patch('helpdesk.email.listdir') as mocked_listdir, \ + mock.patch('helpdesk.email.isfile') as mocked_isfile, \ mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)): mocked_isfile.return_value = True mocked_listdir.return_value = ['filename1', 'filename2'] @@ -193,7 +193,7 @@ class GetEmailParametricTemplate(object): mocked_poplib_server = mock.Mock() mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list) mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib: + with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib: mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server) call_command('get_email') @@ -209,7 +209,7 @@ class GetEmailParametricTemplate(object): # we ignore the second arg as the data item/mime-part is constant (RFC822) mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib: + with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib: mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server) call_command('get_email') @@ -284,8 +284,8 @@ class GetEmailParametricTemplate(object): else: # Test local email reading if self.method == 'local': - with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \ - mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \ + with mock.patch('helpdesk.email.listdir') as mocked_listdir, \ + mock.patch('helpdesk.email.isfile') as mocked_isfile, \ mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=msg.as_string())): mocked_isfile.return_value = True mocked_listdir.return_value = ['filename1', 'filename2'] @@ -306,7 +306,7 @@ class GetEmailParametricTemplate(object): mocked_poplib_server = mock.Mock() mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list) mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib: + with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib: mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server) call_command('get_email') @@ -322,7 +322,7 @@ class GetEmailParametricTemplate(object): # we ignore the second arg as the data item/mime-part is constant (RFC822) mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib: + with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib: mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server) call_command('get_email') @@ -531,8 +531,8 @@ a9eiiQ+3V1v+7wWHXCzq else: # Test local email reading if self.method == 'local': - with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \ - mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \ + with mock.patch('helpdesk.email.listdir') as mocked_listdir, \ + mock.patch('helpdesk.email.isfile') as mocked_isfile, \ mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)): mocked_isfile.return_value = True mocked_listdir.return_value = ['filename1'] @@ -551,7 +551,7 @@ a9eiiQ+3V1v+7wWHXCzq mocked_poplib_server = mock.Mock() mocked_poplib_server.list = mock.Mock(return_value=pop3_mail_list) mocked_poplib_server.retr = mock.Mock(side_effect=lambda x: pop3_emails['1']) - with mock.patch('helpdesk.management.commands.get_email.poplib', autospec=True) as mocked_poplib: + with mock.patch('helpdesk.email.poplib', autospec=True) as mocked_poplib: mocked_poplib.POP3 = mock.Mock(return_value=mocked_poplib_server) call_command('get_email') @@ -566,7 +566,7 @@ a9eiiQ+3V1v+7wWHXCzq # we ignore the second arg as the data item/mime-part is constant (RFC822) mocked_imaplib_server.fetch = mock.Mock(side_effect=lambda x, _: imap_emails[x]) - with mock.patch('helpdesk.management.commands.get_email.imaplib', autospec=True) as mocked_imaplib: + with mock.patch('helpdesk.email.imaplib', autospec=True) as mocked_imaplib: mocked_imaplib.IMAP4 = mock.Mock(return_value=mocked_imaplib_server) call_command('get_email') @@ -701,8 +701,8 @@ class GetEmailCCHandling(TestCase): test_email = "To: queue@example.com\nCc: " + test_email_cc_one + ", " + test_email_cc_one + ", " + test_email_cc_two + ", " + test_email_cc_three + "\nCC: " + test_email_cc_one + ", " + test_email_cc_three + ", " + test_email_cc_four + ", " + ticket_user_emails + "\nFrom: " + test_email_from + "\nSubject: " + test_email_subject + "\n\n" + test_email_body test_mail_len = len(test_email) - with mock.patch('helpdesk.management.commands.get_email.listdir') as mocked_listdir, \ - mock.patch('helpdesk.management.commands.get_email.isfile') as mocked_isfile, \ + with mock.patch('helpdesk.email.listdir') as mocked_listdir, \ + mock.patch('helpdesk.email.isfile') as mocked_isfile, \ mock.patch('builtins.open' if six.PY3 else '__builtin__.open', mock.mock_open(read_data=test_email)): mocked_isfile.return_value = True