diff --git a/helpdesk/management/commands/get_email.py b/helpdesk/management/commands/get_email.py index 99f45f73..b8f752dd 100644 --- a/helpdesk/management/commands/get_email.py +++ b/helpdesk/management/commands/get_email.py @@ -5,18 +5,23 @@ Jutda Helpdesk - A Django powered ticket tracker for small enterprise. (c) Copyright 2008 Jutda. All Rights Reserved. See LICENSE for details. scripts/get_email.py - Designed to be run from cron, this script checks the - POP and IMAP boxes defined for the queues within a + POP and IMAP boxes, or a local mailbox directory, + defined for the queues within a helpdesk, creating tickets from the new messages (or adding to existing tickets if needed) """ from __future__ import print_function import email +import codecs import imaplib import mimetypes import poplib import re import socket +import six +from os import listdir, unlink +from os.path import isfile, join from datetime import timedelta from email.header import decode_header @@ -25,6 +30,7 @@ from optparse import make_option from email_reply_parser import EmailReplyParser +from django import VERSION from django.core.files.base import ContentFile from django.core.management.base import BaseCommand from django.db.models import Q @@ -39,12 +45,17 @@ except ImportError: from helpdesk.lib import send_templated_mail, safe_template_context from helpdesk.models import Queue, Ticket, FollowUp, Attachment, IgnoreEmail +import logging +from time import ctime + class Command(BaseCommand): def __init__(self): BaseCommand.__init__(self) - self.option_list += ( + # Django 1.7 uses different way to specify options than 1.8+ + if VERSION < (1, 8): + self.option_list += ( make_option( '--quiet', '-q', default=False, @@ -52,7 +63,16 @@ class Command(BaseCommand): help='Hide details about each queue/message as they are processed'), ) - help = 'Process Jutda Helpdesk queues and process e-mails via POP3/IMAP as required, feeding them into the helpdesk.' + help = 'Process django-helpdesk queues and process e-mails via POP3/IMAP or from a local mailbox directory as required, feeding them into the helpdesk.' + + def add_arguments(self, parser): + parser.add_argument( + '--quiet', + action='store_true', + dest='quiet', + default=False, + help='Hide details about each queue/message as they are processed', + ) def handle(self, *args, **options): quiet = options.get('quiet', False) @@ -64,6 +84,25 @@ def process_email(quiet=False): email_box_type__isnull=False, allow_email_submission=True): + logger = logging.getLogger('django.helpdesk.queue.' + q.slug) + if not q.logging_type or q.logging_type == 'none': + logging.disable(logging.CRITICAL) #disable all messages + elif q.logging_type == 'info': + logger.setLevel(logging.INFO) + elif q.logging_type == 'warn': + logger.setLevel(logging.WARN) + elif q.logging_type == 'error': + logger.setLevel(logging.ERROR) + elif q.logging_type == 'crit': + logger.setLevel(logging.CRITICAL) + elif q.logging_type == 'debug': + logger.setLevel(logging.DEBUG) + if quiet: + logger.propagate = False # do not propagate to root logger that would log to console + logdir = q.logging_dir or '/var/log/helpdesk/' + handler = logging.FileHandler(logdir + q.slug + '_get_email.log') + logger.addHandler(handler) + if not q.email_box_last_check: q.email_box_last_check = timezone.now()-timedelta(minutes=30) @@ -76,20 +115,20 @@ def process_email(quiet=False): if (q.email_box_last_check + queue_time_delta) > timezone.now(): continue - process_queue(q, quiet=quiet) + process_queue(q, logger=logger) q.email_box_last_check = timezone.now() q.save() -def process_queue(q, quiet=False): - if not quiet: - print("Processing: %s" % q) +def process_queue(q, logger): + logger.info("***** %s: Begin processing mail for django-helpdesk" % ctime()) if q.socks_proxy_type and q.socks_proxy_host and q.socks_proxy_port: try: import socks except ImportError: + logger.error("Queue has been configured with proxy settings, but no socks library was installed. Try to install PySocks via pypi.") raise ImportError("Queue has been configured with proxy settings, but no socks library was installed. Try to install PySocks via pypi.") proxy_type = { @@ -100,7 +139,11 @@ def process_queue(q, quiet=False): socks.set_default_proxy(proxy_type=proxy_type, addr=q.socks_proxy_host, port=q.socks_proxy_port) socket.socket = socks.socksocket else: - socket.socket = socket._socketobject + if six.PY2: + socket.socket = socket._socketobject + elif six.PY3: + import _socket + socket.socket = _socket.socket email_box_type = settings.QUEUE_EMAIL_BOX_TYPE if settings.QUEUE_EMAIL_BOX_TYPE else q.email_box_type @@ -113,22 +156,29 @@ def process_queue(q, quiet=False): if not q.email_box_port: q.email_box_port = 110 server = poplib.POP3(q.email_box_host or settings.QUEUE_EMAIL_BOX_HOST, int(q.email_box_port)) + logger.info("Attempting POP3 server login") + server.getwelcome() server.user(q.email_box_user or settings.QUEUE_EMAIL_BOX_USER) server.pass_(q.email_box_pass or settings.QUEUE_EMAIL_BOX_PASSWORD) messagesInfo = server.list()[1] + logger.info("Received %s messages from POP3 server" % str(len(messagesInfo))) for msg in messagesInfo: msgNum = msg.split(" ")[0] msgSize = msg.split(" ")[1] + logger.info("Processing message %s" % str(msgNum)) full_message = "\n".join(server.retr(msgNum)[1]) - ticket = ticket_from_message(message=full_message, queue=q, quiet=quiet) + ticket = ticket_from_message(message=full_message, queue=q) if ticket: server.dele(msgNum) + logger.info("Successfully processed message %s, deleted from POP3 server" % str(msgNum)) + else: + logger.warn("Message %s was not successfully processed, and will be left on POP3 server" % str(msgNum)) server.quit() @@ -140,36 +190,74 @@ def process_queue(q, quiet=False): if not q.email_box_port: q.email_box_port = 143 server = imaplib.IMAP4(q.email_box_host or settings.QUEUE_EMAIL_BOX_HOST, int(q.email_box_port)) + logger.info("Attempting IMAP server login") + server.login(q.email_box_user or settings.QUEUE_EMAIL_BOX_USER, q.email_box_pass or settings.QUEUE_EMAIL_BOX_PASSWORD) server.select(q.email_box_imap_folder) status, data = server.search(None, 'NOT', 'DELETED') if data: msgnums = data[0].split() + logger.info("Received %s messages from IMAP server" % str(len(msgnums))) for num in msgnums: + logger.info("Processing message %s" % str(num)) status, data = server.fetch(num, '(RFC822)') - ticket = ticket_from_message(message=data[0][1], queue=q, quiet=quiet) + ticket = ticket_from_message(message=data[0][1], queue=q) if ticket: server.store(num, '+FLAGS', '\\Deleted') - + logger.info("Successfully processed message %s, deleted from IMAP server" % str(msgNum)) + else: + logger.warn("Message %s was not successfully processed, and will be left on IMAP server" % str(msgNum)) + server.expunge() server.close() server.logout() + elif email_box_type == 'local': + mail_dir = q.email_box_local_dir or '/var/lib/mail/helpdesk/' + mail = [join(mail_dir, f) for f in listdir(mail_dir) if isfile(join(mail_dir, f))] + logger.info("Found %s messages in local mailbox directory" % str(len(mail))) + for m in mail: + logger.info("Processing message %s" % str(m)) + f = open(m, 'r') + ticket = ticket_from_message(message=f.read(), queue=q, logger=logger) + if ticket: + logger.info("Successfully processed message %s, ticket/comment created." % str(m)) + try: + #unlink(m) #delete message file if ticket was successful + logger.info("Successfully deleted message %s." % str(m)) + except: + logger.error("Unable to delete message %s." % str(m)) + else: + logger.warn("Message %s was not successfully processed, and will be left in local directory" % str(m)) + def decodeUnknown(charset, string): - if not charset: - try: - return string.decode('utf-8','ignore') - except: - return string.decode('iso8859-1','ignore') - return unicode(string, charset) + if six.PY2: + if not charset: + try: + return string.decode('utf-8','ignore') + except: + return string.decode('iso8859-1','ignore') + return unicode(string, charset) + elif six.PY3: + if type(string) is not str: + if not charset: + try: + return codecs.decode(bytes(string),'utf-8','ignore') + except: + return codecs.decode(bytes(string),'iso8859-1','ignore') + return str(string, charset) + return string def decode_mail_headers(string): decoded = decode_header(string) - return u' '.join([unicode(msg, charset or 'utf-8') for msg, charset in decoded]) + if six.PY2: + return u' '.join([unicode(msg, charset or 'utf-8') for msg, charset in decoded]) + elif six.PY3: + return u' '.join([str(msg) for msg, charset in decoded]) -def ticket_from_message(message, queue, quiet): +def ticket_from_message(message, queue, logger): # 'message' must be an RFC822 formatted message. msg = message message = email.message_from_string(msg) @@ -196,7 +284,9 @@ def ticket_from_message(message, queue, quiet): if matchobj: # This is a reply or forward. ticket = matchobj.group('id') + logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket)) else: + logger.info("No tracking ID matched, assuming new ticket") ticket = None counter = 0 @@ -231,7 +321,7 @@ def ticket_from_message(message, queue, quiet): if body_plain: body = body_plain else: - body = _('No plain-text email body available. Please see attachment email_html_body.html.') + body = _('No plain-text email body available. Please see attachment "email_html_body.html".') if body_html: files.append({ @@ -247,6 +337,7 @@ def ticket_from_message(message, queue, quiet): t = Ticket.objects.get(id=ticket) new = False except Ticket.DoesNotExist: + logger.info("Tracking ID not associated with existing ticket. Creating new ticket.") ticket = None priority = 3 @@ -287,16 +378,22 @@ def ticket_from_message(message, queue, quiet): if t.status == Ticket.REOPENED_STATUS: f.new_status = Ticket.REOPENED_STATUS f.title = _('Ticket Re-Opened by E-Mail Received from %(sender_email)s' % {'sender_email': sender_email}) - + f.save() - if not quiet: - print((" [%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace')) + if six.PY2: + logger.info((" [%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace')) + elif six.PY3: + logger.info(" [%s-%s] %s" % (t.queue.slug, t.id, t.title,)) for file in files: if file['content']: - filename = file['filename'].encode('ascii', 'replace').replace(' ', '_') + if six.PY2: + filename = file['filename'].encode('ascii', 'replace').replace(' ', '_') + elif six.PY3: + filename = file['filename'].replace(' ', '_') filename = re.sub('[^a-zA-Z0-9._-]+', '', filename) + logger.info("Found attachment '%s'" % filename) a = Attachment( followup=f, filename=filename, @@ -305,8 +402,7 @@ def ticket_from_message(message, queue, quiet): ) a.file.save(filename, ContentFile(file['content']), save=False) a.save() - if not quiet: - print(" - %s" % filename) + logger.info("Attachment '%s' successfully added to ticket." % filename) context = safe_template_context(t) diff --git a/helpdesk/migrations/0013_email_box_local_dir_and_logging.py b/helpdesk/migrations/0013_email_box_local_dir_and_logging.py new file mode 100644 index 00000000..71ba784e --- /dev/null +++ b/helpdesk/migrations/0013_email_box_local_dir_and_logging.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.1 on 2016-09-14 23:47 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('helpdesk', '0012_queue_default_owner'), + ] + + operations = [ + migrations.AddField( + model_name='queue', + name='email_box_local_dir', + field=models.CharField(blank=True, help_text='If using a local directory, what directory path do you wish to poll for new email? Example: /var/lib/mail/helpdesk/', max_length=200, null=True, verbose_name='E-Mail Local Directory'), + ), + migrations.AddField( + model_name='queue', + name='logging_dir', + field=models.CharField(blank=True, help_text='If logging is enabled, what directory should we use to store log files for this queue? If no directory is set, default to /var/log/helpdesk/', max_length=200, null=True, verbose_name='Logging Directory'), + ), + migrations.AddField( + model_name='queue', + name='logging_type', + field=models.CharField(blank=True, choices=[('none', 'None'), ('debug', 'Debug'), ('info', 'Information'), ('warn', 'Warning'), ('error', 'Error'), ('crit', 'Critical')], help_text='Set the default logging level. All messages at that level or above will be logged to the directory set below. If no level is set, logging will be disabled.', max_length=5, null=True, verbose_name='Logging Type'), + ), + migrations.AlterField( + model_name='queue', + name='email_box_type', + field=models.CharField(blank=True, choices=[('pop3', 'POP 3'), ('imap', 'IMAP'), ('local', 'Local Directory')], help_text='E-Mail server type for creating tickets automatically from a mailbox - both POP3 and IMAP are supported, as well as reading from a local directory.', max_length=5, null=True, verbose_name='E-Mail Box Type'), + ), + ] diff --git a/helpdesk/models.py b/helpdesk/models.py index 8599093a..52a9bd6a 100644 --- a/helpdesk/models.py +++ b/helpdesk/models.py @@ -115,11 +115,12 @@ class Queue(models.Model): email_box_type = models.CharField( _('E-Mail Box Type'), max_length=5, - choices=(('pop3', _('POP 3')), ('imap', _('IMAP'))), + choices=(('pop3', _('POP 3')), ('imap', _('IMAP')), ('local',_('Local Directory'))), blank=True, null=True, help_text=_('E-Mail server type for creating tickets automatically ' - 'from a mailbox - both POP3 and IMAP are supported.'), + 'from a mailbox - both POP3 and IMAP are supported, as well as ' + 'reading from a local directory.'), ) email_box_host = models.CharField( @@ -175,6 +176,15 @@ class Queue(models.Model): 'folders. Default: INBOX.'), ) + email_box_local_dir = models.CharField( + _('E-Mail Local Directory'), + max_length=200, + blank=True, + null=True, + help_text=_('If using a local directory, what directory path do you ' + 'wish to poll for new email? Example: /var/lib/mail/helpdesk/'), + ) + permission_name = models.CharField( _('Django auth permission name'), max_length=50, @@ -223,6 +233,27 @@ class Queue(models.Model): help_text=_('Socks proxy port number. Default: 9150 (default TOR port)'), ) + logging_type = models.CharField( + _('Logging Type'), + max_length=5, + choices=(('none', _('None')), ('debug', _('Debug')), ('info',_('Information')), ('warn', _('Warning')), ('error', _('Error')), ('crit', _('Critical'))), + blank=True, + null=True, + help_text=_('Set the default logging level. All messages at that ' + 'level or above will be logged to the directory set below. ' + 'If no level is set, logging will be disabled.'), + ) + + logging_dir = models.CharField( + _('Logging Directory'), + max_length=200, + blank=True, + null=True, + help_text=_('If logging is enabled, what directory should we use to ' + 'store log files for this queue? ' + 'If no directory is set, default to /var/log/helpdesk/'), + ) + default_owner = models.ForeignKey( settings.AUTH_USER_MODEL, related_name='default_owner', diff --git a/requirements.txt b/requirements.txt index 8a67f496..94148d58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ django-bootstrap-form>=3.1,<4 email-reply-parser django-markdown-deux simplejson +six