Used six to add python3 compatibility along side existing python2, and replaced print statements with logging features. Can now poll a local directory for mail too, not just remote POP3/IMAP.

helpdesk/management/commands/get_email.py

@@ -5,18 +5,23 @@ Jutda Helpdesk - A Django powered ticket tracker for small enterprise.
 (c) Copyright 2008 Jutda. All Rights Reserved. See LICENSE for details.
 
 scripts/get_email.py - Designed to be run from cron, this script checks the
-                       POP and IMAP boxes defined for the queues within a
+                       POP and IMAP boxes, or a local mailbox directory,
+                       defined for the queues within a
                        helpdesk, creating tickets from the new messages (or
                        adding to existing tickets if needed)
 """
 from __future__ import print_function
 
 import email
+import codecs
 import imaplib
 import mimetypes
 import poplib
 import re
 import socket
+import six
+from os import listdir, unlink
+from os.path import isfile, join
 
 from datetime import timedelta
 from email.header import decode_header
@@ -25,6 +30,7 @@ from optparse import make_option
 
 from email_reply_parser import EmailReplyParser
 
+from django import VERSION
 from django.core.files.base import ContentFile
 from django.core.management.base import BaseCommand
 from django.db.models import Q
@@ -39,11 +45,16 @@ except ImportError:
 from helpdesk.lib import send_templated_mail, safe_template_context
 from helpdesk.models import Queue, Ticket, FollowUp, Attachment, IgnoreEmail
 
+import logging
+from time import ctime
+
 
 class Command(BaseCommand):
     def __init__(self):
         BaseCommand.__init__(self)
 
+        # Django 1.7 uses different way to specify options than 1.8+
+        if VERSION < (1, 8):
             self.option_list += (
             make_option(
                 '--quiet', '-q',
@@ -52,7 +63,16 @@ class Command(BaseCommand):
                 help='Hide details about each queue/message as they are processed'),
             )
 
-    help = 'Process Jutda Helpdesk queues and process e-mails via POP3/IMAP as required, feeding them into the helpdesk.'
+    help = 'Process django-helpdesk queues and process e-mails via POP3/IMAP or from a local mailbox directory as required, feeding them into the helpdesk.'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--quiet',
+            action='store_true',
+            dest='quiet',
+            default=False,
+            help='Hide details about each queue/message as they are processed',
+        )
 
     def handle(self, *args, **options):
         quiet = options.get('quiet', False)
@@ -64,6 +84,25 @@ def process_email(quiet=False):
             email_box_type__isnull=False,
             allow_email_submission=True):
 
+        logger = logging.getLogger('django.helpdesk.queue.' + q.slug)
+        if not q.logging_type or q.logging_type == 'none':
+            logging.disable(logging.CRITICAL) #disable all messages
+        elif q.logging_type == 'info':
+            logger.setLevel(logging.INFO)
+        elif q.logging_type == 'warn':
+            logger.setLevel(logging.WARN)
+        elif q.logging_type == 'error':
+            logger.setLevel(logging.ERROR)
+        elif q.logging_type == 'crit':
+            logger.setLevel(logging.CRITICAL)
+        elif q.logging_type == 'debug':
+            logger.setLevel(logging.DEBUG)
+        if quiet:
+            logger.propagate = False # do not propagate to root logger that would log to console
+        logdir = q.logging_dir or '/var/log/helpdesk/'
+        handler = logging.FileHandler(logdir + q.slug + '_get_email.log')
+        logger.addHandler(handler)
+
         if not q.email_box_last_check:
             q.email_box_last_check = timezone.now()-timedelta(minutes=30)
 
@@ -76,20 +115,20 @@ def process_email(quiet=False):
         if (q.email_box_last_check + queue_time_delta) > timezone.now():
             continue
 
-        process_queue(q, quiet=quiet)
+        process_queue(q, logger=logger)
 
         q.email_box_last_check = timezone.now()
         q.save()
 
 
-def process_queue(q, quiet=False):
+def process_queue(q, logger):
-    if not quiet:
+    logger.info("***** %s: Begin processing mail for django-helpdesk" % ctime())
-        print("Processing: %s" % q)
 
     if q.socks_proxy_type and q.socks_proxy_host and q.socks_proxy_port:
         try:
             import socks
         except ImportError:
+            logger.error("Queue has been configured with proxy settings, but no socks library was installed. Try to install PySocks via pypi.")
             raise ImportError("Queue has been configured with proxy settings, but no socks library was installed. Try to install PySocks via pypi.")
 
         proxy_type = {
@@ -100,7 +139,11 @@ def process_queue(q, quiet=False):
         socks.set_default_proxy(proxy_type=proxy_type, addr=q.socks_proxy_host, port=q.socks_proxy_port)
         socket.socket = socks.socksocket
     else:
+        if six.PY2:
             socket.socket = socket._socketobject
+        elif six.PY3:
+            import _socket
+            socket.socket = _socket.socket
 
     email_box_type = settings.QUEUE_EMAIL_BOX_TYPE if settings.QUEUE_EMAIL_BOX_TYPE else q.email_box_type
 
@@ -113,22 +156,29 @@ def process_queue(q, quiet=False):
             if not q.email_box_port: q.email_box_port = 110
             server = poplib.POP3(q.email_box_host or settings.QUEUE_EMAIL_BOX_HOST, int(q.email_box_port))
 
+        logger.info("Attempting POP3 server login")
+
         server.getwelcome()
         server.user(q.email_box_user or settings.QUEUE_EMAIL_BOX_USER)
         server.pass_(q.email_box_pass or settings.QUEUE_EMAIL_BOX_PASSWORD)
 
 
         messagesInfo = server.list()[1]
+        logger.info("Received %s messages from POP3 server" % str(len(messagesInfo)))
 
         for msg in messagesInfo:
             msgNum = msg.split(" ")[0]
             msgSize = msg.split(" ")[1]
+            logger.info("Processing message %s" % str(msgNum))
 
             full_message = "\n".join(server.retr(msgNum)[1])
-            ticket = ticket_from_message(message=full_message, queue=q, quiet=quiet)
+            ticket = ticket_from_message(message=full_message, queue=q)
 
             if ticket:
                 server.dele(msgNum)
+                logger.info("Successfully processed message %s, deleted from POP3 server" % str(msgNum))
+            else:
+                logger.warn("Message %s was not successfully processed, and will be left on POP3 server" % str(msgNum))
 
         server.quit()
 
@@ -140,36 +190,74 @@ def process_queue(q, quiet=False):
             if not q.email_box_port: q.email_box_port = 143
             server = imaplib.IMAP4(q.email_box_host or settings.QUEUE_EMAIL_BOX_HOST, int(q.email_box_port))
 
+        logger.info("Attempting IMAP server login")
+
         server.login(q.email_box_user or settings.QUEUE_EMAIL_BOX_USER, q.email_box_pass or settings.QUEUE_EMAIL_BOX_PASSWORD)
         server.select(q.email_box_imap_folder)
 
         status, data = server.search(None, 'NOT', 'DELETED')
         if data:
             msgnums = data[0].split()
+            logger.info("Received %s messages from IMAP server" % str(len(msgnums)))
             for num in msgnums:
+                logger.info("Processing message %s" % str(num))
                 status, data = server.fetch(num, '(RFC822)')
-                ticket = ticket_from_message(message=data[0][1], queue=q, quiet=quiet)
+                ticket = ticket_from_message(message=data[0][1], queue=q)
                 if ticket:
                     server.store(num, '+FLAGS', '\\Deleted')
+                    logger.info("Successfully processed message %s, deleted from IMAP server" % str(msgNum))
+                else:
+                    logger.warn("Message %s was not successfully processed, and will be left on IMAP server" % str(msgNum))
 
         server.expunge()
         server.close()
         server.logout()
 
+    elif email_box_type == 'local':
+        mail_dir = q.email_box_local_dir or '/var/lib/mail/helpdesk/'
+        mail = [join(mail_dir, f) for f in listdir(mail_dir) if isfile(join(mail_dir, f))]
+        logger.info("Found %s messages in local mailbox directory" % str(len(mail)))
+        for m in mail:
+            logger.info("Processing message %s" % str(m))
+            f = open(m, 'r')
+            ticket = ticket_from_message(message=f.read(), queue=q, logger=logger)
+            if ticket:
+                logger.info("Successfully processed message %s, ticket/comment created." % str(m))
+                try:
+                    #unlink(m) #delete message file if ticket was successful
+                    logger.info("Successfully deleted message %s." % str(m))
+                except:
+                    logger.error("Unable to delete message %s." % str(m))
+            else:
+                logger.warn("Message %s was not successfully processed, and will be left in local directory" % str(m))
+
 
 def decodeUnknown(charset, string):
+    if six.PY2:
         if not charset:
             try:
                 return string.decode('utf-8','ignore')
             except:
                 return string.decode('iso8859-1','ignore')
         return unicode(string, charset)
+    elif six.PY3:
+        if type(string) is not str:
+            if not charset:
+                try:
+                    return codecs.decode(bytes(string),'utf-8','ignore')
+                except:
+                    return codecs.decode(bytes(string),'iso8859-1','ignore')
+            return str(string, charset)
+        return string
 
 def decode_mail_headers(string):
     decoded = decode_header(string)
+    if six.PY2:
         return u' '.join([unicode(msg, charset or 'utf-8') for msg, charset in decoded])
+    elif six.PY3:
+        return u' '.join([str(msg) for msg, charset in decoded])
 
-def ticket_from_message(message, queue, quiet):
+def ticket_from_message(message, queue, logger):
     # 'message' must be an RFC822 formatted message.
     msg = message
     message = email.message_from_string(msg)
@@ -196,7 +284,9 @@ def ticket_from_message(message, queue, quiet):
     if matchobj:
         # This is a reply or forward.
         ticket = matchobj.group('id')
+        logger.info("Matched tracking ID %s-%s" % (queue.slug, ticket))
     else:
+        logger.info("No tracking ID matched, assuming new ticket")
         ticket = None
 
     counter = 0
@@ -231,7 +321,7 @@ def ticket_from_message(message, queue, quiet):
     if body_plain:
         body = body_plain
     else:
-        body = _('No plain-text email body available. Please see attachment email_html_body.html.')
+        body = _('No plain-text email body available. Please see attachment "email_html_body.html".')
 
     if body_html:
         files.append({
@@ -247,6 +337,7 @@ def ticket_from_message(message, queue, quiet):
             t = Ticket.objects.get(id=ticket)
             new = False
         except Ticket.DoesNotExist:
+            logger.info("Tracking ID not associated with existing ticket. Creating new ticket.")
             ticket = None
 
     priority = 3
@@ -290,13 +381,19 @@ def ticket_from_message(message, queue, quiet):
 
     f.save()
 
-    if not quiet:
+    if six.PY2:
-        print((" [%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace'))
+        logger.info((" [%s-%s] %s" % (t.queue.slug, t.id, t.title,)).encode('ascii', 'replace'))
+    elif six.PY3:
+        logger.info(" [%s-%s] %s" % (t.queue.slug, t.id, t.title,))
 
     for file in files:
         if file['content']:
+            if six.PY2:
                 filename = file['filename'].encode('ascii', 'replace').replace(' ', '_')
+            elif six.PY3:
+                filename = file['filename'].replace(' ', '_')
             filename = re.sub('[^a-zA-Z0-9._-]+', '', filename)
+            logger.info("Found attachment '%s'" % filename)
             a = Attachment(
                 followup=f,
                 filename=filename,
@@ -305,8 +402,7 @@ def ticket_from_message(message, queue, quiet):
                 )
             a.file.save(filename, ContentFile(file['content']), save=False)
             a.save()
-            if not quiet:
+            logger.info("Attachment '%s' successfully added to ticket." % filename)
-                print("    - %s" % filename)
 
 
     context = safe_template_context(t)

helpdesk/migrations/0013_email_box_local_dir_and_logging.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.1 on 2016-09-14 23:47
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('helpdesk', '0012_queue_default_owner'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='queue',
+            name='email_box_local_dir',
+            field=models.CharField(blank=True, help_text='If using a local directory, what directory path do you wish to poll for new email? Example: /var/lib/mail/helpdesk/', max_length=200, null=True, verbose_name='E-Mail Local Directory'),
+        ),
+        migrations.AddField(
+            model_name='queue',
+            name='logging_dir',
+            field=models.CharField(blank=True, help_text='If logging is enabled, what directory should we use to store log files for this queue? If no directory is set, default to /var/log/helpdesk/', max_length=200, null=True, verbose_name='Logging Directory'),
+        ),
+        migrations.AddField(
+            model_name='queue',
+            name='logging_type',
+            field=models.CharField(blank=True, choices=[('none', 'None'), ('debug', 'Debug'), ('info', 'Information'), ('warn', 'Warning'), ('error', 'Error'), ('crit', 'Critical')], help_text='Set the default logging level. All messages at that level or above will be logged to the directory set below. If no level is set, logging will be disabled.', max_length=5, null=True, verbose_name='Logging Type'),
+        ),
+        migrations.AlterField(
+            model_name='queue',
+            name='email_box_type',
+            field=models.CharField(blank=True, choices=[('pop3', 'POP 3'), ('imap', 'IMAP'), ('local', 'Local Directory')], help_text='E-Mail server type for creating tickets automatically from a mailbox - both POP3 and IMAP are supported, as well as reading from a local directory.', max_length=5, null=True, verbose_name='E-Mail Box Type'),
+        ),
+    ]

helpdesk/models.py

@@ -115,11 +115,12 @@ class Queue(models.Model):
     email_box_type = models.CharField(
         _('E-Mail Box Type'),
         max_length=5,
-        choices=(('pop3', _('POP 3')), ('imap', _('IMAP'))),
+        choices=(('pop3', _('POP 3')), ('imap', _('IMAP')), ('local',_('Local Directory'))),
         blank=True,
         null=True,
         help_text=_('E-Mail server type for creating tickets automatically '
-            'from a mailbox - both POP3 and IMAP are supported.'),
+            'from a mailbox - both POP3 and IMAP are supported, as well as '
+            'reading from a local directory.'),
         )
 
     email_box_host = models.CharField(
@@ -175,6 +176,15 @@ class Queue(models.Model):
             'folders. Default: INBOX.'),
         )
 
+    email_box_local_dir = models.CharField(
+        _('E-Mail Local Directory'),
+        max_length=200,
+        blank=True,
+        null=True,
+        help_text=_('If using a local directory, what directory path do you '
+            'wish to poll for new email? Example: /var/lib/mail/helpdesk/'),
+        )
+
     permission_name = models.CharField(
         _('Django auth permission name'),
         max_length=50,
@@ -223,6 +233,27 @@ class Queue(models.Model):
         help_text=_('Socks proxy port number. Default: 9150 (default TOR port)'),
     )
 
+    logging_type = models.CharField(
+        _('Logging Type'),
+        max_length=5,
+        choices=(('none', _('None')), ('debug', _('Debug')), ('info',_('Information')), ('warn', _('Warning')), ('error', _('Error')), ('crit', _('Critical'))),
+        blank=True,
+        null=True,
+        help_text=_('Set the default logging level. All messages at that '
+            'level or above will be logged to the directory set below. '
+            'If no level is set, logging will be disabled.'),
+        )
+
+    logging_dir = models.CharField(
+        _('Logging Directory'),
+        max_length=200,
+        blank=True,
+        null=True,
+        help_text=_('If logging is enabled, what directory should we use to '
+            'store log files for this queue? '
+            'If no directory is set, default to /var/log/helpdesk/'),
+        )
+
     default_owner = models.ForeignKey(
         settings.AUTH_USER_MODEL,
         related_name='default_owner',

requirements.txt

@@ -3,3 +3,4 @@ django-bootstrap-form>=3.1,<4
 email-reply-parser
 django-markdown-deux
 simplejson
+six