conf_slk120/bin/.bin/shell/fetch.py

   1 ############################################################
   2 ## Vim Email Client v0.2 - POP3 Client                    ##
   3 ##                                                        ##
   4 ##                                                        ##
   5 ## Created by David Elentok.                              ##
   6 ##                                                        ##
   7 ## This software is released under the GPL License.       ##
   8 ##                                                        ##
   9 ############################################################
  10 # -*- coding: cp1255 -*-
  11 __version__ = "0.2"
  12 __author__ = "David Elentok (3david@gmail.com)"
  13 __copyright__ = "(C) 2005 David Elentok. GNU GPL 2."
  14
  15 ############################################################
  16 ## Imports
  17 import getpass, poplib, email, os, os.path, time, sys
  18 import email.Header
  19 import sgmllib, cStringIO
  20
  21
  22 ############################################################
  23 ## Configuration
  24 mailboxdir = '/home/arrow/Mail/'
  25
  26 ### NOTE: the password field is ignored at the moment
  27 accounts = [ ('z_arrow', 'pop3.163.com', 'z_arrow', '') ]
  28 #accounts = [ ('account1', 'pop3.server1.com', 'username1', 'password1'),
  29              #('account2', 'pop3.server2.com', 'username2', 'password2') ]
  30
  31 # the directory where the html2text.py file can be found
  32 html2text_dir = mailboxdir
  33
  34 sys.path.append(html2text_dir)
  35 import html2text
  36
  37 ############################################################
  38 def get_mailbox_uidls(mailbox):
  39   """This function returns a list of all of the uidl (unique id's)
  40      inside an .mbx file """
  41
  42   mbxfile = "%s\\%s.mbx" % (mailboxdir, mailbox)
  43
  44   print "Opening mbx: [%s]" % mbxfile
  45
  46   if not os.path.exists(mbxfile):
  47     return []
  48
  49   fd = open(mbxfile)
  50
  51   uidls=[]
  52
  53   for line in fd.readlines():
  54     if line[0:7] == "* UIDL:":
  55       list = line.split(':')
  56       uidls.append( list[1].strip() )
  57
  58   fd.close()
  59
  60   return uidls
  61
  62   """This function returns a list of all of the uidl (unique id's) of
  63      all of the messages on the server """
  64
  65 ############################################################
  66 def decode_field(field):
  67   """This function is used for decoding the values of message fields
  68      (To:, Cc:, Subject:, ..) using python's email.Header.decode_header
  69      function."""
  70   field = field.replace('\r\n','')
  71   field = field.replace('\n','')
  72
  73   list = email.Header.decode_header (field)
  74
  75   decoded = " ".join(["%s" % k for (k,v) in list])
  76
  77   #print "Decoding [%s] to [%s]" % (field, decoded)
  78
  79   return decoded
  80
  81
  82 ############################################################
  83 def string2time (strdate):
  84   """ This function converts a string from a date that contains
  85       a string in one of the following formats:
  86       - %a, %d %b %Y %H:%M:%S
  87       - %d %b %Y %H:%M:%S
  88       (since some emails come with a date field in the first format,
  89        and others come with a date field in the second format)
  90
  91       TODO:
  92       - maybe add another try+except to the second time.strptime()
  93       - use the GMT/.. offset instead of ignoring it
  94
  95   """
  96
  97   ## these 3 if blocks are used to ignore the GMT/.. offset
  98   if not strdate.find('(') == -1:
  99     strdate = strdate[0:strdate.find('(')].strip()
 100
 101   if not strdate.find('+') == -1:
 102     strdate = strdate[0:strdate.find('+')].strip()
 103
 104   if not strdate.find('-') == -1:
 105     strdate = strdate[0:strdate.find('-')].strip()
 106
 107   ## convert the date string into a  9-item tuple
 108   try:
 109     t = time.strptime(strdate, "%a, %d %b %Y %H:%M:%S")
 110   except ValueError, e:
 111     t = time.strptime(strdate, "%d %b %Y %H:%M:%S")
 112
 113   return t
 114
 115 ############################################################
 116 def buildTitle (tm, mailFrom, mailSubject):
 117   """ This functions receives 3 parameters:
 118         tm =  a 9-item tuple that represents the email's date and time
 119         mailFrom = the contents of the "From:" field
 120         mailSubject = the contents of the "Subject:" field
 121
 122       and it returns a title for the email message (used with vim's folding)
 123       the title is in this format:
 124         * Date and Time | From           | Subject
 125
 126       the width of "From" is 15 characters by default, it's defined by the
 127       from_column_width variable
 128   """
 129   from_column_width = 15
 130
 131   # Create Time String
 132   strtime = time.strftime("%d/%m/%Y %H:%M",tm)
 133
 134   # Create From String
 135   strfrom = mailFrom
 136
 137   if len(strfrom) > from_column_width:
 138     strfrom=strfrom[0:from_column_width]
 139
 140   strfrom = strfrom.center(from_column_width)
 141
 142   # Create Subject String
 143   title = "* %s | %s | %s\r\n" % (strtime, strfrom, mailSubject)
 144
 145   return title
 146
 147
 148 ############################################################
 149 def handleMsg(mailbox, msg, is_subpart=False, strdate=""):
 150   """ This function handles a message object recursively, it has
 151       several tasks:
 152       - save all of the attachments in the message
 153       - extract all of the text information into the message body
 154       - if the email contains html messages they will be converted
 155         into text and added to the message body
 156       - extract all of the field information (To, Cc, From, ...)
 157         from the message objects
 158
 159   """
 160   global text
 161   global attachments
 162   global fieldFrom, fieldSubject, fieldTime
 163
 164   # Message/RFC822 parts are bundled this way ==============
 165   while isinstance(msg.get_payload(),email.Message.Message):
 166     msg=msg.get_payload()
 167
 168   if not is_subpart:
 169     fieldFrom = ""
 170     fieldSubject = ""
 171     fieldTime = None    # fieldTime is a 9-item tuple
 172     text = ""           # the text contents of a message
 173     attachments = ""
 174
 175   ## Set the "From" Field ==================================
 176   if fieldFrom == "" and msg['From'] != None:
 177     text += "To: %s\n" % decode_field(msg['To'])
 178     if msg['Cc'] != None:
 179       text += "Cc: %s\n" % decode_field(msg['Cc'])
 180     if msg['Bcc'] != None:
 181       text += "Bcc: %s\n" % decode_field(msg['Bcc'])
 182     text += "From: %s\n" % decode_field(msg['From'])
 183     fieldFrom = decode_field(msg['From'])
 184
 185   ## Set the "Subject" Field ===============================
 186   if fieldSubject == "" and msg['Subject'] != None:
 187     fieldSubject = decode_field(msg['Subject'])
 188     text += "Subject: %s\n" % fieldSubject
 189
 190   ## Set the "Date" Field ==================================
 191   if fieldTime == None and msg['Date'] != None:
 192     fieldTime = string2time(msg['Date'])
 193     strdate = time.strftime("%Y%m%d%H%M", fieldTime)
 194
 195   ## Handle multipart messages recursively =================
 196   if msg.is_multipart():
 197     for submsg in msg.get_payload():
 198       handleMsg(mailbox, submsg, True, strdate)
 199   else:
 200     fname = msg.get_filename()
 201     if fname == None:
 202       if msg.get_content_type() == 'text/plain':
 203         text += "\n%s" % msg.get_payload(decode=1)
 204       else:
 205         fname = "message.htm"
 206
 207     ## Save an attachment to a file ========================
 208     if not fname == None:
 209       fname = decode_field(fname)
 210       filename = "%s\\att_%s\\%s_%s" % (mailboxdir, mailbox, strdate, fname)
 211       org_filename = filename
 212       i = 1
 213       while os.path.exists(filename):
 214         path, ext = os.path.splitext(org_filename)
 215         filename = "%s (%d)%s" % (path, i, ext)
 216         i = i + 1
 217
 218       print " Found part: %s" % filename  # for debugging purposes
 219       attachments += "%s\n" % filename
 220       fd = open (filename, "wb")
 221       data = msg.get_payload(decode=1)
 222       fd.write(data)
 223
 224       # convert an html message to text
 225       if fname == "message.htm":
 226         try:
 227           strio = cStringIO.StringIO()
 228           html2text.html2text_file(data, out=strio.write)
 229           text += strio.getvalue()
 230           strio.close()
 231         except sgmllib.SGMLParseError, e:
 232           print e
 233
 234       fd.close()
 235
 236   # if this is the toplevel message (the first function that was called by
 237   # fetch_mailbox, then return the title of the message
 238   if not is_subpart and fieldTime != None:
 239     title = buildTitle(fieldTime, fieldFrom, fieldSubject)
 240     return title
 241
 242 ############################################################
 243 def fetch_mailbox((mailbox, host, user, passwd)):
 244   """ This function gets an account object as input, this object is a 4-item
 245       tuple that contains the mailbox name, the address of the pop3 server,
 246       the username to use and the password.
 247
 248       What it does is download all of the messages in the server, save the
 249       attachments to the $mailbox/attach directory, and save all of the email
 250       messages into one .mbx file.
 251
 252       This function returns the amount of new messages found
 253
 254       TODO:
 255       - find a way to encrypt "passwd" instead of asking it from the user
 256         each time.
 257
 258   """
 259
 260   global text, attachments
 261
 262   ## login to the pop3 server ==============================
 263   print
 264   print "###### Connecting to %s" % host
 265   M = poplib.POP3(host)
 266   M.set_debuglevel(1)
 267   M.user(user)
 268   M.pass_(getpass.getpass())
 269
 270   ## create the mailbox and attachments directories if required
 271   if not os.path.exists (mailboxdir):
 272     print "Creating Directory %s", mailboxdir
 273     os.mkdir (mailboxdir)
 274
 275   att_dir = "%s\\att_%s" % (mailboxdir, mailbox)
 276   if not os.path.exists (att_dir):
 277     print "Creating Directory %s", att_dir
 278     os.mkdir (att_dir)
 279
 280
 281   ## get list of uidls in the mailbox file =================
 282   uidls = get_mailbox_uidls(mailbox)
 283
 284   ## get number of messages ================================
 285   numMessages = len(M.list()[1])
 286   print "There are %d messages on the server" % numMessages
 287
 288
 289   ## get uidls from server and compare with the uidls in the
 290   ## mailbox ===============================================
 291   uidls_srv = M.uidl()
 292   list = uidls_srv[1]
 293   fetchlist = []
 294   for item in list:
 295     msgno, uidl = item.split(' ')
 296     msgno = int(msgno)
 297     if not uidl in uidls:
 298       print "Found new message: (%d, %s)" % (msgno, uidl)
 299       fetchlist.append(msgno)
 300
 301   print "There are %d new messages on the server" % len(fetchlist)
 302
 303   alltext = "" ## this variable contains the mbox contents
 304
 305   ## go over all of the emails =============================
 306   for i in fetchlist:
 307
 308     flatmsg = ""
 309
 310     ## retreive message
 311 #     for line in M.retr(i+1)[1]:
 312     for line in M.retr(i)[1]:
 313       flatmsg += line + "\r\n"
 314
 315     ## parse message
 316     msg = email.message_from_string (flatmsg)
 317
 318     ## handle Email.message object
 319     title = handleMsg(mailbox, msg)
 320
 321
 322     msgtext = "%s\n%s* UIDL: %s\n%s\n\n" % (''.center(70,'#'), title, uidl, text)
 323     if not attachments == "":
 324       msgtext += "#### Attachments:\n%s" % attachments
 325
 326     alltext = msgtext.replace('\r\n','\n') + alltext
 327
 328   ## add 'alltext' to the beginning of the mailbox file ====
 329   mboxfile = "%s\\%s.mbx" % (mailboxdir, mailbox)
 330   contents = ""
 331   if os.path.exists(mboxfile):
 332     mbox = open(mboxfile, "rt")
 333     contents = mbox.read()
 334     mbox.close()
 335
 336   mbox = open(mboxfile, "wt")
 337   mbox.write (alltext)
 338   if contents != "":
 339     mbox.write (contents)
 340
 341   mbox.close()
 342
 343   return len(fetchlist)
 344
 345 ############################################################
 346
 347 if __name__ == "__main__":
 348   summary = []
 349   for account in accounts:
 350     newMessages = fetch_mailbox (account)
 351     if newMessages > 0:
 352       summary.append((account[0], newMessages))
 353
 354   if len(summary) > 0:
 355     print
 356     print "Summary: "
 357
 358     for account,msgs in summary:
 359       print "  %s: %d new messages" % (account,msgs)