![[Home page]](../PythonPowered.gif "Home page"){kind=small}
| Books | Code | Blog | Python | Author | Train | Find | ©M.Lutz |
File: pyedit-products/unzipped/PP4E/Internet/Email/mailtools/mailFetcher.py
"""
###############################################################################
retrieve, delete, match mail from a POP server (see __init__ for docs, test)
###############################################################################
"""
import poplib, mailconfig, sys # client's mailconfig on sys.path
print('user:', mailconfig.popusername) # script dir, pythonpath, changes
from .mailParser import MailParser # for headers matching (4E: .)
from .mailTool import MailTool, SilentMailTool # trace control supers (4E: .)
# Dec-2014 evil "monkeypatch": else bombs on quit() when 2048
if hasattr(poplib, '_MAXLINE'): # Dec2015: not in older libs!
poplib._MAXLINE *= 1000
# index/server msgnum out of synch tests
class DeleteSynchError(Exception): pass # msg out of synch in del
class TopNotSupported(Exception): pass # can't run synch test
class MessageSynchError(Exception): pass # index list out of sync
class MailFetcher(MailTool):
"""
fetch mail: connect, fetch headers+mails, delete mails
works on any machine with Python+Inet; subclass me to cache
implemented with the POP protocol; IMAP requires new class;
4E: handles decoding of full mail text on fetch for parser;
"""
def __init__(self, popserver=None, popuser=None, poppswd=None, hastop=True):
self.popServer = popserver or mailconfig.popservername
self.popUser = popuser or mailconfig.popusername
self.srvrHasTop = hastop
self.popPassword = poppswd # ask later if None
self.saveserver = None # 1.5
def connect(self):
"""
establish POP server connection for mailbox operations
password fetched from file or code provided by subclasses
if run in a GUI's thread, caller should avoid GUI popup
"""
# Jan2014, 1.5, experimental: servers that limit logins (outlook.com)
# *NOT USED*: this didn't work in testing, but retained for future use
if hasattr(mailconfig, 'popusesOneLogin') and self.saveserver:
return self.saveserver
self.trace('Connecting...')
self.getPassword() # via file, GUI, or console
# Dec2015 and Jan2014: support SSL/TLS (post book)
# TLS is an extension to basic SSL: use one or the other
# all use an optional ':port' at end of server name, else default for type
# unlike smtplib, must strip port manually if present
if ':' not in self.popServer:
srvr, port = self.popServer, None
else:
srvr, port = self.popServer.split(':')
# May2016: timeouts increased, configurable
timeout = mailconfig.popTimeout
if getattr(mailconfig, 'popusesSSL', False): # set and True?
# start ssl pop session, encrypted # default port=995
print('Using SSL POP on', srvr, port)
server = poplib.POP3_SSL(srvr, port or 995, timeout=timeout)
elif getattr(mailconfig, 'popusesTLS', False): # set and True?
# start tls pop session, encrypted # default port=110
print('Using TLS/SSL POP on', srvr, port) # uses SSL auto
server = poplib.POP3(srvr, port or 110, timeout=timeout)
server.stls()
server.ehlo()
else:
# original/book: plain pop # default port=110
print('Using basic POP on', srvr, port)
server = poplib.POP3(srvr, port or 110, timeout=timeout)
# back to original/book code
server.user(self.popUser) # connect,login POP server
server.pass_(self.popPassword) # pass is a reserved word
self.trace(server.getwelcome()) # print returned greeting
# Jan2014, 1.5: support servers that limit logins (outlook.com)
if hasattr(mailconfig, 'popusesOneLogin'):
self.saveserver = server
return server
def disconnect(self, server):
"""
quit the server connection, unlock mailbox
"""
# Jan2014, 1.5: support servers that limit logins (outlook.com);
# *NOT USED* - TBD: what does a persistent login do to mail box locks?
if not hasattr(mailconfig, 'popusesOneLogin'):
server.quit() # unlocks mailbox
# use setting in client's mailconfig on import search path;
# to tailor, this can be changed in class or per instance;
fetchEncoding = mailconfig.fetchEncoding
def decodeFullText(self, messageBytes):
"""
4E, Py3.1: decode full fetched mail text bytes to str Unicode string;
done at fetch, for later display or parsing (full mail text is always
Unicode thereafter); decode with per-class or per-instance setting, or
common types; could also try headers inspection, or intelligent guess
from structure; in Python 3.2/3.3, this step may not be required: if so,
change to return message line list intact; for more details see Chapter 13;
an 8-bit encoding such as latin-1 will likely suffice for most emails, as
ASCII is the original standard; this method applies to entire/full message
text, which is really just one part of the email encoding story: Message
payloads and Message headers may also be encoded per email, MIME, and
Unicode standards; see Chapter 13 and mailParser and mailSender for more;
"""
text = None
kinds = [self.fetchEncoding] # try user setting first
kinds += ['ascii', 'latin1', 'utf8'] # then try common types
kinds += [sys.getdefaultencoding()] # and platform dflt (may differ)
for kind in kinds: # may cause mail saves to fail
try:
text = [line.decode(kind) for line in messageBytes]
break
except (UnicodeError, LookupError): # LookupError: bad name
pass
if text == None:
# try returning headers + error msg, else except may kill client;
# still try to decode headers per ascii, other, platform default;
blankline = messageBytes.index(b'')
hdrsonly = messageBytes[:blankline]
commons = ['ascii', 'latin1', 'utf8']
for common in commons:
try:
text = [line.decode(common) for line in hdrsonly]
break
except UnicodeError:
pass
else: # none worked
try:
text = [line.decode() for line in hdrsonly] # platform dflt?
except UnicodeError:
text = ['From: (sender of unknown Unicode format headers)']
text += ['', '--Sorry: mailtools cannot decode this mail content!--']
return text
def downloadMessage(self, msgnum):
"""
load full raw text of one mail msg, given its
POP relative msgnum; caller must parse content
"""
self.trace('load ' + str(msgnum))
server = self.connect()
try:
resp, msglines, respsz = server.retr(msgnum)
finally:
self.disconnect(server) # 1.5
msglines = self.decodeFullText(msglines) # raw bytes to Unicode str
return '\n'.join(msglines) # concat lines for parsing
def downloadAllHeaders(self, progress=None, loadfrom=1):
"""
get sizes, raw header text only, for all or new msgs
begins loading headers from message number loadfrom
use loadfrom to load newly arrived mails only
use downloadMessage to get a full msg text later
progress is a function called with (count, total);
returns: [headers text], [mail sizes], loadedfull?
4E: add mailconfig.fetchlimit to support large email
inboxes: if not None, only fetches that many headers,
and returns others as dummy/empty mail; else inboxes
like one of mine (4K emails) are not practical to use;
4E: pass loadfrom along to downloadAllMsgs (a buglet);
"""
if not self.srvrHasTop: # not all servers support TOP
# naively load full msg text
return self.downloadAllMsgs(progress, loadfrom)
else:
self.trace('loading headers')
fetchlimit = mailconfig.fetchlimit
server = self.connect() # mbox now locked until quit
try:
resp, msginfos, respsz = server.list() # 'num size' lines list
msgCount = len(msginfos) # alt to srvr.stat[0]
msginfos = msginfos[loadfrom-1:] # drop already loadeds
allsizes = [int(x.split()[1]) for x in msginfos]
allhdrs = []
for msgnum in range(loadfrom, msgCount+1): # poss empty
if progress: progress(msgnum, msgCount) # run callback
if fetchlimit and (msgnum <= msgCount - fetchlimit):
# skip, add dummy hdrs
hdrtext = 'Subject: --mail skipped--\n\n'
allhdrs.append(hdrtext)
else:
# fetch, retr hdrs only
resp, hdrlines, respsz = server.top(msgnum, 0)
hdrlines = self.decodeFullText(hdrlines)
allhdrs.append('\n'.join(hdrlines))
finally:
self.disconnect(server) # 1.5 # make sure unlock mbox
assert len(allhdrs) == len(allsizes)
self.trace('load headers exit')
return allhdrs, allsizes, False
def downloadAllMessages(self, progress=None, loadfrom=1):
"""
load full message text for all msgs from loadfrom..N,
despite any caching that may be being done in the caller;
much slower than downloadAllHeaders, if just need hdrs;
4E: support mailconfig.fetchlimit: see downloadAllHeaders;
could use server.list() to get sizes of skipped emails here
too, but clients probably don't care about these anyhow;
"""
self.trace('loading full messages')
fetchlimit = mailconfig.fetchlimit
server = self.connect()
try:
(msgCount, msgBytes) = server.stat() # inbox on server
allmsgs = []
allsizes = []
for i in range(loadfrom, msgCount+1): # empty if low >= high
if progress: progress(i, msgCount)
if fetchlimit and (i <= msgCount - fetchlimit):
# skip, add dummy mail
mailtext = 'Subject: --mail skipped--\n\nMail skipped.\n'
allmsgs.append(mailtext)
allsizes.append(len(mailtext))
else:
# fetch, retr full mail
(resp, message, respsz) = server.retr(i) # save text on list
message = self.decodeFullText(message)
allmsgs.append('\n'.join(message)) # leave mail on server
allsizes.append(respsz) # diff from len(msg)
finally:
self.disconnect(server) # 1.5 # unlock the mail box
assert len(allmsgs) == (msgCount - loadfrom) + 1 # msg nums start at 1
#assert sum(allsizes) == msgBytes # not if loadfrom > 1
return allmsgs, allsizes, True # not if fetchlimit
def deleteMessages(self, msgnums, progress=None):
"""
delete multiple msgs off server; assumes email inbox
unchanged since msgnums were last determined/loaded;
use if msg headers not available as state information;
fast, but poss dangerous: see deleteMessagesSafely
"""
self.trace('deleting mails')
server = self.connect()
try:
for (ix, msgnum) in enumerate(msgnums): # don't reconnect for each
if progress: progress(ix+1, len(msgnums))
server.dele(msgnum)
finally: # changes msgnums: reload
self.disconnect(server) # 1.5
def deleteMessagesSafely(self, msgnums, synchHeaders, progress=None):
"""
delete multiple msgs off server, but use TOP fetches to
check for a match on each msg's header part before deleting;
assumes the email server supports the TOP interface of POP,
else raises TopNotSupported - client may call deleteMessages;
use if the mail server might change the inbox since the email
index was last fetched, thereby changing POP relative message
numbers; this can happen if email is deleted in a different
client; some ISPs may also move a mail from inbox to the
undeliverable box in response to a failed download;
synchHeaders must be a list of already loaded mail hdrs text,
corresponding to selected msgnums (requires state); raises
exception if any out of synch with the email server; inbox is
locked until quit, so it should not change between TOP check
and actual delete: synch check must occur here, not in caller;
may be enough to call checkSynchError+deleteMessages, but check
each msg here in case deletes and inserts in middle of inbox;
"""
if not self.srvrHasTop:
raise TopNotSupported('Safe delete cancelled')
self.trace('deleting mails safely')
errmsg = 'Message %s out of synch with server.\n'
errmsg += 'Delete terminated at this message.\n'
errmsg += 'Mail client may require restart or reload.'
server = self.connect() # locks inbox till quit
try: # don't reconnect for each
(msgCount, msgBytes) = server.stat() # inbox size on server
for (ix, msgnum) in enumerate(msgnums):
if progress: progress(ix+1, len(msgnums))
if msgnum > msgCount: # msgs deleted
raise DeleteSynchError(errmsg % msgnum)
resp, hdrlines, respsz = server.top(msgnum, 0) # hdrs only
hdrlines = self.decodeFullText(hdrlines)
msghdrs = '\n'.join(hdrlines)
if not self.headersMatch(msghdrs, synchHeaders[msgnum-1]):
raise DeleteSynchError(errmsg % msgnum)
else:
server.dele(msgnum) # safe to delete this msg
finally: # changes msgnums: reload
self.disconnect(server) # 1.5 # unlock inbox on way out
def checkSynchError(self, synchHeaders):
"""
check to see if already loaded hdrs text in synchHeaders
list matches what is on the server, using the TOP command in
POP to fetch headers text; use if inbox can change due to
deletes in other client, or automatic action by email server;
raises except if out of synch, or error while talking to server;
for speed, only checks last in last: this catches inbox deletes,
but assumes server won't insert before last (true for incoming
mails); check inbox size first: smaller if just deletes; else
top will differ if deletes and newly arrived messages added at
end; result valid only when run: inbox may change after return;
"""
self.trace('synch check')
errormsg = 'Message index out of synch with mail server.\n'
errormsg += 'Mail client may require restart or reload.'
server = self.connect()
try:
lastmsgnum = len(synchHeaders) # 1..N
(msgCount, msgBytes) = server.stat() # inbox size
if lastmsgnum > msgCount: # fewer now?
raise MessageSynchError(errormsg) # none to cmp
if self.srvrHasTop:
resp, hdrlines, respsz = server.top(lastmsgnum, 0) # hdrs only
hdrlines = self.decodeFullText(hdrlines)
lastmsghdrs = '\n'.join(hdrlines)
if not self.headersMatch(lastmsghdrs, synchHeaders[-1]):
raise MessageSynchError(errormsg)
finally:
self.disconnect(server) # 1.5
def headersMatch(self, hdrtext1, hdrtext2):
""""
may not be as simple as a string compare: some servers add
a "Status:" header that changes over time; on one ISP, it
begins as "Status: U" (unread), and changes to "Status: RO"
(read, old) after fetched once - throws off synch tests if
new when index fetched, but have been fetched once before
delete or last-message check; "Message-id:" line is unique
per message in theory, but optional, and can be anything if
forged; match more common: try first; parsing costly: try last
"""
# try match by simple string compare
if hdrtext1 == hdrtext2:
self.trace('Same headers text')
return True
# try match without status lines
split1 = hdrtext1.splitlines() # s.split('\n'), but no final ''
split2 = hdrtext2.splitlines()
strip1 = [line for line in split1 if not line.startswith('Status:')]
strip2 = [line for line in split2 if not line.startswith('Status:')]
if strip1 == strip2:
self.trace('Same without Status')
return True
# try mismatch by message-id headers if either has one
msgid1 = [line for line in split1 if line[:11].lower() == 'message-id:']
msgid2 = [line for line in split2 if line[:11].lower() == 'message-id:']
if (msgid1 or msgid2) and (msgid1 != msgid2):
self.trace('Different Message-Id')
return False
# try full hdr parse and common headers if msgid missing or trash
tryheaders = ('From', 'To', 'Subject', 'Date')
tryheaders += ('Cc', 'Return-Path', 'Received')
msg1 = MailParser().parseHeaders(hdrtext1)
msg2 = MailParser().parseHeaders(hdrtext2)
for hdr in tryheaders: # poss multiple Received
if msg1.get_all(hdr) != msg2.get_all(hdr): # case insens, dflt None
self.trace('Diff common headers')
return False
# all common hdrs match and don't have a diff message-id
self.trace('Same common headers')
return True
def getPassword(self):
"""
get POP password if not yet known
not required until go to server
from client-side file or subclass method
Dec2015: user rstrip() instead of [:-1] in case no \n
"""
if not self.popPassword:
try:
self.getPasswordFromFile()
except:
self.popPassword = self.askPopPassword() # on instance
def getPasswordFromFile(self):
"""
Dec2015: factor this code out so callable from PyMailGUI main thread;
that program can't allow a mail thread to popup a password dialog;
any exceptions (e.g., non-existent file/path) handled by the caller;
"""
localfile = open(mailconfig.poppasswdfile)
self.popPassword = localfile.readline().rstrip() # on inst
#self.trace('local file password' + repr(self.popPassword)) # nope
def askPopPassword(self):
"""
if run in a GUI's spawned thread, caller should avoid GUI popup
"""
assert False, 'Subclass must define method' # GUIs beware!
################################################################################
# specialized subclasses
################################################################################
class MailFetcherConsole(MailFetcher):
def askPopPassword(self):
import getpass
prompt = 'Password for %s on %s?' % (self.popUser, self.popServer)
return getpass.getpass(prompt)
class SilentMailFetcher(SilentMailTool, MailFetcher):
pass # replaces trace