#!/bin/env python 
#########################################################################
# use ftp to copy all files from a remote site/directory to a local dir;
# e.g., run me periodically from a unix cron job to mirror an ftp site;
# script assumes this is a flat directory--see the mirror program in 
# Python's Tools directory for a version that handles subdirectories;
# see uploadflat.py--use storbinary to upload, retrlines for text, etc.;
# could use binary xfer always, but doesn't map dos/unix line feeds;
# for gifs, etc., local file mode must be 'wb' (binary) not 'w' on Win98;
# the retrlines method strips crlf's on lines sent to the data callback;
# transfer block size defaults to 8K bytes if ommitted for retrbinary;
# getpass.getpass is like raw_input, but doesn't show input as typed;
# to mirror with subdirectories, parse the result of the FTP().dir()
# method to detect remote directories, use os.mkdir(path) to create
# local directories of same name, and recursion to download subdirs:
# see Tools/scripts/ftpmirror.py in the Python source distribution.
#########################################################################

import os, sys, ftplib
from getpass import getpass

remotesite = 'home.rmi.net'
remotedir  = 'public_html'
remoteuser = 'lutz'
remotepass = getpass('Please enter password for %s: ' % remotesite)
localdir   = (len(sys.argv) > 1 and sys.argv[1]) or '.'
cleanall   = raw_input('Clean local directory first? ')[:1] in ['y', 'Y']

print 'connecting...'
connection = ftplib.FTP(remotesite)                 # connect to ftp site
connection.login(remoteuser, remotepass)            # login as user/password
connection.cwd(remotedir)                           # cd to directory to copy

if cleanall:
    for localname in os.listdir(localdir):          # try to delete all locals
        try:                                        # first to remove old files
            print 'deleting local', localname
            os.remove(os.path.join(localdir, localname))
        except:
            print 'cannot delete local', localname

count = 0                                           # download all remote files
remotefiles = connection.nlst()                     # nlst() gives files list
                                                    # dir()  gives full details
for remotename in remotefiles:
    localname = os.path.join(localdir, remotename) 
    print 'copying', remotename, 'to', localname
    if remotename[-4:] == 'html' or remotename[-3:] == 'txt':
        # use ascii mode xfer
        localfile = open(localname, 'w')
        callback  = lambda line, file=localfile: file.write(line + '\n')
        connection.retrlines('RETR ' + remotename, callback)
    else:
        # use binary mode xfer
        localfile = open(localname, 'wb')
        connection.retrbinary('RETR ' + remotename, localfile.write)
    localfile.close()
    count = count+1

connection.quit()
print 'Done:', count, 'files downloaded.'