Large built-in toolset:
socket, signal, select, thread, glob, shutil, tempfile, …
· See Python library manual for the full story
Large third-party domain:
PySerial, PyUSB, Pexpect, Twisted, MailMan, CORBA ORBs, …
· See the web for specific domains
Python tools: sys
· Python-related exports
· path: initialized from PYTHONPATH, changeable
· platform: ‘sunos’, ‘win32’, ‘linux2’, etc.
· sys.exit(N), sys.exc_info(), sys.executable, sys.version, sys.modules …
>>> import sys
>>> sys.path
['.', '/usr/local/lib/python', ... ]
>>> sys.platform
'sunos4'
>>> if sys.platform[:3] == 'win':
print 'on windows'
on windows
>>> sys.path
['C:/Python25', 'C:\\Python25\\Lib\\idlelib', 'C:\\WINDOWS\\system32\\python25.zip', …
>>> sys.executable
'C:\\Python25\\pythonw.exe'
>>> sys.version
'2.5 (r25:51908, Sep 19 2006, 09:52:17) [MSC v.1310 32 bit (Intel)]'
>>> sys.modules.keys()
…loaded module names…
System tools: os
· POSIX bindings: operating system exports
· ~200 attributes on some platforms, + nested “os.path” module
· Mostly portable, some calls absent on standard Windows Python
· Cygwin Python adds full UNIX call set on Windows
Content survey
· Shell environment variables
os.environ
· Running shell commands, programs:
os.system, os.popen,
os.popen2/3/4, os.startfile
· Spawning processes:
os.fork, os.pipe, os.exec,
os.waitpid, os.kill
· Descriptor files, with locking:
os.open, os.read, os.write
· File processing:
os.remove, os.rename,
os.mkfifo, os.mkdir, os.rmdir
· Administrative tools:
os.getcwd, os.chdir,
os.chmod, os.getpid, os.listdir
· Portability tools:
os.sep, os.pathsep,
os.curdir, os.path.split, os.path.join
· os.path nested submodule: pathname tools
os.path.exists('filepathname')
os.path.isdir('filepathname')
os.path.getsize('filepathname')
>>> import os
>>> listing = os.popen("ls *.py").readlines()
>>> for name in listing: print name,
...
cheader1.py
finder1.py
summer.py
>>> for name in listing: os.system("vi " + name)
...
os.popen('cmd', 'w').write('data')
(i, o) = os.popen2('cmd')
(i, o, e) = os.popen3('cmd')
(i, o_e) = os.popen4('cmd')
os.startfile('file')
·
See “pty” module, “pexpect” extension, to automate interactive programs without
deadlocks
·
See new “subprocess” module in 2.5+ for more
low-level control over streams
Example: testing command-line scripts
Shell environment variables: os
· os.environ: read/write access to shell variables
· normal dictionary interface
>>> import os
>>> os.environ['USER']
'mlutz'
>>> os.environ['USER'] = 'Bob' # changes for process and its children
Arguments and streams: sys
· sys.argv: command-line arguments
· sys.stdin/stdout/stderr: standard stream files
· sys.executable is path to running interpreter
% type play.py
#!/usr/local/bin/python
import sys
print sys.argv
sys.stdout.write("ta da!\n") # same as: print 'ta da!'
% python play.py -x -i spammify
['play.py', '-x', '-i', 'spammify']
ta da!
·
See getopt,
optparse modules for parsing complex
command lines
· Built-in file objects
For most file applications
· Processing binary data on Windows
Use “rb” and “wb” to suppress line-end translations
· Module os descriptor-based file tools
For special/advanced file processing modes
· Module os filename tools
Deletions, renamings, etc.
· Module os.path tools
File existence, directory tests, size, etc.
· See also
Sockets, pipes, fifos, shelves, DBM files
Single directories
C:\temp>python
>>> import os
>>> os.popen('dir /B').readlines()
['about-pp.html\012', 'python1.5.tar.gz\012', 'about-pp2e.html\012',
'about-ppr2e.html\012', 'newdir\012']
>>> os.popen('ls C:\PP2ndEd').readlines()
['README.txt\012', 'cdrom\012', 'chapters\012', 'etc\012',
'examples\012',
'examples.tar.gz\012', 'figures\012', 'shots\012']
>>> import glob
>>>
glob.glob('C:\PP2ndEd\*')
['C:\\PP2ndEd\\examples.tar.gz',
'C:\\PP2ndEd\\README.txt',
'C:\\PP2ndEd\\shots',
'C:\\PP2ndEd\\figures', 'C:\\PP2ndEd\\examples',
'C:\\PP2ndEd\\etc',
'C:\\PP2ndEd\\chapters', 'C:\\PP2ndEd\\cdrom']
>>> os.listdir('C:\PP2ndEd')
['examples.tar.gz', 'README.txt', 'shots', 'figures', 'examples', 'etc',
'chapters', 'cdrom']
>>> os.listdir(".")
['summer.out', 'summer.py', 'table1.txt', ... ]
Directory trees
>>> import os
>>> def lister(dummy, dirname, filesindir):
... print '[' + dirname + ']'
... for fname in filesindir:
... print os.path.join(dirname, fname) # handle one file
...
>>> os.path.walk('.', lister, None)
[.]
.\about-pp.html
.\python1.5.tar.gz
.\about-pp2e.html
.\about-ppr2e.html
.\newdir
[.\newdir]
.\newdir\temp1
.\newdir\temp2
.\newdir\temp3
.\newdir\more
[.\newdir\more]
.\newdir\more\xxx.txt
.\newdir\more\yyy.txt
>>> import os
>>> for (thisDir, dirsHere, filesHere) in os.walk('.'):
print thisDir, '=>'
for filename in filesHere:
print '\t', filename
. =>
w9xpopen.exe
py.ico
pyc.ico
README.txt
NEWS.txt
…
C:\temp>python
>>> import find
>>> find.find('*')
['.\\about-pp.html', '.\\about-pp2e.html', '.\\about-ppr2e.html',
'.\\newdir', '.\\newdir\\more', '.\\newdir\\more\\xxx.txt',
'.\\newdir\\more\\yyy.txt', '.\\newdir\\temp1', '.\\newdir\\temp2',
'.\\newdir\\temp3', '.\\python1.5.tar.gz']
# list files in dir tree by recursion
import sys, os
def mylister(currdir):
print '[' + currdir + ']'
for file in os.listdir(currdir): # list files here
path = os.path.join(currdir, file) # add dir path back
if not os.path.isdir(path):
print path
else:
mylister(path) # recur into subdirs
if __name__ == '__main__':
mylister(sys.argv[1]) # dir name in cmdline
Example: finding large files
>>> import glob, string, os
>>> glob.glob("*.py")
['cheader1.py', 'finder1.py', 'summer.py']
>>> for name in glob.glob("*.py"):
... os.rename(name, string.upper(name))
...
>>> glob.glob("*.PY")
['FINDER1.PY', 'SUMMER.PY', 'CHEADER1.PY']
#!/usr/bin/python
########################################################
# custom version of the now deprecated find module
# in the standard library--import as "PyTools.find";
# equivalent to the original, but uses os.path.walk,
# has no support for pruning subdirs in the tree, and
# is instrumented to be runnable as a top-level script;
# results list sort differs slightly for some trees;
# exploits tuple unpacking in function argument lists;
########################################################
import fnmatch, os
def find(pattern, startdir=os.curdir):
matches = []
os.path.walk(startdir, findvisitor, (matches, pattern))
matches.sort()
return matches
def findvisitor((matches, pattern), thisdir, nameshere):
for name in nameshere:
if fnmatch.fnmatch(name, pattern):
fullpath = os.path.join(thisdir, name)
matches.append(fullpath)
if __name__ == '__main__':
import sys
namepattern, startdir = sys.argv[1], sys.argv[2]
for name in find(namepattern, startdir):
print name
¨ Spawns (copies) a program
¨ Parent and child run independently
¨ fork spawns processes; system/popen spawn commands
¨ Not on Windows, today (use threads or spawnv)
# starts programs until you type 'q'
import os
parm = 0
while 1:
parm = parm+1
pid = os.fork()
if pid == 0: # copy process
os.execlp('python', 'python', 'child.py', str(parm)) # overlay program
assert 0, 'error starting program' # shouldn't return
else:
print 'Child is', pid
if raw_input() == 'q': break
Example: cross-linking streams
¨ Input Þ connect stdin to program’s stdout: raw_input
¨ Output Þ connect stdout to program’s stdin: print
¨ Also see: os.popen2 call
file: ipc.py
import os
def spawn(prog, args):
pipe1 = os.pipe() # (parent input, child output)
pipe2 = os.pipe() # (child input, parent output)
pid = os.fork() # make a copy of this process
if pid:
# in parent process
os.close(pipe1[1]) # close child ends here
os.close(pipe2[0])
os.dup2(pipe1[0], 0) # sys.stdin = pipe1[0]
os.dup2(pipe2[1], 1) # sys.stdout = pipe2[1]
else:
# in child process
os.close(pipe1[0]) # close parent ends here
os.close(pipe2[1])
os.dup2(pipe2[0], 0) # sys.stdin = pipe2[0]
os.dup2(pipe1[1], 1) # sys.stdout = pipe1[1]
cmd = (prog,) + args
os.execv(prog, cmd) # overlay new program
¨ Runs function calls in parallel, share global (module) memory
¨ Portable: runs on Windows, Solaris, any with pthreads
¨ Global interpreter lock: one thread running code at a time
¨ Thread switches on bytecode counter and long-running calls
¨ Must still synchronize concurrent updates with thread locks
¨ C extensions release and acquire global lock too
import
thread
def
counter(myId, count):
# synchronize stdout access to avoid multi
prints on 1 line
for i in range(count):
mutex.acquire()
print '[%s] => %s' % (myId, i)
mutex.release()
mutex
= thread.allocate_lock()
for
i in range(10):
thread.start_new(counter, (i, 100))
import
time
time.sleep(10)
print
'Main thread exiting.'
Output
.
.
.
[3] => 98
[4] => 98
[5] => 98
[7] => 98
[8] => 98
[0] => 99
[9] => 98
[6] => 99
[1] => 99
[2] => 99
[3] => 99
[4] => 99
[5] => 99
[7] => 99
[8] => 99
[9] => 99
Main thread exiting.
Locking concurrent updaters
Fails:
# fails on windows due to concurrent updates;
# works if check-interval set higher or lock
# acquire/release calls made around the adds
import thread, time
count = 0
def adder():
global count
count = count + 1 # update shared global
count = count + 1 # thread swapped out before returns
for i in range(100):
thread.start_new(adder, ()) # start 100 update threads
time.sleep(5)
print count
Works:
import thread, time, sys
mutex = thread.allocate_lock()
count = 0
def adder():
global count
mutex.acquire()
count = count + 1 # update shared global
count = count + 1 # thread swapped out before returns
mutex.release()
for i in range(100):
thread.start_new(adder, ()) # start 100 update threads
time.sleep(5)
print count
See also:
“threading” module’s class-based interface
“Queue” module’s thread-safe queue get/put
import threading
class
mythread(threading.Thread): #
subclass Thread object
def __init__(self, myId, count):
self.myId = myId
self.count = count
threading.Thread.__init__(self)
def run(self): # run provides thread
logic
for i in range(self.count): # still synch stdout access
stdoutmutex.acquire()
print '[%s] => %s' % (self.myId,
i)
stdoutmutex.release()
stdoutmutex =
threading.Lock() # same as
thread.allocate_lock()
thread = mythread()
thread.start()
thread.join() # wait for exit
Examples: Queue module, on CD Extras/PP3E
· For starting programs on Windows
· Spawnv like fork+exec for Unix
· See also: os.system(“start file.py”)
############################################################
# do something simlar by forking process
instead of threads
# this doesn't currently work on Windows,
because it has no
# os.fork call; use os.spawnv to start
programs on Windows
# instead; spawnv is roughly like a
fork+exec combination;
############################################################
import os, sys
for i in range(10):
if sys.platform[:3] == 'win':
path = r'C:\program files\python\python.exe'
os.spawnv(os.P_DETACH, path,
('python',
'thread-basics6.py'))
else:
pid = os.fork()
if pid != 0:
print 'Process %d spawned' % pid
else:
os.execlp('python', 'python', 'thread-basics6.py')
print 'Main process exiting.'
site-forward.py
#######################################################
# Create forward link pages for relocating
a web site.
# Generates one page for every existing
site file;
# upload the generated files to your old
web site.
#######################################################
import os, string
uploaddir = 'rmi-forward' # where to store forward files
servername = 'starship.python.net' # where site is relocating to
homedir = '~lutz/home' # where site will be rooted
sitefilesdir = 'public_html' # where site files live locally
templatename = 'template.html' # template for generated pages
template
= open(templatename).read()
sitefiles = os.listdir(sitefilesdir) # filenames, no dir prefix
count = 0
for filename in sitefiles:
fwdname = os.path.join(uploaddir, filename) # or +os.sep+filename
print 'creating', filename, 'as', fwdname
filetext = string.replace(template, '$server$', servername)
filetext = string.replace(filetext, '$home$', homedir)
filetext = string.replace(filetext, '$file$', filename)
open(fwdname, 'w').write(filetext)
count = count + 1
print 'Last file =>\n', filetext
print 'Done:', count, 'forward files
created.'
template.html
<HTML><BODY>
<H1>This page has moved</H1>
<P>This page now lives at this address:
<P><A
HREF="http://$server$/$home$/$file$">
http://$server$/$home$/$file$</A>
<P>Please click on the new address to jump to
this page, and
update any links accordingly.
</P>
</BODY></HTML>
¨ pack1 puts files in a single file, with separator lines
¨ unpack1 recreates original files from a ‘pack1’ file
¨ unmore unpacks result of a “more <files>” command
file: pack1.py
#!/usr/local/bin/python
import sys # load the system module
marker = ':'*6
for name in sys.argv[1:]: # for all command arguments
input = open(name, 'r') # open the next input file
print marker + name # write a separator line
print input.read(), # write the file's contents
file: unpack1.py
#!/usr/local/bin/python
import sys
marker = ':'*6
for line in sys.stdin.readlines(): # for all input lines
if line[:6] != marker:
print line, # write real lines
else:
sys.stdout = open(line[6:-1], 'w')
Unmore: scripts, functions, classes
· ‘more’ writes 3 lines before each file
::::::::::::::
filename
::::::::::::::
<file text>
file unmore.py
#!/usr/local/bin/python
# unpack result of "more x y z > f"
# usage: "% unmore.py f" or "% unmore.py < f"
# uses simple top-level script logic
import sys
marker = ':'*14
try:
input = open(sys.argv[1], "r")
except:
input = sys.stdin
output = sys.stdout
while 1:
line = input.readline()
if not line: # end of file?
break
elif line[:14] != marker: # text line?
output.write(line)
else: # file prefix
fname = input.readline()[:-1] # strip eoln ('\n')
print 'creating', `fname`
output = open(fname, "w") # next output
line = input.readline() # end of prefix
if line[:14] != marker:
print "OOPS!"; sys.exit(1)
print 'Done.'
Adding a functional interface
file: unmore2.py
#!/usr/local/bin/python
# unpack result of "more x y z > f"
# usage: "unmore2.py f" or "unmore2.py < f"
# packages unpacking logic as an importable function
import sys
marker = ':'*14
def unmore(input):
output = sys.stdout
while 1:
line = input.readline()
if not line: # end of file?
break
elif line[:14] != marker: # text line?
output.write(line)
else: # file prefix
fname = input.readline()[:-1] # strip eoln
print 'creating', `fname`
output = open(fname, "w") # next output
if input.readline()[:14] != marker:
print "OOPS!"; sys.exit(1)
if __name__ == '__main__':
if len(sys.argv) == 1:
unmore(sys.stdin) # unmore2.py < f
else:
unmore(open(sys.argv[1], 'r')) # unmore2.py f
print 'Done.'
% unmore2.py t.more
creating 't1.txt'
creating 't2.txt'
Done.
% python
>>> from unmore2 import unmore
>>> unmore(open("t.more", "r"))
creating 't1.txt'
creating 't2.txt'
Using a class framework
file: unmore3.py
#!/usr/local/bin/python
# unpack result of "more x y z > f"
# usage: "unmore3.py -i f -v?" or "unmore3.py -v? < f"
# uses StreamApp class from "Programming Python"
from apptools import StreamApp
marker = ':'*14
class UnmoreApp(StreamApp): # app subclass
def run(self): # start/run/stop
while 1:
line = self.readline()
if not line: # end of file?
break
elif line[:14] != marker: # text line?
self.write(line)
else: # file prefix
fname = self.readline()[:-1]
self.message('creating ' + `fname`)
self.setOutput(fname)
if self.readline()[:14] != marker:
self.exit("OOPS!")
if __name__ == '__main__': UnmoreApp().main() # make/run
% setenv PYTHONPATH
".:/pp/examples/other/framewrk:/usr/local/lib/python"
% unmore3.py -i t.more -v
UnmoreApp start.
creating 't1.txt'
creating 't2.txt'
UnmoreApp done.
% python
>>> from unmore3 import UnmoreApp
>>> app = UnmoreApp()
>>> app.setInput("t.more")
>>> app.main()
These are suggested reading, if you are looking for something to do during the lab session.
File: fixeoln_one.py
#########################################################
# Use:
"python fixeoln_one.py [tounix|todos] filename".
#
convert end-lines in the single text file whose name
# is
passed in on the command line, to the target form
# (unix
or dos). The _one, _dir, and _all
converters
#
resuse the convert function here; we could implement
# this
by inspecting command-line argument patterns
#
instead of writing 3 separate scripts, but that can
# become
complex for the user. convertEndlines
changes
#
endlines only if necessary--lines that are already in
# the
target format are left unchanged, so it's okay to
#
convert a file > once in any of the 3 fixeoln scripts.
#########################################################
def
convertEndlines(format, fname): # convert one file
newlines = [] #
todos: \n => \r\n
for line in open(fname,
'r').readlines(): # tounix:
\r\n => \n
if format == 'todos':
if line[-1:] == '\n' and
line[-2:-1] != '\r':
line = line[:-1] + '\r\n'
elif format == 'tounix': # avoids IndexError
if line[-2:] == '\r\n': # slices are scaled
line = line[:-2] + '\n'
newlines.append(line)
open(fname, 'w').writelines(newlines)
if
__name__ == '__main__':
import sys
errmsg = 'Required arguments missing:
["todos"|"tounix"] filename'
assert (len(sys.argv) == 3 and sys.argv[1]
in ['todos', 'tounix']), errmsg
convertEndlines(sys.argv[1], sys.argv[2])
print 'Converted', sys.argv[2]
File: fixeoln_dir.py
#########################################################
# Use: "python
fixeoln_dir.py [tounix|todos] patterns?".
#
convert end-lines in all the text files in the current
#
directory (only: does not recurse to subdirectories).
#
Resuses converter in the single-file _one version.
#########################################################
import
sys, glob
from
fixeoln_one import convertEndlines
listonly
= 0
patts =
['*.py', '*.txt', '*.c', '*.cxx', '*.h', '*.i', 'makefile*', 'output*']
if
__name__ == '__main__':
errmsg = 'Required first argument missing:
"todos" or "tounix"'
assert (len(sys.argv) >= 2 and
sys.argv[1] in ['todos', 'tounix']), errmsg
if len(sys.argv) > 2: # glob anyhow: '*' not applied
on dos
patts = sys.argv[2:] # though not really needed on
linux
filelists = map(glob.glob, patts) # name matches in this dir only
count = 0
for list in filelists:
for fname in list:
print count+1, '=>', fname
if not listonly:
convertEndlines(sys.argv[1],
fname)
count = count + 1
print 'Converted %d files' % count
File: fixeoln_all.py
#########################################################
# Use:
"python fixeoln_all.py [tounix|todos] patterns?".
# find and
convert end-of-lines in all text files
# at
and below the directory where this script is
# run
(the dir you are in when you type 'python').
# If
needed, tries to use the Python find.py lib
#
module, else reads the output of a unix-style
# find
executable command; we could also use a
# find
-exec option to spawn a coverter script.
# Uses
default filename patterns list if absent.
#
#
Example:
# cd Html\Examples
# python ..\..\Tools\fixeoln_all.py tounix
#
#
converts any DOS end-lines to UNIX end-lines, in
# all
text files in and below the Examples directory
#
(i.e., all source-code files). Replace
"tounix" with
#
"todos" to convert any UNIX end-lines to DOS form
#
instead. This script only changes files
that need to
# be
changed, so it's safe to run brute-force from a
#
root-level directory to force platform conformance.
#
"python ..\..\Tools\fixeoln_all.py tounix *.txt"
#
converts just .txt files (quote on UNIX: "*.txt").
# See
also: fixeoln_one and fixeoln_dir versions.
#########################################################
import
os, sys, string
debug = 0
pyfind = 0
# force py find
listonly
= 0
def
findFiles(patts, debug=debug, pyfind=pyfind):
try:
if sys.platform[:3] == 'win' or pyfind:
print 'Using Python find'
import find # use python
lib find.py
matches = map(find.find,
patts) # start dir default =
'.'
else:
print 'Using find executable'
matches = []
for patt in patts:
findcmd = 'find . -name
"%s" -print' % patt # run find
command
lines =
os.popen(findcmd).readlines() #
remove endlines
matches.append(map(string.strip, lines)) # lambda x: x[:-1]
except:
assert 0, 'Sorry - cannot find files'
if debug: print matches
return matches
if
__name__ == '__main__':
from fixeoln_dir import patts
from fixeoln_one import convertEndlines
errmsg = 'Required first argument missing:
"todos" or "tounix"'
assert (len(sys.argv) >= 2 and
sys.argv[1] in ['todos', 'tounix']), errmsg
if len(sys.argv) > 2: # quote in unix shell
patts = sys.argv[2:] # else tries to expand
matches = findFiles(patts)
count = 0
for matchlist in matches: # a list of lists
for fname in matchlist: # one per pattern
print count+1, '=>', fname
if not listonly:
convertEndlines(sys.argv[1],
fname)
count = count + 1
print 'Converted %d files' % count
Click here to go to
lab exercises
Click here to go to
exercise solutions
Click here to go to
solution source files
Click here to go to
lecture example files