""" ================================================================================ subprocproxy.py - run user code file (part of PyEdit [3.0]) Used by the Run-Code Capture mode to launch program files with custom context. The target program (argv[1]) executes as though it were run directly, but with a custom input() function, and the standard streams of this proxy's process. Target is an absolute path, and any command-line arguments for it are argv[2:]; it runs in this process (streams) + program (modules) + namespace (globals). In more detail: PyEdit uses subprocess.Popen to run this script in a spawned process, with this script's standard streams redirected to objects mapped to the GUI (stdout and stderr are read by polling; stdin is provided on request). Because the target file's code run by the exec() here runs in the same process as this proxy, its streams also map to PyEdit's GUI automatically. The initial trick here is to redefine the built-in input() to flush prompts out on a new line; otherwise, input is required in the GUI _before_ prompts appear. This also changes CWD because we can't do so in the Popen call used to start this proxy: we need to run this file first here, before the target script. We also need to forge command-line and import-path settings in stdlib modules, and other run-time context in the shared main-module namespace (i.e., __file__). Besides input(), later additions here write a frozen exe's temp folder name for deletion in PyEdit on process kills; force frozen exe streams to UTF-8 and binary; and apply Python executable and path configurations as described ahead. The following work too, but error messages have extra lines and no filenames: py3 -c 'exec(open("fixinput.py").read()); exec(open("scr.py").read())' py3 -c 'import runpy; runpy.run_path("fixinput.py"); runpy.run_path("scr.py")' Note that this script may be run as source code (for source and Mac app pkgs), or as a frozen executable (for Windows and Linux frozen exe pkgs, because no Python executable may be present). In the latter case, exception output edits differ here, and this script's launch in PyEdit must pass shell=True on Windows to suppress an extra Command Prompt. See build-app-exe/windows/build.py docs; this proxy runs as frozen exe only if the user gave no preferred Python config. Former caveat: always run by same Python as PyEdit (3.X), regardless of '#!' lines, both when run as source and as frozen Window/Linux executable. UPDATE: users can now set Python-interpreter and module-import paths to be used in Capture mode, in textConfig.py. This may be required for frozen app/exe distributions, as the frozen Python does not reflect the user's installs, and no separate Python is assumed. See textConfig.py for more. This proxy is also now runnable on both Python 2.X and 3.X, because of this extension: it allows the target script to be run under either version too. Former caveat: when run as a frozen PyInstaller exe on Windows, killing this by closing the Gui run window pops up a Command Prompt momentarily. UPDATE: this popup is now avoided, by running the Windows taskkill with subprocess.Popen() magic: see launch/kill code in textEditor.py for more. Caveats: 1) This uses "_X" names, function scope locals, and "del" to minimize impact to the spawned code's namespace, but a few (stdlib modules) are left behind. 2) PYTHONPATH may or may not work, depending on how PyEdit was launched: on Mac (at least), it's used if PyEdit itself was launched from a command line. 3) When run as a frozen PyInstaller exe (Windows+Linux), this incurs a startup speed hit to unpack its content to a temp folder (see also temps prune below). PyInstaller one-folder builds do not unpack, but create huge install folders that are not an option for PyEdit itself, but might be for this proxy - and would obviate the need to clean up its temp folders on forced kills. ================================================================================ """ import sys, os, traceback #------------------------------------------------------------------------------- # Import the user config file # # Details: this proxy uses import-path extensions for any local user libs # (textEditor uses the python exe config to override the frozen proxy py). # Config file is in same dir as this proxy __main__, in all launch modes. # PYTHONPATH may work too, but requires running PyEdit from a shell on Mac. #------------------------------------------------------------------------------- _exedir = None if hasattr(sys, 'frozen') and sys.platform.startswith(('win', 'linux')): # Windows+Linux PyInstaller executables distro, frozen proxy: # use exe's path for textConfig.py (see fixfrozenpaths.py for # the full story; it's not imported here to minimize coupling); _exepath = sys.argv[0] _exedir = os.path.abspath(os.path.dirname(_exepath)) if _exedir.endswith('__pyedit-component-data__'): # nested in pymailgui build subdir?: look up _exedir = os.path.abspath(os.path.join(_exedir, '..')) sys.path.append(_exedir) from textConfig import RunCode_PYTHONPATH_APPENDS # default is empty list from textConfig import RunCode_PYTHONPATH_PREPENDS # default is empty list # remove proxy's dir from import paths: irrelevant to script (unless PyEdit) _mydir = os.path.dirname(os.path.abspath(__file__)) while _mydir in sys.path: sys.path.remove(_mydir) # leave no footprints (mostly) # drop pyinstaller path kludge too if _exedir != None: while _exedir in sys.path: sys.path.remove(_exedir) #------------------------------------------------------------------------------- # Clean up frozen-executable temp folders # # UPDATE: the prune of temp folders here was replaced with a stdout write of # this proxy's temp folder name. The parent PyEdit process reads and saves # this folder name, and prunes it on run-window close (subproc kill) if it is # still present, and only when this proxy is run as a frozen executable. # Downside - the proxy generates an extra output line if run outside PyEdit. # # The prune code, retained as example only, was error-prone. It assumed that # the full content of an active PyEdit or proxy (or other frozen exe) instance # was in-use and hence unremovable, causing rmtree() to abort. In practice, # the prune of an active instance's temp folder would not abort until some of # its content was already removed. In tests, prunes deleted 25 files before # reaching the first lock -- all dlls and perhaps harmless, but perhaps not. # # Other options explored: # # 1) Prescanning the tree to detect any-file-in-use cases would be slow, and # may trigger a race condition: the active instance may grab or release any # files between the prescan and the prune. # 2) Allowing the proxy to exit more cleanly by using a non-forcing scheme # (e.g. ctrl-break or C-lib calls on Windows, Popen.terminate() on Unix) # may help, but would not be portable, and requires too much console magic # on Windows (spawning at all is absurdly complex on that platform). # 3) A PyInstaller one-dir frozen package avoids the temp-dir unpack altogether, # but users get a massive folder that obscures the exe, and restructuring # this folder manually proved elusive (see build-app-exe/windows/bbild.py). # # Defunct but relevant prune-code notes follow: # # Details: when this proxy runs as a PyInstaller frozen executable, it unpacks # a large folder to the system temp folder (hence the startup delay), which is # left behind as trash each time a spawned program is killed by the user on # run-window close (or crashes outright). This is especially bad on Windows, # where the temp folder is never automatically purged: each forced-kill of # spawned code leaves behind a huge, persistent temp folder. # # To fix, clean up prior temp trash here; Windows disallows deletes for items # still in use (presumably: this is key to correctness here). This does not # apply to proxies run as source code, including py2app Mac app bundles. We # could run this in the PyEdit exe too, but that is less likely to leave crud. #------------------------------------------------------------------------------- # # write my temp folder name to the parent PyEdit, for auto-prune on exit # if hasattr(sys, 'frozen') and sys.platform.startswith(('win', 'linux')): # Windows+Linux PyInstaller executables distro, frozen proxy; # not: Mac app, source-code distro, or frozen but configured Py; # flush() is required, else PyEdit GUI may hang waiting for this! sys.stdout.write('%s\n' % sys._MEIPASS) sys.stdout.flush() # # DEFUNCT: the prior-run prune option (see above) (delete me soon) # """ if hasattr(sys, 'frozen') and sys.platform.startswith(('win', 'linux')): # Windows+Linux PyInstaller executables distro, frozen proxy: # prune prior run temp dirs, from crashes or Run Code kills; # trace output here (if any) shows up in Capture run window; def _pruneTemps(trace=True): import shutil mytemp = sys._MEIPASS temproot = os.path.dirname(mytemp) myfolder = os.path.basename(mytemp) if trace: print('Scanning ' + temproot) # 3.X flush=True not an option sys.stdout.flush() for itemname in os.listdir(temproot): if itemname.startswith('_MEI') and itemname != myfolder: if trace: print('\tReaping' + itemname) sys.stdout.flush() fullpath = os.path.join(temproot, itemname) try: # assume in-use = locked (e.g., PyEdit parent!) shutil.rmtree(fullpath) except Exception as why: if trace: print('\t\tCannot reap %s [%s]' % (fullpath, why)) sys.stdout.flush() _pruneTemps(False) # hide names """ #------------------------------------------------------------------------------- # Wrap stdout/stderr to ensure stream flushes and UTF8 encoding for frozen exes # # When this proxy is run as a frozen Windows PyInstaller (not when source code), # its stdout/stderr output stream appears to sometimes be delayed by buffering, # depite PYTHONUNBUFFERED being passed to Popen's env and binary streams being # invoked via its universal_newlines=False. # # This doesn't happen when the proxy is run in source form (including Mac apps), # where "-u" is passed to a python -- strongly suggesting that PYTHONUNBUFFERED # is ignored by freezes. This may be an artifact of a Popen shell layer, or # Pystaller's "--noconsole" or Python options support (that a .spec file might # be able to address), but this project has no time to explore further. # # As a simple WORKAROUND, force these streams to flush() each line here as it # is written by the target script. The PyEdit parent also forces UTF8 stream # encoding in the spawnee and manually decodes binary-mode lines to support # Unicode stream contents, but this appears unrelated to the buffering issue. # # UPDATE: the frozen exe proxy _also_ does not respect PYTHONIOENCODING env # vars passed through subprocess.Popen, and hence cannot handle arbitrary # Unicode (this should make the stream UTF8, but doesn't). This seems to be # a general issue, in PyInstaller's boot loader, subprocess.Popen, or both. # The most likely suspect is PyInstaller's bootloader for "--noconsole" exes, # because the required env vars are indeed passed to this proxy and the code # it runs; per a contexdump.py run: # # PYTHONIOENCODING # UTF8 # PYTHONUNBUFFERED # True # # These settings are dropped somewhere in the PyInstaller/Python startup chain # (PyInstaller has null-write wrappers that don't seem used for Popen streams). # Worse in this case, a PyInstaller spec file may not help: unlike the "-u" # unbuffered streams mode, IO encoding is not a Python command arg, and is # not a Python run-time option supported by spec files. PyInstaller 'runtime # hooks' also seem irrelevant, as they run too late, after a Python has started. # # TO FIX the Unicode issue, _also_ go under the hood of the io text layer and # write encoded bytes (not Unocode code points) to the stream... as probably # should have happened automatically, but Python 3.X's -u has grown confused. # Alas, this project has faced one too many frozen-executable issues to tarry. #------------------------------------------------------------------------------- if hasattr(sys, 'frozen') and sys.platform.startswith(('win', 'linux')): class _AutoFlushEncode: def __init__(self, stream): self.stream = stream def write(self, data): #res = self.stream.write(data) res = self.stream.buffer.write(data.encode('UTF8')) # yes, eek. if data.endswith('\n'): self.stream.flush() return res def writelines(self, lines): for line in lines: self.write(line) def __getattr__(self, attr): # and everything else return getattr(self.stream, attr) sys.stdout = _AutoFlushEncode(sys.stdout) sys.stderr = _AutoFlushEncode(sys.stderr) #------------------------------------------------------------------------------- # Customize and replace input() #------------------------------------------------------------------------------- def input(prompt=''): """ force input() to print prompt with newline and flush; needed for GUI that separates prompt and input line; this also blocks GUIs (e.g., if PyEdit runs itself!); """ if prompt: sys.stdout.write(prompt + '\n') sys.stdout.flush() return sys.stdin.readline().rstrip('\n') if sys.version[0] >= '3': import builtins # it's input in python 3.X builtin_input = builtins.input # save as a fallback option builtins.input = input # reset for imported mods too elif sys.version[0] == '2': raw_input = input # redef raw_input() in 2.X del input # restore 2.X input() import __builtin__ builtin_raw_input = __builtin__.raw_input # ditto, but for python 2.X __builtin__.raw_input = raw_input # reset for imported mods too #------------------------------------------------------------------------------- # Forge global context - files, args, imports, __file__ #------------------------------------------------------------------------------- _scriptpath = sys.argv[1] # abs target script=cmd arg #1 _scriptdir = os.path.dirname(_scriptpath) # any args for target are #2..N # tbd: strip script's dir in argv? os.chdir(_scriptdir) # cd to script's run dir: files sys.argv = sys.argv[1:] # strip proxy at front of args sys.path.insert(0, _scriptdir) # add script dir to import path __file__ = _scriptpath # __name__ is still __main__ # add any local libs' import paths, augmenting sys.path presets if RunCode_PYTHONPATH_APPENDS: for _dirname in RunCode_PYTHONPATH_APPENDS: if os.path.isdir(_dirname) and _dirname not in sys.path: sys.path.append(_dirname) # ditto, but to front of path for precedence, and in order given if RunCode_PYTHONPATH_PREPENDS: for _dirname in reversed(RunCode_PYTHONPATH_PREPENDS): if os.path.isdir(_dirname): sys.path.insert(0, _dirname) # even if later on path # delete globals del RunCode_PYTHONPATH_APPENDS, RunCode_PYTHONPATH_PREPENDS # code run is source, not frozen: clear flag in case it checks (e.g., Pyedit) if hasattr(sys, 'frozen'): _proxyfrozen = sys.frozen # but save for exc parsing test delattr(sys, 'frozen') else: _proxyfrozen = False #------------------------------------------------------------------------------- # Run script's code in this process + program + namespace #------------------------------------------------------------------------------- _text = open(_scriptpath, 'rb').read() # use bytes to ignore encoding try: # compile and run - as if pasted here _code = compile(_text, _scriptpath, 'exec') # syntax errors happen here exec(_code) # in my namespace: input, _file_ except Exception as why: # errors same as console - sans self display = traceback.format_exc() # the usual error display display = display.split('\n') # chop up on line breaks # one less output line to strip if frozen exe (not Mac app) isexe = _proxyfrozen and _proxyfrozen != 'macosx_app' if type(why) is SyntaxError: # drop tb header + proxy line(s) del display[0:(3 - isexe)] # first 3 (or 2) lines else: # drop the proxy's line(s) only del display[1:(3 - isexe)] # line 2 and 3 (or just 2) sys.stderr.write('\n'.join(display)) # as if script run directly #------------------------------------------------------------------------------- # And exit proxy+script process... (unless killed by window or PyEdit close) #-------------------------------------------------------------------------------