""" ================================================================================ [3.0] Patch Python's multiprocessing (MP) module to work with frozen single-file executables generated by PyInstaller on Windows (only). Cut-and-paste verbatim from this 'official' PyInstaller recipe, except where the code has been marked with added "# ML" comments: https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Multiprocessing In short, MP on Windows must unpickle the callable to run in a new Python process, and this in turn must import the enclosing module; that leads to problems when there is only an executable and no module. And yes, this is A WORKAROUND FOR A WORKAROUND: PyEdit switched to MP for its Grep search due to a random Python/Tk threads crash, and MP doesn't work for the new PyInstaller Windows executables unless patched as here. No, battery dependence is *not* an absolute good. On the upside, PyInstaller fails on Mac for ActiveState Tk, which led to using py2app's app bundles on Mac which don't have this problem; though ActiveState's Tk has Dock menu issues that may require trying Homebrew Python and Tk on Mac, because python.org Mac Python doesn't link to Tk 8.6 where this bug may be fixed. (Did that say "upside"?) ABOUT THIS CODE: This code does fix the problem, but was unchanged here to serve as an example of coding issues. First, it came with one outright bug: it had Python 3.X import logic, but uses Python 2.X print statements that required conversion to 2.X+3.X form (parenthesis do the job). One wonders: was this tested at all on 3.X before it was posted? Beyond the bug, it has other issues that don't exactly inspire a warm and/or fuzzy confidence in the correctness of it or the product it addresses. In the end, all it does is set and unset a simple env var around the constructor, and does nothing at all outside Windows. This seems an peculiar "fix" in the first place; but this patch code also: - Tries to import a non-Windows module--for no apparent reason - Resorts to brittle monkey-patching, when subclassing would be cleaner - Is needlessly convoluted by multiple nested tests for frozen code, when its class is a complete no-op and fully unnecessary if non-frozen. Somehow, it manages to sneak in a so-very-smart super() call anyhow. (Snarky, sure; but this workaround is really off-the-charts hackish.) This code also came with a self-test which did not produce its expected results in either 3.X or 2.X on Windows in IDLE; it worked in a Command Prompt, but was not as clear about this as it might have been. Its self-test also didn't prove any sort of process connection in any event. An alternative test was added to compensate on both counts. Despite its issues, this code has been propagated to multiple forums. In the forums' defense, one thread suggests a cleaner (but unverified and with the same bugs and caveats above) subclassing approach: ... class _Popen(): ... class MyProcess(multiprocessing.Process): _Popen = _Popen # and use MyProcess instead of multiprocessing.Process ... See: http://stackoverflow.com/questions/ 24944558/pyinstaller-built-windows-exe-fails-with-multiprocessing There are also (a woefully few) more details in Python's MP module docs: https://docs.python.org/3/library/ multiprocessing.html#multiprocessing.freeze_support ================================================================================ """ # ML: THE PATCH***************************************************************** import os import sys # Module multiprocessing is organized differently in Python 3.4+ try: # Python 3.4+ if sys.platform.startswith('win'): import multiprocessing.popen_spawn_win32 as forking else: import multiprocessing.popen_fork as forking # ML: WHY? THIS IS UNUSED! except ImportError: import multiprocessing.forking as forking if sys.platform.startswith('win'): # ML: WHY NOT TEST FROZEN HERE? # First define a modified version of Popen. # ML: this is a no-op otherwise class _Popen(forking.Popen): def __init__(self, *args, **kw): if hasattr(sys, 'frozen'): # We have to set original _MEIPASS2 value from sys._MEIPASS # to get --onefile mode working. os.putenv('_MEIPASS2', sys._MEIPASS) try: super(_Popen, self).__init__(*args, **kw) finally: if hasattr(sys, 'frozen'): # On some platforms (e.g. AIX) 'os.unsetenv()' is not # available. In those cases we cannot delete the variable # but only set it to the empty string. The bootloader # can handle this case. if hasattr(os, 'unsetenv'): os.unsetenv('_MEIPASS2') else: os.putenv('_MEIPASS2', '') # Second override 'Popen' class with our modified version. # This happens in all contexts, whether __main__ or not. forking.Popen = _Popen # ML: TESTS********************************************************************* # Example for testing multiprocessing. import multiprocessing # ML: THIS TEST CODE DISABLE FOR PATCH USE class SendeventProcess(multiprocessing.Process): def __init__(self, resultQueue): self.resultQueue = resultQueue multiprocessing.Process.__init__(self) self.start() def run(self): print('SendeventProcess') # ML: ADDED () self.resultQueue.put((1, 2)) print('SendeventProcess') # ML: ADDED () # ML: a simpler test? def sendEventTask(resultQueue): print('task') resultQueue.put((1, 2)) print('task') # ML: MAIN REQUIREMENTS********************************************************* if __name__ == '__main__': # Also enable support for using multiprocessing (MP) in # single-file frozen binaries on Windows: # - On Windows calling this function here is necessary. # - On Linux/OS X (and if not frozen) it does nothing. # This is required only in the frozen program's main, which # is run in the process that spawns the MP child process. multiprocessing.freeze_support() # ML: MUST CALL THIS HERE # ML: added this test print('test') resultQueue = multiprocessing.Queue() multiprocessing.Process(target=sendEventTask, args=(resultQueue,)).start() result = resultQueue.get(block=True) print(result) print('test') # ML: what came with the patch print('main') # ML: ADDED () resultQueue = multiprocessing.Queue() SendeventProcess(resultQueue) # ML: calls own start() print('main') # ML: ADDED () """ Expected output on both Windows and Unix (Mac OS X) when run from a system command prompt (not from IDLE!): test task task (1, 2) test main main SendeventProcess SendeventProcess """