#!/usr/bin/env python3
"""
=========================================================================
Collects all standard modules with paths, and inserts them into the
config file for PyInstaller, so app users can run near-arbitrary Python
code without installing a Python separately.  Else, frozen apps/exes
include _only_ modules used by the frozen script.  Run me from a build.py
or other script and name this folder as a hooks dir in PyInstaller cmd.

This script's output file, "hook-os.py", is triggered at build time by
an import of module name "os" but ONLY when building the subprocproxy
executable, per the "--additional-hooks-dir" in its PyInstaller command
in build.py.  The full stdlib need be included only when launching user
code, not in PyEdit (textEditor) in general.  This doesn't impact "os"
itself, and hooks are a simpler alternative to PyIstaller spec files.

Inspired by ideas and derived from initial code (modified much here) at:
    http://grokbase.com/t/python/pythonmac-sig/07a3r5s1k2/
        was-py2app-how-to-include-the-entire-standard-library-of-modules

CAVEAT: this adds only modules in the install's Lib (lib) source dir.
It may miss any others among standard-lib zipfiles, built-in or other
modules coded in C, and any elsewhere.  See PyDoc for all the many sources
of modules in a Python.  Builtin modules on sys.builtin_module_names are
likely okay, as they should be present in the bundle's Python (they are
statically linked in, presumably).  Other C mods in DLLs were added here,
and the stdlib zip file is absent in 3.5 on Windows.

Skipping site-packages underscores the OTHER ISSUE here: even if the
frozen app/exe includes all of Python's stdlib, it cannot include any
extension libraries users may have installed on their own local machine
and typically in their site-packages folder.  Hence, users should also
be able to fall back on a separately-installed Python and module
search-path settings if needed.

PyEdit does this by allowing both to be configured in textConfig.py.
This file's path settings will be required for nontrivial programs,
because PyEdit cannot push the user's PYTHONPATH env setting down to
spawned code portably.  For PyInstalller, PyEdit runs a source-code
proxy with the Python (and its libs) configured, else a frozen proxy
with the Python (and its freeze-time libs) in the executable.

IDLE can avoid this mess by assuming it's installed with a full Python:
its app starts up with the frozen app's minimal Python, but then restarts
itself with the full/general Python which was installed alongside it.
Other IDEs like Eclipse+PyDev require users to specify installed Python
interpreters and import paths in their configuration GUIs.

Lots of programs need to run arbitrary Python code given at runtime.
This should really be a switchable option in py2App and PyInstaller...
=========================================================================
"""

import os
import os.path
import sys
import pprint

# or via sys.executable?
path = 'C:\\Users\\mark\\AppData\\Local\\Programs\\Python\\Python35\\Lib'
pyds = 'C:\\Users\\mark\\AppData\\Local\\Programs\\Python\\Python35\\DLLs'

traceonly = False
modules = []

skip = ['site-packages',    # skip local extensions (but see caveat above)
        'test',             # skip py selftest (and other), and idle dev gui 
        'idlelib']          # dropped lib-tk (why? + PyEdit adds tkinter anyhow)

skip += ['lib2to3',         # added: skip the 2.X=>3.X converter utility
         'tests']           # added: skip individual package tests

print('Building full stdlib list')

# special-case .pyd dynamic-load mods on Windows (dir not visited on Lib walk)
for item in os.listdir(pyds):
    if item.endswith('.pyd') and not item.startswith('_'):   # _ used by others
        modules.append(item[:-4])

# walk the main Lib folder
for root, dirs, files in os.walk(path):
    # prune explicit-skip dirs
    for item in dirs.copy():
        if item in skip:
            print('Omitting:', os.path.join(root, item))
            dirs.remove(item)

    # prune plat-* folders: not on sys.path, and unimportable
    for item in dirs.copy():
        if item.startswith('plat-'):
            print('Omitting:', os.path.join(root, item))
            dirs.remove(item)

    # process files in this root
    for f in files:
        d = root.replace(path, '')
        full_path = os.path.join(d, f)

        # Fix up slashes
        if '\\' in full_path:
            full_path = full_path.replace('\\', '.')
        if '/' in full_path:
            full_path = full_path.replace('/', '.')
        if full_path[0] == '.':
            full_path = full_path[1:]
        
        # Ignore .pyc and .pyo files (assume .pyd never mixed into Lib)
        if f.endswith('.py') and not f.startswith('_'):
            full_path = full_path[:-3]

            # Save each part of the file path as part of the module name:
            # foo.foo1.foo2.py has a package foo, a sub-package foo1,
            # and a module foo2. Save foo, foo.foo1, and foo.foo1.foo2.

            section_total = full_path.count('.')
            start = 0
            for x in range(section_total):
                stop = full_path.find('.', start)
                if stop != -1:
                    package = full_path[:stop]
                    if package and package not in modules:
                        modules.append(package)
                start = stop + 1
        
            if full_path and full_path not in modules:
                modules.append(full_path)

print('The number of modules is', len(modules))
for mod in modules:
    print(mod)
if traceonly: sys.exit()

# Store in a PyInstaller hook file in this dir
hook_file = open('hook-os.py', 'w', encoding='utf8')              # added utf8
formatmodules = pprint.pformat(modules)                           # not one line
hook_file.write('# Generated by %s\n' % __file__)
hook_file.write("print('\\n**RUNNING PYEDIT HOOK FILE**\\n')\n")
hook_file.write('hiddenimports = ' + formatmodules)
hook_file.close()