#!/usr/bin/env python3
"""
=========================================================================
Collects all standard modules with paths, and inserts them into the
config file for py2app, so app users can run near-arbitrary Python code.
Everything added here will be "baked in" to the app bundle's Python.

Else, frozen apps/exes include _only_ modules used by the frozen script.
Run me from a build.py or other script to expand setup.py's includes line.

Inspired by ideas and derived from initial code (modified much here) at:
    http://grokbase.com/t/python/pythonmac-sig/07a3r5s1k2/
        was-py2app-how-to-include-the-entire-standard-library-of-modules

CAVEAT: this adds only modules in the install's Lib (lib) source dir.
It may miss any others among standard-lib zipfiles, built-in or other
modules coded in C, and any elsewhere.  See PyDoc for all the many sources
of modules in a Python.  Builtin modules on sys.builtin_module_names are
likely okay, as they should be present in the bundle's Python (they are
statically linked in, presumably).  Other C mods in lib-dynload were
added here, and the stdlib zip file is absent in 3.5 on Mac.

Skipping site-packages underscores the OTHER ISSUE here: even if the
frozen app/exe includes all of Python's stdlib, it cannot include any
extension libraries users may have installed on their own local machine.
Hence, users should also be able to fall back on a separately-installed
Python and module search-path settings if needed.  PyEdit does this by
allowing both to be configured in textConfig.py.  This file's path
settings will be required for nontrivial programs, because PyEdit cannot
push the user's PYTHONPATH env setting down to spawned code portably.

IDLE can avoid this mess by assuming it's installed with a full Python:
its app starts up with the frozen app's minimal Python, but then restarts
itself with the full/general Python which was installed alongside it.
Other IDEs like Eclipse+PyDev require users to specify installed Python
interpreters and import paths in their configuration GUIs.

Lots of programs need to run arbitrary Python code given at runtime.
This should really be a switchable option in py2App and PyInstaller...
=========================================================================
"""

import os
import os.path
import sys
import pprint
path = '/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5'

traceonly = False
modules = []

skip = ['site-packages',    # skip local extensions (but see caveat above)
        'test',             # skip py selftest and idle dev gui 
        'idlelib']          # dropped lib-tk (why? + PyEdit adds tkinter anyhow)

skip += ['lib2to3',         # added: skip the 2.X=>3.X converter utility
         'tests',           # added: skip individual package tests
         'config-3.5m']     # added: seems irrelevant, and bombs in code here

print('Building full stdlib list')

for root, dirs, files in os.walk(path):
    # prune skips: the odd nesting order avoids a dirs.copy()
    for item in skip:
        if item in dirs:    
            print('Omitting:', os.path.join(root, item))
            dirs.remove(item)

    # process files in this root
    for f in files:
        d = root.replace(path, '')
        full_path = os.path.join(d, f)

        # Fix up slashes
        if '\\' in full_path:
            full_path = full_path.replace('\\', '.')
        if '/' in full_path:
            full_path = full_path.replace('/', '.')
        if full_path[0] == '.':
            full_path = full_path[1:]

        # Collapse some auto-visible subdirs (on sys.path)
        """
        if full_path.startswith('plat-mac.'):              # defunct?
            full_path = full_path[9:]
        if full_path.startswith('lib-scriptpackages.'):    # defunct?
            full_path = full_path[18:]            
        """
        if full_path.startswith('plat-darwin.'):           # still present
            full_path = full_path[12:]                     # but 12 (not 11)
        if full_path.startswith('lib-dynload.'):           # added (new?)
            full_path = full_path[12:]
        if full_path.startswith('.'):                      # what for, this?
            continue                                       # don't drop here
        
        # Ignore .pyc and .pyo files (added: .so for lib-dynload)
        if f.endswith(('.py', '.so')) and not f.startswith('_'):

            # added unportable .so hack: fix me
            cmodprefix = '.cpython-35m-darwin.so'
            if f.endswith(cmodprefix):
                full_path = full_path[:-len(cmodprefix)] 
            else:
                full_path = full_path[:-3]

            # Save each part of the file path as part of the module name:
            # foo.foo1.foo2.py has a package foo, a sub-package foo1,
            # and a module foo2. Save foo, foo.foo1, and foo.foo1.foo2.py.

            section_total = full_path.count('.')
            start = 0
            for x in range(section_total):
                stop = full_path.find('.', start)
                if stop != -1:
                    package = full_path[:stop]
                    if package and package not in modules:
                        modules.append(package)
                start = stop + 1
        
            if full_path and full_path not in modules:
                modules.append(full_path)

# modules.remove('anydbm') # This module fails for some reason. (Now moot.)

print('The number of modules is', len(modules))
for mod in modules:
    print(mod)
if traceonly: sys.exit()

# Replace includes line in setup.py (via base-setup.py)
mac_file = open('setup-base.py', 'r', encoding='utf8')     # added utf8: ©, etc.
lines = mac_file.readlines()
mac_file.close()

for x in range(len(lines)):
    if lines[x].startswith('INCLUDES ='):
        formatmodules = pprint.pformat(modules)            # not one giant line:
        lines[x] = 'INCLUDES = ' + formatmodules           # else IDLE etc choke!
        lines.insert(x, '# Generated by %s\n' % __file__)
        break

mac_file = open('setup.py', 'w', encoding='utf8')          # allow copyright, etc.
mac_file.writelines(lines)
mac_file.close()