fix pyinstaller

amongst other things, fix #1746

1 files changed, 37 insertions, 89 deletions

diff --git a/release/rtool.py b/release/rtool.py
index 3bc3fdaa..f40e8dcb 100755
--- a/release/rtool.py
+++ b/release/rtool.py
@@ -8,10 +8,9 @@ import runpy
 import shlex
 import shutil
 import subprocess
-import sys
 import tarfile
 import zipfile
-from os.path import join, abspath, normpath, dirname, exists, basename
+from os.path import join, abspath, dirname, exists, basename
 
 import click
 import pysftp
@@ -20,8 +19,13 @@ import pysftp
 # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
 if platform.system() == "Windows":
     VENV_BIN = "Scripts"
+    PYINSTALLER_ARGS = [
+        # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
+        "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
+    ]
 else:
     VENV_BIN = "bin"
+    PYINSTALLER_ARGS = []
 
 # ZipFile and tarfile have slightly different APIs. Fix that.
 if platform.system() == "Windows":
@@ -46,13 +50,9 @@ PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
 PYINSTALLER_DIST = join(BUILD_DIR, "binaries")
 
 VENV_DIR = join(BUILD_DIR, "venv")
-VENV_PIP = join(VENV_DIR, VENV_BIN, "pip")
-VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller")
 
 # Project Configuration
 VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
-PROJECT_NAME = "mitmproxy"
-PYTHON_VERSION = "py2.py3"
 BDISTS = {
     "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
     "pathod": ["pathoc", "pathod"]
@@ -62,20 +62,20 @@ if platform.system() == "Windows":
 
 TOOLS = [
     tool
-    for tools in BDISTS.values()
+    for tools in sorted(BDISTS.values())
     for tool in tools
 ]
 
 
-def get_version() -> str:
-    return runpy.run_path(VERSION_FILE)["VERSION"]
-
-
 def git(args: str) -> str:
     with chdir(ROOT_DIR):
         return subprocess.check_output(["git"] + shlex.split(args)).decode()
 
 
+def get_version() -> str:
+    return runpy.run_path(VERSION_FILE)["VERSION"]
+
+
 def get_snapshot_version() -> str:
     last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2)
     tag_dist = int(tag_dist)
@@ -109,26 +109,12 @@ def archive_name(bdist: str) -> str:
 
 
 def wheel_name() -> str:
-    return "{project}-{version}-{py_version}-none-any.whl".format(
-        project=PROJECT_NAME,
+    return "mitmproxy-{version}-py3-none-any.whl".format(
         version=get_version(),
-        py_version=PYTHON_VERSION
     )
 
 
 @contextlib.contextmanager
-def empty_pythonpath():
-    """
-    Make sure that the regular python installation is not on the python path,
-    which would give us access to modules installed outside of our virtualenv.
-    """
-    pythonpath = os.environ.get("PYTHONPATH", "")
-    os.environ["PYTHONPATH"] = ""
-    yield
-    os.environ["PYTHONPATH"] = pythonpath
-
-
-@contextlib.contextmanager
 def chdir(path: str):
     old_dir = os.getcwd()
     os.chdir(path)
@@ -156,58 +142,8 @@ def contributors():
             f.write(contributors_data.encode())
 
 
-@cli.command("wheel")
-def make_wheel():
-    """
-    Build wheel
-    """
-    with empty_pythonpath():
-        if exists(DIST_DIR):
-            shutil.rmtree(DIST_DIR)
-
-        print("Creating wheel...")
-        subprocess.check_call(
-            [
-                "python3", "./setup.py", "-q",
-                "bdist_wheel", "--dist-dir", DIST_DIR, "--universal"
-            ],
-            cwd=ROOT_DIR
-        )
-
-        print("Creating virtualenv for test install...")
-        if exists(VENV_DIR):
-            shutil.rmtree(VENV_DIR)
-        subprocess.check_call(["python3", "-m", "virtualenv", "-q", VENV_DIR])
-
-        with chdir(DIST_DIR):
-            print("Install wheel into virtualenv...")
-            # lxml...
-            if platform.system() == "Windows" and sys.version_info[0] == 3:
-                subprocess.check_call(
-                    [VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"]
-                )
-            subprocess.check_call([VENV_PIP, "install", "-q", wheel_name()])
-
-            print("Running tools...")
-            for tool in TOOLS:
-                tool = join(VENV_DIR, VENV_BIN, tool)
-                print("> %s --version" % tool)
-                print(subprocess.check_output([tool, "--version"]).decode())
-
-            print("Virtualenv available for further testing:")
-            print("source %s" % normpath(join(VENV_DIR, VENV_BIN, "activate")))
-
-
 @cli.command("bdist")
-@click.option("--use-existing-wheel/--no-use-existing-wheel", default=False)
-@click.argument(
-    "pyinstaller_version",
-    envvar="PYINSTALLER_VERSION",
-    default="git+https://github.com/pyinstaller/pyinstaller.git@483c819d6a256b58db6740696a901bd41c313f0c"
-)
-@click.argument("setuptools_version", envvar="SETUPTOOLS_VERSION", default="setuptools>=25.1.0,!=25.1.1")
-@click.pass_context
-def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version):
+def make_bdist():
     """
     Build a binary distribution
     """
@@ -216,37 +152,40 @@ def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version)
     if exists(PYINSTALLER_DIST):
         shutil.rmtree(PYINSTALLER_DIST)
 
-    if not use_existing_wheel:
-        ctx.invoke(make_wheel)
-
-    print("Installing PyInstaller and setuptools...")
-    subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version, setuptools_version])
-    print(subprocess.check_output([VENV_PIP, "freeze"]).decode())
+    os.makedirs(DIST_DIR, exist_ok=True)
 
-    for bdist, tools in BDISTS.items():
+    for bdist, tools in sorted(BDISTS.items()):
         with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
             for tool in tools:
+                # We can't have a folder and a file with the same name.
+                if tool == "mitmproxy":
+                    tool = "mitmproxy_main"
                 # This is PyInstaller, so it messes up paths.
                 # We need to make sure that we are in the spec folder.
                 with chdir(PYINSTALLER_SPEC):
                     print("Building %s binary..." % tool)
+                    excludes = []
+                    if tool != "mitmweb":
+                        excludes.append("mitmproxy.tools.web")
+                    if tool != "mitmproxy_main":
+                        excludes.append("mitmproxy.tools.console")
                     subprocess.check_call(
                         [
-                            VENV_PYINSTALLER,
+                            "pyinstaller",
                             "--clean",
                             "--workpath", PYINSTALLER_TEMP,
                             "--distpath", PYINSTALLER_DIST,
                             "--additional-hooks-dir", PYINSTALLER_HOOKS,
-                            # PyInstaller 3.2 does not handle Python 3.5's ucrt correctly.
-                            "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
                             "--onefile",
                             "--console",
                             "--icon", "icon.ico",
                             # This is PyInstaller, so setting a
                             # different log level obviously breaks it :-)
                             # "--log-level", "WARN",
-                            tool
                         ]
+                        + [x for e in excludes for x in ["--exclude-module", e]]
+                        + PYINSTALLER_ARGS
+                        + [tool]
                     )
                     # Delete the spec file - we're good without.
                     os.remove("{}.spec".format(tool))
@@ -255,6 +194,15 @@ def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version)
                 executable = join(PYINSTALLER_DIST, tool)
                 if platform.system() == "Windows":
                     executable += ".exe"
+
+                # Remove _main suffix from mitmproxy executable
+                if executable.startswith("mitmproxy_main"):
+                    shutil.move(
+                        executable,
+                        executable.replace("_main", "")
+                    )
+                    executable = executable.replace("_main", "")
+
                 print("> %s --version" % executable)
                 print(subprocess.check_output([executable, "--version"]).decode())
 
@@ -306,7 +254,7 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel,
             if wheel:
                 files.append(wheel_name())
             if bdist:
-                for bdist in BDISTS.keys():
+                for bdist in sorted(BDISTS.keys()):
                     files.append(archive_name(bdist))
 
             for f in files: