improve release workflow

3 files changed, 388 insertions, 82 deletions

diff --git a/release/README.md b/release/README.md
index d15f88a0..044e0dc9 100644
--- a/release/README.md
+++ b/release/README.md
@@ -1,12 +1,12 @@
 # Release Checklist
 
-Make sure run all these steps on the correct branch you want to create a new
+Make sure to run all these steps on the correct branch you want to create a new
 release for! The command examples assume that you have a git remote called
 `upstream` that points to the `mitmproxy/mitmproxy` repo.
 
 - Verify that `mitmproxy/version.py` is correct
 - Update CHANGELOG
-- Update CONTRIBUTORS
+- Update CONTRIBUTORS: `git shortlog -n -s > CONTRIBUTORS`
 - Verify that all CI tests pass
 - Create a major version branch - e.g. `v4.x`. Assuming you have a remote repo called `upstream` that points to the mitmproxy/mitmproxy repo::
   - `git checkout -b v4.x upstream/master`
@@ -20,16 +20,20 @@ release for! The command examples assume that you have a git remote called
 - Wait for tag CI to complete
 
 ## GitHub Release
-- Create release notice on Github [here](https://github.com/mitmproxy/mitmproxy/releases/new) if not already auto-created by the tag.
-- We DO NOT upload release artifacts to GitHub anymore. Simply add the following snippet to the notice:
+- Create release notice on Github
+  [here](https://github.com/mitmproxy/mitmproxy/releases/new) if not already
+  auto-created by the tag.
+- We DO NOT upload release artifacts to GitHub anymore. Simply add the
+  following snippet to the notice:
   `You can find the latest release packages on our snapshot server: https://snapshots.mitmproxy.org/v<version number here>`
 
 ## PyPi
-- Upload the whl file you downloaded in the prevous step
-- `twine upload ./tmp/snap/mitmproxy-4.0.0-py3-none-any.whl`
+- The created wheel is uploaded to PyPi automatically
+- Please check https://pypi.python.org/pypi/mitmproxy about the latest version
 
 ## Homebrew
-- The Homebrew maintainers are typically very fast and detect our new relese within a day.
+- The Homebrew maintainers are typically very fast and detect our new relese
+  within a day.
 - If you feel the need, you can run this from a macOS machine:
   `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/v<version number here>`
 
@@ -52,9 +56,10 @@ release for! The command examples assume that you have a git remote called
   - Check the build details page again
 
 ## Website
- - Update version here: https://github.com/mitmproxy/www/blob/master/src/config.toml
- - `./build && ./upload-test`
- - If everything looks alright: `./upload-prod`
+ - Update version here:
+   https://github.com/mitmproxy/www/blob/master/src/config.toml
+ - Run `./build && ./upload-test`
+ - If everything looks alright, run `./upload-prod`
 
 ## Docs
   - Make sure you've uploaded the previous version's docs to archive
@@ -64,4 +69,5 @@ release for! The command examples assume that you have a git remote called
 
 ## Prepare for next release
 
- - Last but not least, bump the version on master in [https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) for major releases.
+ - Last but not least, bump the version on master in
+   [https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) for major releases.
diff --git a/release/ci.py b/release/ci.py
index 94b1f13d..7550aae4 100755
--- a/release/ci.py
+++ b/release/ci.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+import glob
+import re
 import contextlib
 import os
 import platform
@@ -73,17 +75,17 @@ TOOLS = [
 TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
 BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
 if TAG:
-    VERSION = TAG
+    VERSION = re.sub('^v', '', TAG)
     UPLOAD_DIR = VERSION
 elif BRANCH:
-    VERSION = BRANCH
+    VERSION = re.sub('^v', '', BRANCH)
     UPLOAD_DIR = "branches/%s" % VERSION
 else:
     print("Could not establish build name - exiting." % BRANCH)
     sys.exit(0)
 
-
 print("BUILD VERSION=%s" % VERSION)
+print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR)
 
 
 def archive_name(bdist: str) -> str:
@@ -99,23 +101,6 @@ def archive_name(bdist: str) -> str:
     )
 
 
-def wheel_name() -> str:
-    return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
-
-
-def installer_name() -> str:
-    ext = {
-        "Windows": "exe",
-        "Darwin": "dmg",
-        "Linux": "run"
-    }[platform.system()]
-    return "mitmproxy-{version}-{platform}-installer.{ext}".format(
-        version=VERSION,
-        platform=PLATFORM_TAG,
-        ext=ext,
-    )
-
-
 @contextlib.contextmanager
 def chdir(path: str):
     old_dir = os.getcwd()
@@ -134,7 +119,7 @@ def cli():
 
 @cli.command("info")
 def info():
-    print("Version: %s" % VERSION)
+    click.echo("Version: %s" % VERSION)
 
 
 @cli.command("build")
@@ -142,23 +127,41 @@ def build():
     """
     Build a binary distribution
     """
+    os.makedirs(DIST_DIR, exist_ok=True)
+    if "WHEEL" in os.environ:
+        build_wheel()
+    else:
+        click.echo("Not building wheels.")
+    build_pyinstaller()
+
+
+def build_wheel():
+    click.echo("Building wheel...")
+    subprocess.check_call([
+        "python",
+        "setup.py",
+        "-q",
+        "bdist_wheel",
+        "--dist-dir", DIST_DIR,
+    ])
+
+    whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
+    click.echo("Found wheel package: {}".format(whl))
+
+    subprocess.check_call([
+        "tox",
+        "-e", "wheeltest",
+        "--",
+        whl
+    ])
+
+
+def build_pyinstaller():
     if exists(PYINSTALLER_TEMP):
         shutil.rmtree(PYINSTALLER_TEMP)
     if exists(PYINSTALLER_DIST):
         shutil.rmtree(PYINSTALLER_DIST)
 
-    os.makedirs(DIST_DIR, exist_ok=True)
-
-    if "WHEEL" in os.environ:
-        print("Building wheel...")
-        subprocess.check_call(
-            [
-                "python",
-                "setup.py", "-q", "bdist_wheel",
-                "--dist-dir", "release/dist",
-            ]
-        )
-
     for bdist, tools in sorted(BDISTS.items()):
         with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
             for tool in tools:
@@ -168,7 +171,7 @@ def build():
                 # This is PyInstaller, so it messes up paths.
                 # We need to make sure that we are in the spec folder.
                 with chdir(PYINSTALLER_SPEC):
-                    print("Building %s binary..." % tool)
+                    click.echo("Building %s binary..." % tool)
                     excludes = []
                     if tool != "mitmweb":
                         excludes.append("mitmproxy.tools.web")
@@ -209,11 +212,11 @@ def build():
                     )
                     executable = executable.replace("_main", "")
 
-                print("> %s --version" % executable)
-                print(subprocess.check_output([executable, "--version"]).decode())
+                click.echo("> %s --version" % executable)
+                click.echo(subprocess.check_output([executable, "--version"]).decode())
 
                 archive.add(executable, basename(executable))
-        print("Packed {}.".format(archive_name(bdist)))
+        click.echo("Packed {}.".format(archive_name(bdist)))
 
 
 def is_pr():
@@ -229,25 +232,40 @@ def is_pr():
 @cli.command("upload")
 def upload():
     """
-        Upload snapshot to snapshot server
+        Upload build artifacts to snapshot server and
+        upload wheel package to PyPi
     """
     # This requires some explanation. The AWS access keys are only exposed to
     # privileged builds - that is, they are not available to PRs from forks.
     # However, they ARE exposed to PRs from a branch within the main repo. This
     # check catches that corner case, and prevents an inadvertent upload.
     if is_pr():
-        print("Refusing to upload a pull request")
+        click.echo("Refusing to upload a pull request")
         return
+
     if "AWS_ACCESS_KEY_ID" in os.environ:
-        subprocess.check_call(
-            [
-                "aws", "s3", "cp",
-                "--acl", "public-read",
-                DIST_DIR + "/",
-                "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
-                "--recursive",
-            ]
-        )
+        subprocess.check_call([
+            "aws", "s3", "cp",
+            "--acl", "public-read",
+            DIST_DIR + "/",
+            "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
+            "--recursive",
+        ])
+
+    upload_pypi = (
+        TAG and
+        "WHEEL" in os.environ and
+        "TWINE_USERNAME" in os.environ and
+        "TWINE_PASSWORD" in os.environ
+    )
+    if upload_pypi:
+        filename = "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
+        click.echo("Uploading {} to PyPi...".format(filename))
+        subprocess.check_call([
+            "twine",
+            "upload",
+            join(DIST_DIR, filename)
+        ])
 
 
 @cli.command("decrypt")
diff --git a/release/rtool.py b/release/rtool.py
index 68ff02ab..14a0a078 100755
--- a/release/rtool.py
+++ b/release/rtool.py
@@ -1,22 +1,78 @@
 #!/usr/bin/env python3
 
 import contextlib
+import fnmatch
 import os
-import sys
 import platform
+import re
 import runpy
 import shlex
+import shutil
 import subprocess
-from os.path import join, abspath, dirname
+import tarfile
+import zipfile
+from os.path import join, abspath, dirname, exists, basename
 
-import cryptography.fernet
 import click
+import cryptography.fernet
+import pysftp
+
+# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
+# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
+if platform.system() == "Windows":
+    VENV_BIN = "Scripts"
+    PYINSTALLER_ARGS = [
+        # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
+        "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
+    ]
+else:
+    VENV_BIN = "bin"
+    PYINSTALLER_ARGS = []
 
+# ZipFile and tarfile have slightly different APIs. Fix that.
+if platform.system() == "Windows":
+    def Archive(name):
+        a = zipfile.ZipFile(name, "w")
+        a.add = a.write
+        return a
+else:
+    def Archive(name):
+        return tarfile.open(name, "w:gz")
+
+PLATFORM_TAG = {
+    "Darwin": "osx",
+    "Windows": "windows",
+    "Linux": "linux",
+}.get(platform.system(), platform.system())
 
 ROOT_DIR = abspath(join(dirname(__file__), ".."))
 RELEASE_DIR = join(ROOT_DIR, "release")
+
+BUILD_DIR = join(RELEASE_DIR, "build")
 DIST_DIR = join(RELEASE_DIR, "dist")
+
+PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
+# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
+PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
+PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
+PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
+
+VENV_DIR = join(BUILD_DIR, "venv")
+
+# Project Configuration
 VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
+BDISTS = {
+    "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
+    "pathod": ["pathoc", "pathod"]
+}
+if platform.system() == "Windows":
+    BDISTS["mitmproxy"].remove("mitmproxy")
+
+TOOLS = [
+    tool
+    for tools in sorted(BDISTS.values())
+    for tool in tools
+]
 
 
 def git(args: str) -> str:
@@ -25,8 +81,50 @@ def git(args: str) -> str:
 
 
 def get_version(dev: bool = False, build: bool = False) -> str:
-    x = runpy.run_path(VERSION_FILE)
-    return x["get_version"](dev, build, True)
+    version = runpy.run_path(VERSION_FILE)["VERSION"]
+    version = re.sub(r"\.dev.+?$", "", version)  # replace dev suffix if present.
+
+    last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2)
+    commit = commit.lstrip("g")[:7]
+    tag_dist = int(tag_dist)
+
+    if tag_dist > 0 and dev:
+        dev_tag = ".dev{tag_dist:04}".format(tag_dist=tag_dist)
+    else:
+        dev_tag = ""
+
+    if tag_dist > 0 and build:
+        # The wheel build tag (we use the commit) must start with a digit, so we include "0x"
+        build_tag = "-0x{commit}".format(commit=commit)
+    else:
+        build_tag = ""
+
+    return version + dev_tag + build_tag
+
+
+def set_version(dev: bool) -> None:
+    """
+    Update version information in mitmproxy's version.py to either include the dev version or not.
+    """
+    v = get_version(dev)
+    with open(VERSION_FILE) as f:
+        content = f.read()
+    content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(v), content)
+    with open(VERSION_FILE, "w") as f:
+        f.write(content)
+
+
+def archive_name(bdist: str) -> str:
+    if platform.system() == "Windows":
+        ext = "zip"
+    else:
+        ext = "tar.gz"
+    return "{project}-{version}-{platform}.{ext}".format(
+        project=bdist,
+        version=get_version(),
+        platform=PLATFORM_TAG,
+        ext=ext
+    )
 
 
 def wheel_name() -> str:
@@ -35,6 +133,19 @@ def wheel_name() -> str:
     )
 
 
+def installer_name() -> str:
+    ext = {
+        "Windows": "exe",
+        "Darwin": "dmg",
+        "Linux": "run"
+    }[platform.system()]
+    return "mitmproxy-{version}-{platform}-installer.{ext}".format(
+        version=get_version(),
+        platform=PLATFORM_TAG,
+        ext=ext,
+    )
+
+
 @contextlib.contextmanager
 def chdir(path: str):
     old_dir = os.getcwd()
@@ -51,6 +162,24 @@ def cli():
     pass
 
 
+@cli.command("encrypt")
+@click.argument('infile', type=click.File('rb'))
+@click.argument('outfile', type=click.File('wb'))
+@click.argument('key', envvar='RTOOL_KEY')
+def encrypt(infile, outfile, key):
+    f = cryptography.fernet.Fernet(key.encode())
+    outfile.write(f.encrypt(infile.read()))
+
+
+@cli.command("decrypt")
+@click.argument('infile', type=click.File('rb'))
+@click.argument('outfile', type=click.File('wb'))
+@click.argument('key', envvar='RTOOL_KEY')
+def decrypt(infile, outfile, key):
+    f = cryptography.fernet.Fernet(key.encode())
+    outfile.write(f.decrypt(infile.read()))
+
+
 @cli.command("contributors")
 def contributors():
     """
@@ -63,31 +192,184 @@ def contributors():
             f.write(contributors_data.encode())
 
 
-@cli.command("homebrew-pr")
-def homebrew_pr():
+@cli.command("wheel")
+def make_wheel():
+    """
+    Build a Python wheel
+    """
+    set_version(True)
+    try:
+        subprocess.check_call([
+            "tox", "-e", "wheel",
+        ], env={
+            **os.environ,
+            "VERSION": get_version(True),
+        })
+    finally:
+        set_version(False)
+
+
+@cli.command("bdist")
+def make_bdist():
     """
-    Create a new Homebrew PR
+    Build a binary distribution
     """
-    if platform.system() != "Darwin":
-        print("You need to run this on macOS to create a new Homebrew PR. Sorry.")
-        sys.exit(1)
+    if exists(PYINSTALLER_TEMP):
+        shutil.rmtree(PYINSTALLER_TEMP)
+    if exists(PYINSTALLER_DIST):
+        shutil.rmtree(PYINSTALLER_DIST)
+
+    os.makedirs(DIST_DIR, exist_ok=True)
+
+    for bdist, tools in sorted(BDISTS.items()):
+        with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
+            for tool in tools:
+                # We can't have a folder and a file with the same name.
+                if tool == "mitmproxy":
+                    tool = "mitmproxy_main"
+                # This is PyInstaller, so it messes up paths.
+                # We need to make sure that we are in the spec folder.
+                with chdir(PYINSTALLER_SPEC):
+                    print("Building %s binary..." % tool)
+                    excludes = []
+                    if tool != "mitmweb":
+                        excludes.append("mitmproxy.tools.web")
+                    if tool != "mitmproxy_main":
+                        excludes.append("mitmproxy.tools.console")
+
+                    # Overwrite mitmproxy/version.py to include commit info
+                    set_version(True)
+                    try:
+                        subprocess.check_call(
+                            [
+                                "pyinstaller",
+                                "--clean",
+                                "--workpath", PYINSTALLER_TEMP,
+                                "--distpath", PYINSTALLER_DIST,
+                                "--additional-hooks-dir", PYINSTALLER_HOOKS,
+                                "--onefile",
+                                "--console",
+                                "--icon", "icon.ico",
+                                # This is PyInstaller, so setting a
+                                # different log level obviously breaks it :-)
+                                # "--log-level", "WARN",
+                            ]
+                            + [x for e in excludes for x in ["--exclude-module", e]]
+                            + PYINSTALLER_ARGS
+                            + [tool]
+                        )
+                    finally:
+                        set_version(False)
+                    # Delete the spec file - we're good without.
+                    os.remove("{}.spec".format(tool))
+
+                # Test if it works at all O:-)
+                executable = join(PYINSTALLER_DIST, tool)
+                if platform.system() == "Windows":
+                    executable += ".exe"
+
+                # Remove _main suffix from mitmproxy executable
+                if "_main" in executable:
+                    shutil.move(
+                        executable,
+                        executable.replace("_main", "")
+                    )
+                    executable = executable.replace("_main", "")
+
+                print("> %s --version" % executable)
+                print(subprocess.check_output([executable, "--version"]).decode())
+
+                archive.add(executable, basename(executable))
+        print("Packed {}.".format(archive_name(bdist)))
+
 
-    print("Creating a new PR with Homebrew...")
+@cli.command("upload-release")
+@click.option('--username', prompt=True)
+@click.password_option(confirmation_prompt=False)
+@click.option('--repository', default="pypi")
+def upload_release(username, password, repository):
+    """
+    Upload wheels to PyPI
+    """
+    filename = wheel_name()
+    print("Uploading {} to {}...".format(filename, repository))
     subprocess.check_call([
-        "brew",
-        "bump-formula-pr",
-        "--url", "https://github.com/mitmproxy/mitmproxy/archive/v{}.tar.gz".format(get_version()),
-        "mitmproxy",
+        "twine",
+        "upload",
+        "-u", username,
+        "-p", password,
+        "-r", repository,
+        join(DIST_DIR, filename)
     ])
 
 
-@cli.command("encrypt")
-@click.argument('infile', type=click.File('rb'))
-@click.argument('outfile', type=click.File('wb'))
-@click.argument('key', envvar='RTOOL_KEY')
-def encrypt(infile, outfile, key):
-    f = cryptography.fernet.Fernet(key.encode())
-    outfile.write(f.encrypt(infile.read()))
+@cli.command("upload-snapshot")
+@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True)
+@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22)
+@click.option("--user", envvar="SNAPSHOT_USER", prompt=True)
+@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem"))
+@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True)
+@click.option("--wheel/--no-wheel", default=False)
+@click.option("--bdist/--no-bdist", default=False)
+@click.option("--installer/--no-installer", default=False)
+def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer):
+    """
+    Upload snapshot to snapshot server
+    """
+    with pysftp.Connection(host=host,
+                           port=port,
+                           username=user,
+                           private_key=private_key,
+                           private_key_pass=private_key_password) as sftp:
+        dir_name = "snapshots/v{}".format(get_version())
+        sftp.makedirs(dir_name)
+        with sftp.cd(dir_name):
+            files = []
+            if wheel:
+                files.append(wheel_name())
+            if bdist:
+                for bdist in sorted(BDISTS.keys()):
+                    files.append(archive_name(bdist))
+            if installer:
+                files.append(installer_name())
+
+            for f in files:
+                local_path = join(DIST_DIR, f)
+                remote_filename = re.sub(
+                    r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()),
+                    get_version(True, True),
+                    f
+                )
+                symlink_path = "../{}".format(f.replace(get_version(), "latest"))
+
+                # Upload new version
+                print("Uploading {} as {}...".format(f, remote_filename))
+                with click.progressbar(length=os.stat(local_path).st_size) as bar:
+                    # We hide the file during upload
+                    sftp.put(
+                        local_path,
+                        "." + remote_filename,
+                        callback=lambda done, total: bar.update(done - bar.pos)
+                    )
+
+                # Delete old versions
+                old_version = f.replace(get_version(), "*")
+                for f_old in sftp.listdir():
+                    if fnmatch.fnmatch(f_old, old_version):
+                        print("Removing {}...".format(f_old))
+                        sftp.remove(f_old)
+
+                # Show new version
+                sftp.rename("." + remote_filename, remote_filename)
+
+                # update symlink for the latest release
+                if sftp.lexists(symlink_path):
+                    print("Removing {}...".format(symlink_path))
+                    sftp.remove(symlink_path)
+                if f != wheel_name():
+                    # "latest" isn't a proper wheel version, so this could not be installed.
+                    # https://github.com/mitmproxy/mitmproxy/issues/1065
+                    sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path)
 
 
 if __name__ == "__main__":