make build script amazing

5 files changed, 170 insertions, 98 deletions

diff --git a/README.mkd b/README.mkd
index 1aac71c4..d686e933 100644
--- a/README.mkd
+++ b/README.mkd
@@ -57,7 +57,7 @@ $ git clone https://github.com/mitmproxy/mitmproxy.git
 $ git clone https://github.com/mitmproxy/netlib.git
 $ git clone https://github.com/mitmproxy/pathod.git
 $ cd mitmproxy
-$ ./dev
+$ source ./dev
 ```
 
 The *dev* script will create a virtualenv environment in a directory called
diff --git a/release/build.py b/release/build.py
index 3bb60ca4..0c81986b 100755
--- a/release/build.py
+++ b/release/build.py
@@ -1,15 +1,13 @@
 #!/usr/bin/env python
 
 from os.path import dirname, realpath, join, exists, normpath
-from os import chdir, mkdir, getcwd
+from os import chdir
 import os
 import shutil
 import subprocess
-import tempfile
 import glob
 import re
 from shlex import split
-from contextlib import contextmanager
 import click
 
 # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
@@ -25,9 +23,9 @@ dist_dir = join(mitmproxy_dir, "dist")
 test_venv_dir = join(root_dir, "venv.mitmproxy-release")
 
 projects = ("netlib", "pathod", "mitmproxy")
-tools = ["mitmweb", "mitmdump", "pathod", "pathoc"]
-if os.name != "nt":
-    tools.append("mitmproxy")
+tools = ["mitmproxy", "mitmdump", "mitmweb", "pathod", "pathoc"]
+if os.name == "nt":
+    tools.remove("mitmproxy")
 
 version_files = (join(root_dir, x) for x in (
     "mitmproxy/libmproxy/version.py",
@@ -35,6 +33,7 @@ version_files = (join(root_dir, x) for x in (
     "netlib/netlib/version.py"
 ))
 
+
 @click.group(chain=True)
 def cli():
     """
@@ -44,15 +43,21 @@ def cli():
 
 
 @cli.command("contributors")
-def update_contributors():
+def contributors():
+    """
+    Update CONTRIBUTORS.md
+    """
     print("Updating CONTRIBUTORS.md...")
-    contributors = subprocess.check_output(split("git shortlog -n -s"))
+    contributors_data = subprocess.check_output(split("git shortlog -n -s"))
     with open(join(mitmproxy_dir, "CONTRIBUTORS"), "w") as f:
-        f.write(contributors)
+        f.write(contributors_data)
 
 
 @cli.command("docs")
-def render_docs():
+def docs():
+    """
+    Render the docs
+    """
     print("Rendering the docs...")
     subprocess.check_call([
         "cshape",
@@ -61,11 +66,65 @@ def render_docs():
     ])
 
 
+@cli.command("set-version")
+@click.argument('version')
+def set_version(version):
+    """
+    Update version information
+    """
+    print("Update versions...")
+    version = ", ".join(version.split("."))
+    for version_file in version_files:
+        print("Update %s..." % version_file)
+        with open(version_file, "rb") as f:
+            content = f.read()
+        new_content = re.sub(r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content)
+        with open(version_file, "wb") as f:
+            f.write(new_content)
+
+
+@cli.command("git")
+@click.argument('args', nargs=-1, required=True)
+def git(args):
+    """
+    Run a git command on every project
+    """
+    args = ["git"] + list(args)
+    for project in projects:
+        print("%s> %s..." % (project, " ".join(args)))
+        subprocess.check_call(
+            args,
+            cwd=join(root_dir, project)
+        )
+
+
+@cli.command("sdist")
+def sdist():
+    """
+    Build a source distribution
+    """
+    # Make sure that the regular python installation is not on the python path!
+    os.environ["PYTHONPATH"] = ""
+
+    print("Building release...")
+    if exists(dist_dir):
+        shutil.rmtree(dist_dir)
+    for project in projects:
+        print("Creating %s source distribution..." % project)
+        subprocess.check_call(
+            ["python", "./setup.py", "-q", "sdist", "--dist-dir", dist_dir, "--formats=gztar"],
+            cwd=join(root_dir, project)
+        )
+
+
 @cli.command("test")
 @click.pass_context
 def test(ctx):
+    """
+    Test the source distribution
+    """
     if not exists(dist_dir):
-        ctx.invoke(release)
+        ctx.invoke(sdist)
 
     # Make sure that the regular python installation is not on the python path!
     os.environ["PYTHONPATH"] = ""
@@ -89,50 +148,7 @@ def test(ctx):
         print(subprocess.check_output([tool, "--version"]))
 
     print("Virtualenv available for further testing:")
-    print(normpath(join(test_venv_dir, venv_bin, "activate")))
-
-
-@cli.command("release")
-def release():
-    os.environ["PYTHONPATH"] = ""
-
-    print("Building release...")
-
-    if exists(dist_dir):
-        shutil.rmtree(dist_dir)
-    for project in projects:
-        print("Creating %s source distribution..." % project)
-        subprocess.check_call(
-            ["python", "./setup.py", "-q", "sdist", "--dist-dir", dist_dir, "--formats=gztar"],
-            cwd=join(root_dir, project)
-        )
-
-
-@cli.command("set-version")
-@click.argument('version')
-def set_version(version):
-    version = ", ".join(version.split("."))
-    for version_file in version_files:
-        with open(version_file, "rb") as f:
-            content = f.read()
-        new_content = re.sub(r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content)
-        with open(version_file, "wb") as f:
-            f.write(new_content)
-
-
-@cli.command("add-tag")
-@click.argument('version')
-def git_tag(version):
-    for project in projects:
-        print("Tagging %s..." % project)
-        subprocess.check_call(
-            ["git", "tag", version],
-            cwd=join(root_dir, project)
-        )
-        subprocess.check_call(
-            ["git", "push", "--tags"],
-            cwd=join(root_dir, project)
-        )
+    print("source %s" % normpath(join(test_venv_dir, venv_bin, "activate")))
 
 
 @cli.command("upload")
@@ -140,6 +156,9 @@ def git_tag(version):
 @click.password_option(confirmation_prompt=False)
 @click.option('--repository', default="pypi")
 def upload_release(username, password, repository):
+    """
+    Upload source distributions to PyPI
+    """
     print("Uploading distributions...")
     subprocess.check_call([
         "twine",
@@ -151,5 +170,44 @@ def upload_release(username, password, repository):
     ])
 
 
+"""
+
+TODO: Fully automate build process.
+This skeleton is missing OSX builds and updating mitmproxy.org.
+
+
+@cli.command("wizard")
+@click.option('--version', prompt=True)
+@click.option('--username', prompt=True)
+@click.password_option(confirmation_prompt=False)
+@click.option('--repository', default="pypi")
+@click.option('--test/--no-test', default=True)
+@click.pass_context
+def wizard(ctx, version, username, password, repository, test):
+    ""
+    Interactive Release Wizard
+    ""
+
+    for project in projects:
+        if subprocess.check_output(["git", "status", "--porcelain"], cwd=join(root_dir, project)):
+            raise RuntimeError("%s repository is not clean." % project)
+
+    if test:
+        ctx.invoke(sdist)
+        ctx.invoke(test)
+        click.confirm("Please test the release now. Is it ok?", abort=True)
+
+    ctx.invoke(set_version, version=version)
+    ctx.invoke(docs)
+    ctx.invoke(contributors)
+
+    ctx.invoke(git, args=["commit", "-a", "-m", "bump version"])
+    ctx.invoke(git, args=["tag", "v" + version])
+    ctx.invoke(git, args=["push", "--tags"])
+    ctx.invoke(sdist)
+    ctx.invoke(upload_release, username=username, password=password, repository=repository)
+    click.echo("All done!")
+"""
+
 if __name__ == "__main__":
     cli()
diff --git a/release/release-checklist b/release/release-checklist
deleted file mode 100644
index b12a4b82..00000000
--- a/release/release-checklist
+++ /dev/null
@@ -1,42 +0,0 @@
-
-- Check the version number:
-
-    mitmproxy/libmproxy/version.py
-    netlib/netlib/version.py
-    pathod/libpathod/version.py
-
-- Ensure that the website style assets have been compiled for production, and
-synced to the docs.
-
-- Render the docs, update CONTRIBUTORS file:
-    ./release/build.py docs contributors
-
-- Run the test release, make sure the output is sensible
-    ./release/build.py release
-
-- Build the OSX binaries
-    - Follow instructions in osx-binaries
-    - Move to download dir:
-        mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download
-
-- Move all source distributions from mitmproxy's dist folder to the server:
-    mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download
-
-- Tag with the version number, and do:
-    git push --tags
-  Tag and push v0.12 for all projects:
-    ./release/build.py tag v0.12
-
-- Upload to pypi:
-
-    ./release/build.py upload
-
-  Be careful: Pypi requires you to bump the version number if you want to do any further changes.
-
-- Now bump the version number to be ready for the next cycle:
-
-    TODO: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now?
-
-    mitmproxy/libmproxy/version.py
-    netlib/netlib/version.py
-    pathod/libpathod/version.py
\ No newline at end of file
diff --git a/release/release-checklist.md b/release/release-checklist.md
new file mode 100644
index 00000000..e6d9ae1f
--- /dev/null
+++ b/release/release-checklist.md
@@ -0,0 +1,55 @@
+# Release Checklist
+
+## Test
+
+  - Create the source distributions, make sure the output is sensible:  
+    `./release/build.py release`  
+    All source distributions can be found in `./dist`.
+
+  - Test the source distributions:  
+    `./release/build.py test`  
+    This creates a new virtualenv in `../venv.mitmproxy-release` and installs the distributions from `./dist` into it.
+
+## Release
+
+  - Verify that repositories are in a clean state:
+    `./release/build.py git status`
+
+  - Update the version number in `version.py` for all projects:  
+    `./release/build.py set-version 0.13`
+
+  - Ensure that the website style assets have been compiled for production, and synced to the docs.
+
+  - Render the docs, update CONTRIBUTORS file:  
+    `./release/build.py docs contributors`
+  
+  - Make version bump commit for all projects, tag and push it:
+    `./release/build.py git commit -am "bump version"`
+    `./release/build.py git tag v0.13`  
+    `./release/build.py git push --tags`
+
+  - Recreate the source distributions with updated version information:  
+    `./release/build.py sdist`
+
+  - Build the OSX binaries
+    - Follow instructions in osx-binaries
+    - Move to download dir:  
+      `mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download`
+
+  - Move all source distributions from `./dist` to the server:  
+    `mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download`
+
+  - Upload distributions in `./dist` to PyPI:  
+    `./release/build.py upload`  
+    You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`.
+    ([more info](https://tom-christie.github.io/articles/pypi/))
+
+  - Now bump the version number to be ready for the next cycle:
+
+    **TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? 
+    We should probably just leave it as-is and only bump once we actually do the next release.
+    
+    Also, we need a release policy. I propose the following:
+      - By default, every release is a new minor (`0.x`) release and it will be pushed for all three projects.
+      - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release for a single project.
+        This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)`
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 01dc2ae7..da080bc1 100644
--- a/setup.py
+++ b/setup.py
@@ -39,6 +39,7 @@ dev_deps = {
     "nose-cov>=1.6",
     "coveralls>=0.4.1",
     "click>=4.1",
+    "twine>=1.5.0",
     "pathod>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION),
     "countershape"
 }