From fe03a656a99447d410421d5fd249882941eaf1c4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 22 Jul 2015 02:43:45 +0200 Subject: make build script amazing --- README.mkd | 2 +- release/build.py | 168 +++++++++++++++++++++++++++++-------------- release/release-checklist | 42 ----------- release/release-checklist.md | 55 ++++++++++++++ setup.py | 1 + 5 files changed, 170 insertions(+), 98 deletions(-) delete mode 100644 release/release-checklist create mode 100644 release/release-checklist.md diff --git a/README.mkd b/README.mkd index 1aac71c4..d686e933 100644 --- a/README.mkd +++ b/README.mkd @@ -57,7 +57,7 @@ $ git clone https://github.com/mitmproxy/mitmproxy.git $ git clone https://github.com/mitmproxy/netlib.git $ git clone https://github.com/mitmproxy/pathod.git $ cd mitmproxy -$ ./dev +$ source ./dev ``` The *dev* script will create a virtualenv environment in a directory called diff --git a/release/build.py b/release/build.py index 3bb60ca4..0c81986b 100755 --- a/release/build.py +++ b/release/build.py @@ -1,15 +1,13 @@ #!/usr/bin/env python from os.path import dirname, realpath, join, exists, normpath -from os import chdir, mkdir, getcwd +from os import chdir import os import shutil import subprocess -import tempfile import glob import re from shlex import split -from contextlib import contextmanager import click # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes @@ -25,9 +23,9 @@ dist_dir = join(mitmproxy_dir, "dist") test_venv_dir = join(root_dir, "venv.mitmproxy-release") projects = ("netlib", "pathod", "mitmproxy") -tools = ["mitmweb", "mitmdump", "pathod", "pathoc"] -if os.name != "nt": - tools.append("mitmproxy") +tools = ["mitmproxy", "mitmdump", "mitmweb", "pathod", "pathoc"] +if os.name == "nt": + tools.remove("mitmproxy") version_files = (join(root_dir, x) for x in ( "mitmproxy/libmproxy/version.py", @@ -35,6 +33,7 @@ version_files = (join(root_dir, x) for x in ( "netlib/netlib/version.py" )) + @click.group(chain=True) def cli(): """ @@ -44,15 +43,21 @@ def cli(): @cli.command("contributors") -def update_contributors(): +def contributors(): + """ + Update CONTRIBUTORS.md + """ print("Updating CONTRIBUTORS.md...") - contributors = subprocess.check_output(split("git shortlog -n -s")) + contributors_data = subprocess.check_output(split("git shortlog -n -s")) with open(join(mitmproxy_dir, "CONTRIBUTORS"), "w") as f: - f.write(contributors) + f.write(contributors_data) @cli.command("docs") -def render_docs(): +def docs(): + """ + Render the docs + """ print("Rendering the docs...") subprocess.check_call([ "cshape", @@ -61,11 +66,65 @@ def render_docs(): ]) +@cli.command("set-version") +@click.argument('version') +def set_version(version): + """ + Update version information + """ + print("Update versions...") + version = ", ".join(version.split(".")) + for version_file in version_files: + print("Update %s..." % version_file) + with open(version_file, "rb") as f: + content = f.read() + new_content = re.sub(r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content) + with open(version_file, "wb") as f: + f.write(new_content) + + +@cli.command("git") +@click.argument('args', nargs=-1, required=True) +def git(args): + """ + Run a git command on every project + """ + args = ["git"] + list(args) + for project in projects: + print("%s> %s..." % (project, " ".join(args))) + subprocess.check_call( + args, + cwd=join(root_dir, project) + ) + + +@cli.command("sdist") +def sdist(): + """ + Build a source distribution + """ + # Make sure that the regular python installation is not on the python path! + os.environ["PYTHONPATH"] = "" + + print("Building release...") + if exists(dist_dir): + shutil.rmtree(dist_dir) + for project in projects: + print("Creating %s source distribution..." % project) + subprocess.check_call( + ["python", "./setup.py", "-q", "sdist", "--dist-dir", dist_dir, "--formats=gztar"], + cwd=join(root_dir, project) + ) + + @cli.command("test") @click.pass_context def test(ctx): + """ + Test the source distribution + """ if not exists(dist_dir): - ctx.invoke(release) + ctx.invoke(sdist) # Make sure that the regular python installation is not on the python path! os.environ["PYTHONPATH"] = "" @@ -89,50 +148,7 @@ def test(ctx): print(subprocess.check_output([tool, "--version"])) print("Virtualenv available for further testing:") - print(normpath(join(test_venv_dir, venv_bin, "activate"))) - - -@cli.command("release") -def release(): - os.environ["PYTHONPATH"] = "" - - print("Building release...") - - if exists(dist_dir): - shutil.rmtree(dist_dir) - for project in projects: - print("Creating %s source distribution..." % project) - subprocess.check_call( - ["python", "./setup.py", "-q", "sdist", "--dist-dir", dist_dir, "--formats=gztar"], - cwd=join(root_dir, project) - ) - - -@cli.command("set-version") -@click.argument('version') -def set_version(version): - version = ", ".join(version.split(".")) - for version_file in version_files: - with open(version_file, "rb") as f: - content = f.read() - new_content = re.sub(r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content) - with open(version_file, "wb") as f: - f.write(new_content) - - -@cli.command("add-tag") -@click.argument('version') -def git_tag(version): - for project in projects: - print("Tagging %s..." % project) - subprocess.check_call( - ["git", "tag", version], - cwd=join(root_dir, project) - ) - subprocess.check_call( - ["git", "push", "--tags"], - cwd=join(root_dir, project) - ) + print("source %s" % normpath(join(test_venv_dir, venv_bin, "activate"))) @cli.command("upload") @@ -140,6 +156,9 @@ def git_tag(version): @click.password_option(confirmation_prompt=False) @click.option('--repository', default="pypi") def upload_release(username, password, repository): + """ + Upload source distributions to PyPI + """ print("Uploading distributions...") subprocess.check_call([ "twine", @@ -151,5 +170,44 @@ def upload_release(username, password, repository): ]) +""" + +TODO: Fully automate build process. +This skeleton is missing OSX builds and updating mitmproxy.org. + + +@cli.command("wizard") +@click.option('--version', prompt=True) +@click.option('--username', prompt=True) +@click.password_option(confirmation_prompt=False) +@click.option('--repository', default="pypi") +@click.option('--test/--no-test', default=True) +@click.pass_context +def wizard(ctx, version, username, password, repository, test): + "" + Interactive Release Wizard + "" + + for project in projects: + if subprocess.check_output(["git", "status", "--porcelain"], cwd=join(root_dir, project)): + raise RuntimeError("%s repository is not clean." % project) + + if test: + ctx.invoke(sdist) + ctx.invoke(test) + click.confirm("Please test the release now. Is it ok?", abort=True) + + ctx.invoke(set_version, version=version) + ctx.invoke(docs) + ctx.invoke(contributors) + + ctx.invoke(git, args=["commit", "-a", "-m", "bump version"]) + ctx.invoke(git, args=["tag", "v" + version]) + ctx.invoke(git, args=["push", "--tags"]) + ctx.invoke(sdist) + ctx.invoke(upload_release, username=username, password=password, repository=repository) + click.echo("All done!") +""" + if __name__ == "__main__": cli() diff --git a/release/release-checklist b/release/release-checklist deleted file mode 100644 index b12a4b82..00000000 --- a/release/release-checklist +++ /dev/null @@ -1,42 +0,0 @@ - -- Check the version number: - - mitmproxy/libmproxy/version.py - netlib/netlib/version.py - pathod/libpathod/version.py - -- Ensure that the website style assets have been compiled for production, and -synced to the docs. - -- Render the docs, update CONTRIBUTORS file: - ./release/build.py docs contributors - -- Run the test release, make sure the output is sensible - ./release/build.py release - -- Build the OSX binaries - - Follow instructions in osx-binaries - - Move to download dir: - mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download - -- Move all source distributions from mitmproxy's dist folder to the server: - mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download - -- Tag with the version number, and do: - git push --tags - Tag and push v0.12 for all projects: - ./release/build.py tag v0.12 - -- Upload to pypi: - - ./release/build.py upload - - Be careful: Pypi requires you to bump the version number if you want to do any further changes. - -- Now bump the version number to be ready for the next cycle: - - TODO: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? - - mitmproxy/libmproxy/version.py - netlib/netlib/version.py - pathod/libpathod/version.py \ No newline at end of file diff --git a/release/release-checklist.md b/release/release-checklist.md new file mode 100644 index 00000000..e6d9ae1f --- /dev/null +++ b/release/release-checklist.md @@ -0,0 +1,55 @@ +# Release Checklist + +## Test + + - Create the source distributions, make sure the output is sensible: + `./release/build.py release` + All source distributions can be found in `./dist`. + + - Test the source distributions: + `./release/build.py test` + This creates a new virtualenv in `../venv.mitmproxy-release` and installs the distributions from `./dist` into it. + +## Release + + - Verify that repositories are in a clean state: + `./release/build.py git status` + + - Update the version number in `version.py` for all projects: + `./release/build.py set-version 0.13` + + - Ensure that the website style assets have been compiled for production, and synced to the docs. + + - Render the docs, update CONTRIBUTORS file: + `./release/build.py docs contributors` + + - Make version bump commit for all projects, tag and push it: + `./release/build.py git commit -am "bump version"` + `./release/build.py git tag v0.13` + `./release/build.py git push --tags` + + - Recreate the source distributions with updated version information: + `./release/build.py sdist` + + - Build the OSX binaries + - Follow instructions in osx-binaries + - Move to download dir: + `mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download` + + - Move all source distributions from `./dist` to the server: + `mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download` + + - Upload distributions in `./dist` to PyPI: + `./release/build.py upload` + You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`. + ([more info](https://tom-christie.github.io/articles/pypi/)) + + - Now bump the version number to be ready for the next cycle: + + **TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? + We should probably just leave it as-is and only bump once we actually do the next release. + + Also, we need a release policy. I propose the following: + - By default, every release is a new minor (`0.x`) release and it will be pushed for all three projects. + - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release for a single project. + This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)` \ No newline at end of file diff --git a/setup.py b/setup.py index 01dc2ae7..da080bc1 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ dev_deps = { "nose-cov>=1.6", "coveralls>=0.4.1", "click>=4.1", + "twine>=1.5.0", "pathod>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION), "countershape" } -- cgit v1.2.3