From b1908e6639c9ced1ef8ce2e6d8d51a073e1ee89c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Aug 2015 17:25:38 +1200 Subject: Updates build -> rtool to avoid clashing with standard pyinstaller dirs checklist updates script adaptations --- .gitignore | 1 + build | 285 --------------------------------------------------- osx-binaries | 33 +++--- release-checklist.md | 57 ++++++----- requirements.txt | 1 + rtool | 284 ++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 334 insertions(+), 327 deletions(-) delete mode 100755 build create mode 100755 rtool diff --git a/.gitignore b/.gitignore index 356e5994..dfd65745 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ MANIFEST /venv /release +/build diff --git a/build b/build deleted file mode 100755 index 49e3213f..00000000 --- a/build +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/env python -from __future__ import ( - absolute_import, print_function, division, unicode_literals -) -from contextlib import contextmanager -from os.path import dirname, realpath, join, exists, normpath -import os -import shutil -import subprocess -import glob -import re -import shlex -import click - -# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes -# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ -if os.name == "nt": - VENV_BIN = "Scripts" -else: - VENV_BIN = "bin" - -RELEASE_DIR = join(dirname(realpath(__file__))) -ROOT_DIR = join(RELEASE_DIR, "..") -MITMPROXY_DIR = join(ROOT_DIR, "mitmproxy") -DIST_DIR = join(MITMPROXY_DIR, "dist") -TEST_VENV_DIR = join(RELEASE_DIR, "venv") - -PROJECTS = ("netlib", "pathod", "mitmproxy") -TOOLS = { - "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathod", "pathoc"], - "netlib": [] -} -if os.name == "nt": - TOOLS["mitmproxy"].remove("mitmproxy") -VERSION_FILES = { - "mitmproxy": normpath(join(ROOT_DIR, "mitmproxy/libmproxy/version.py")), - "pathod": normpath(join(ROOT_DIR, "pathod/libpathod/version.py")), - "netlib": normpath(join(ROOT_DIR, "netlib/netlib/version.py")), -} - - -@contextmanager -def empty_pythonpath(): - """ - Make sure that the regular python installation is not on the python path, - which would give us access to modules installed outside of our virtualenv. - """ - pythonpath = os.environ["PYTHONPATH"] - os.environ["PYTHONPATH"] = "" - yield - os.environ["PYTHONPATH"] = pythonpath - - -@contextmanager -def chdir(path): - old_dir = os.getcwd() - os.chdir(path) - yield - os.chdir(old_dir) - - -@click.group(chain=True) -def cli(): - """ - mitmproxy build tool - """ - pass - - -@cli.command("contributors") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS -) -def contributors(projects): - """ - Update CONTRIBUTORS.md - """ - for project in PROJECTS: - if project not in projects: - continue - with chdir(os.path.join(ROOT_DIR, project)): - print("Updating %s/CONTRIBUTORS..."%project) - contributors_data = subprocess.check_output( - shlex.split("git shortlog -n -s") - ) - with open("CONTRIBUTORS", "w+") as f: - f.write(contributors_data) - - -@cli.command("docs") -def docs(): - """ - Render the docs - """ - print("Rendering the docs...") - subprocess.check_call([ - "cshape", - join(MITMPROXY_DIR, "doc-src"), - join(MITMPROXY_DIR, "doc") - ]) - - -@cli.command("set-version") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS -) -@click.argument('version') -def set_version(projects, version): - """ - Update version information - """ - print("Update versions...") - version = ", ".join(version.split(".")) - for project, version_file in VERSION_FILES.items(): - if project not in projects: - continue - print("Update %s..." % version_file) - with open(version_file, "rb") as f: - content = f.read() - new_content = re.sub( - r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, - content - ) - with open(version_file, "wb") as f: - f.write(new_content) - - -@cli.command("git") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS -) -@click.argument('args', nargs=-1, required=True) -def git(projects, args): - """ - Run a git command on every project - """ - args = ["git"] + list(args) - for project in projects: - print("%s> %s..." % (project, " ".join(args))) - subprocess.check_call( - args, - cwd=join(ROOT_DIR, project) - ) - - -@cli.command("sdist") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS -) -def sdist(projects): - """ - Build a source distribution - """ - with empty_pythonpath(): - print("Building release...") - if exists(DIST_DIR): - shutil.rmtree(DIST_DIR) - for project in projects: - print("Creating %s source distribution..." % project) - subprocess.check_call( - [ - "python", "./setup.py", - "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar" - ], - cwd=join(ROOT_DIR, project) - ) - - -@cli.command("test") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS -) -@click.pass_context -def test(ctx, projects): - """ - Test the source distribution - """ - if not exists(DIST_DIR): - ctx.invoke(sdist) - - with empty_pythonpath(): - print("Creating virtualenv for test install...") - if exists(TEST_VENV_DIR): - shutil.rmtree(TEST_VENV_DIR) - subprocess.check_call(["virtualenv", "-q", TEST_VENV_DIR]) - - pip = join(TEST_VENV_DIR, VENV_BIN, "pip") - with chdir(DIST_DIR): - for project in projects: - print("Installing %s..." % project) - dist = join(ROOT_DIR, project) - subprocess.check_call([pip, "install", "-q", dist]) - - print("Running binaries...") - for project in projects: - for tool in TOOLS[project]: - tool = join(TEST_VENV_DIR, VENV_BIN, tool) - print(tool) - print(subprocess.check_output([tool, "--version"])) - - print("Virtualenv available for further testing:") - print( - "source %s" % normpath( - join(TEST_VENV_DIR, VENV_BIN, "activate") - ) - ) - - -@cli.command("upload") -@click.option('--username', prompt=True) -@click.password_option(confirmation_prompt=False) -@click.option('--repository', default="pypi") -def upload_release(username, password, repository): - """ - Upload source distributions to PyPI - """ - print("Uploading distributions...") - subprocess.check_call([ - "twine", - "upload", - "-u", username, - "-p", password, - "-r", repository, - "%s/*" % DIST_DIR - ]) - - -# TODO: Fully automate build process. -# This wizard is missing OSX builds and updating mitmproxy.org. -@cli.command("wizard") -@click.option('--version', prompt=True) -@click.option('--username', prompt="PyPI Username") -@click.password_option(confirmation_prompt=False, prompt="PyPI Password") -@click.option('--repository', default="pypi") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS -) -@click.pass_context -def wizard(ctx, version, username, password, repository, projects): - """ - Interactive Release Wizard - """ - for project in projects: - if subprocess.check_output( - ["git", "status", "--porcelain"], cwd=join(ROOT_DIR, project) - ): - raise RuntimeError("%s repository is not clean." % project) - - # Build test release - ctx.invoke(sdist, projects=projects) - ctx.invoke(test, projects=projects) - click.confirm("Please test the release now. Is it ok?", abort=True) - - # bump version, update docs and contributors - ctx.invoke(set_version, version=version, projects=projects) - ctx.invoke(docs) - ctx.invoke(contributors) - - # version bump commit + tag - ctx.invoke( - git, args=["commit", "-a", "-m", "bump version"], projects=projects - ) - ctx.invoke(git, args=["tag", "v" + version], projects=projects) - ctx.invoke(git, args=["push"], projects=projects) - ctx.invoke(git, args=["push", "--tags"], projects=projects) - - # Re-invoke sdist with bumped version - ctx.invoke(sdist, projects=projects) - click.confirm("All good, can upload to PyPI?", abort=True) - ctx.invoke( - upload_release, - username=username, password=password, repository=repository - ) - click.echo("All done!") - - -if __name__ == "__main__": - cli() diff --git a/osx-binaries b/osx-binaries index 3cfe0913..657dbae9 100755 --- a/osx-binaries +++ b/osx-binaries @@ -7,10 +7,11 @@ # https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Setuptools-Entry-Point # Once this is fixed, we can ditch the redundant command scripts. -VENV=../venv.mitmproxy +VENV=./venv PYINST_CMD="$VENV/bin/pyinstaller -F --clean" TMPDIR=./tmp CACHE="~/Library/Application Support/pyinstaller" +MITMPROXY="../mitmproxy" if [ ! -d $VENV ] @@ -44,23 +45,23 @@ rm -f dist/* rm -rf $TMPDIR rm -rf $CACHE -$PYINST_CMD ./release/mitmdump.spec -echo "Running mitmdump..." -./dist/mitmdump --version || exit 1 +# $PYINST_CMD $MITMPROXY/release/mitmdump.spec +# echo "Running mitmdump..." +# ./dist/mitmdump --version || exit 1 -$PYINST_CMD ./release/mitmproxy.spec +$PYINST_CMD $MITMPROXY/release/mitmproxy.spec echo "Running mitmproxy..." ./dist/mitmproxy --version || exit 1 -$PYINST_CMD ./release/mitmweb.spec -echo "Running mitmweb..." -./dist/mitmweb --version || exit 1 +# $PYINST_CMD $MITMPROXY/release/mitmweb.spec +# echo "Running mitmweb..." +# ./dist/mitmweb --version || exit 1 -DST=osx-mitmproxy-`./dist/mitmdump --shortversion 2>&1` -mkdir -p $TMPDIR/$DST -cp ./dist/mitmproxy $TMPDIR/$DST -cp ./dist/mitmdump $TMPDIR/$DST -cshape ./doc-src $TMPDIR/$DST/doc - -cd $TMPDIR -tar -czvf $DST.tar.gz $DST +# DST=osx-mitmproxy-`./dist/mitmdump --shortversion 2>&1` +# mkdir -p $TMPDIR/$DST +# cp ./dist/mitmproxy $TMPDIR/$DST +# cp ./dist/mitmdump $TMPDIR/$DST +# cshape ./doc-src $TMPDIR/$DST/doc +# +# cd $TMPDIR +# tar -czvf $DST.tar.gz $DST diff --git a/release-checklist.md b/release-checklist.md index e6d9ae1f..59fa7623 100644 --- a/release-checklist.md +++ b/release-checklist.md @@ -1,35 +1,37 @@ # Release Checklist -## Test - - - Create the source distributions, make sure the output is sensible: - `./release/build.py release` - All source distributions can be found in `./dist`. +## Check out release versions - - Test the source distributions: - `./release/build.py test` - This creates a new virtualenv in `../venv.mitmproxy-release` and installs the distributions from `./dist` into it. - -## Release + - Check out the versions of pathod, netlib and mitmproxy due to be released - Verify that repositories are in a clean state: - `./release/build.py git status` - - - Update the version number in `version.py` for all projects: - `./release/build.py set-version 0.13` + `./build git status` - Ensure that the website style assets have been compiled for production, and synced to the docs. - Render the docs, update CONTRIBUTORS file: - `./release/build.py docs contributors` - - - Make version bump commit for all projects, tag and push it: - `./release/build.py git commit -am "bump version"` - `./release/build.py git tag v0.13` - `./release/build.py git push --tags` + `./build docs contributors` + + +## Test - - Recreate the source distributions with updated version information: - `./release/build.py sdist` + - Test the source distributions: + + `./build test` + + This does the following: + - creates a venv in release/venv + - creates source distributions in release/release + - installs the source distributions in the venv + - and runs all installed tools + + +## Release + + - Make a release commit for all projects, tag and push it: + `./build git commit -am "Release v0.13"` + `./build git tag v0.13` + `./build git push --tags` - Build the OSX binaries - Follow instructions in osx-binaries @@ -40,16 +42,19 @@ `mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download` - Upload distributions in `./dist` to PyPI: - `./release/build.py upload` + `./build upload` You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`. ([more info](https://tom-christie.github.io/articles/pypi/)) - Now bump the version number to be ready for the next cycle: - **TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? + **TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? We should probably just leave it as-is and only bump once we actually do the next release. - + + - Bump the version number in `version.py` for all projects: + `./build set-version 0.13` + Also, we need a release policy. I propose the following: - By default, every release is a new minor (`0.x`) release and it will be pushed for all three projects. - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release for a single project. - This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)` \ No newline at end of file + This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)` diff --git a/requirements.txt b/requirements.txt index de1b90b2..0bb2e378 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ click>=4.1 twine>=1.5.0 +pyinstaller diff --git a/rtool b/rtool new file mode 100755 index 00000000..356ea963 --- /dev/null +++ b/rtool @@ -0,0 +1,284 @@ +#!/usr/bin/env python +from __future__ import ( + absolute_import, print_function, division, unicode_literals +) +from os.path import join +import contextlib +import os.path +import os +import shutil +import subprocess +import glob +import re +import shlex +import click + +# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes +# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ +if os.name == "nt": + VENV_BIN = "Scripts" +else: + VENV_BIN = "bin" + +RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) +DIST_DIR = join(RELEASE_DIR, "release") +ROOT_DIR = join(RELEASE_DIR, "..") +MITMPROXY_DIR = join(ROOT_DIR, "mitmproxy") +TEST_VENV_DIR = join(RELEASE_DIR, "venv") + +PROJECTS = ("netlib", "pathod", "mitmproxy") +TOOLS = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathod", "pathoc"], + "netlib": [] +} +if os.name == "nt": + TOOLS["mitmproxy"].remove("mitmproxy") +VERSION_FILES = { + "mitmproxy": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"), + "pathod": join(ROOT_DIR, "pathod/libpathod/version.py"), + "netlib": join(ROOT_DIR, "netlib/netlib/version.py"), +} + + +@contextlib.contextmanager +def empty_pythonpath(): + """ + Make sure that the regular python installation is not on the python path, + which would give us access to modules installed outside of our virtualenv. + """ + pythonpath = os.environ["PYTHONPATH"] + os.environ["PYTHONPATH"] = "" + yield + os.environ["PYTHONPATH"] = pythonpath + + +@contextlib.contextmanager +def chdir(path): + old_dir = os.getcwd() + os.chdir(path) + yield + os.chdir(old_dir) + + +@click.group(chain=True) +def cli(): + """ + mitmproxy build tool + """ + pass + + +@cli.command("contributors") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS +) +def contributors(projects): + """ + Update CONTRIBUTORS.md + """ + for project in PROJECTS: + if project not in projects: + continue + with chdir(os.path.join(ROOT_DIR, project)): + print("Updating %s/CONTRIBUTORS..."%project) + contributors_data = subprocess.check_output( + shlex.split("git shortlog -n -s") + ) + with open("CONTRIBUTORS", "w+") as f: + f.write(contributors_data) + + +@cli.command("docs") +def docs(): + """ + Render the docs + """ + print("Rendering the docs...") + subprocess.check_call([ + "cshape", + join(MITMPROXY_DIR, "doc-src"), + join(MITMPROXY_DIR, "doc") + ]) + + +@cli.command("set-version") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS +) +@click.argument('version') +def set_version(projects, version): + """ + Update version information + """ + print("Update versions...") + version = ", ".join(version.split(".")) + for project, version_file in VERSION_FILES.items(): + if project not in projects: + continue + print("Update %s..." % version_file) + with open(version_file, "rb") as f: + content = f.read() + new_content = re.sub( + r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, + content + ) + with open(version_file, "wb") as f: + f.write(new_content) + + +@cli.command("git") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS +) +@click.argument('args', nargs=-1, required=True) +def git(projects, args): + """ + Run a git command on every project + """ + args = ["git"] + list(args) + for project in projects: + print("%s> %s..." % (project, " ".join(args))) + subprocess.check_call( + args, + cwd=join(ROOT_DIR, project) + ) + + +@cli.command("sdist") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS +) +def sdist(projects): + """ + Build a source distribution + """ + with empty_pythonpath(): + print("Building release...") + if os.path.exists(DIST_DIR): + shutil.rmtree(DIST_DIR) + for project in projects: + print("Creating %s source distribution..." % project) + subprocess.check_call( + [ + "python", "./setup.py", + "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar" + ], + cwd=join(ROOT_DIR, project) + ) + + +@cli.command("mkvenv") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS +) +@click.pass_context +def mkvenv(ctx, projects): + """ + make a venv and test the source distribution + """ + ctx.invoke(sdist) + with empty_pythonpath(): + print("Creating virtualenv for test install...") + if os.path.exists(TEST_VENV_DIR): + shutil.rmtree(TEST_VENV_DIR) + subprocess.check_call(["virtualenv", "-q", TEST_VENV_DIR]) + + pip = join(TEST_VENV_DIR, VENV_BIN, "pip") + with chdir(DIST_DIR): + for project in projects: + print("Installing %s..." % project) + dist = join(ROOT_DIR, project) + subprocess.check_call([pip, "install", "-q", dist]) + + print("Running binaries...") + for project in projects: + for tool in TOOLS[project]: + tool = join(TEST_VENV_DIR, VENV_BIN, tool) + print(tool) + print(subprocess.check_output([tool, "--version"])) + + print("Virtualenv available for further testing:") + print( + "source %s" % os.path.normpath( + join(TEST_VENV_DIR, VENV_BIN, "activate") + ) + ) + + +@cli.command("upload") +@click.option('--username', prompt=True) +@click.password_option(confirmation_prompt=False) +@click.option('--repository', default="pypi") +def upload_release(username, password, repository): + """ + Upload source distributions to PyPI + """ + print("Uploading distributions...") + subprocess.check_call([ + "twine", + "upload", + "-u", username, + "-p", password, + "-r", repository, + "%s/*" % DIST_DIR + ]) + + +# TODO: Fully automate build process. +# This wizard is missing OSX builds and updating mitmproxy.org. +@cli.command("wizard") +@click.option('--version', prompt=True) +@click.option('--username', prompt="PyPI Username") +@click.password_option(confirmation_prompt=False, prompt="PyPI Password") +@click.option('--repository', default="pypi") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS +) +@click.pass_context +def wizard(ctx, version, username, password, repository, projects): + """ + Interactive Release Wizard + """ + for project in projects: + if subprocess.check_output( + ["git", "status", "--porcelain"], cwd=join(ROOT_DIR, project) + ): + raise RuntimeError("%s repository is not clean." % project) + + # Build test release + ctx.invoke(sdist, projects=projects) + ctx.invoke(test, projects=projects) + click.confirm("Please test the release now. Is it ok?", abort=True) + + # bump version, update docs and contributors + ctx.invoke(set_version, version=version, projects=projects) + ctx.invoke(docs) + ctx.invoke(contributors) + + # version bump commit + tag + ctx.invoke( + git, args=["commit", "-a", "-m", "bump version"], projects=projects + ) + ctx.invoke(git, args=["tag", "v" + version], projects=projects) + ctx.invoke(git, args=["push"], projects=projects) + ctx.invoke(git, args=["push", "--tags"], projects=projects) + + # Re-invoke sdist with bumped version + ctx.invoke(sdist, projects=projects) + click.confirm("All good, can upload to PyPI?", abort=True) + ctx.invoke( + upload_release, + username=username, password=password, repository=repository + ) + click.echo("All done!") + + +if __name__ == "__main__": + cli() -- cgit v1.2.3