#!/usr/bin/env python from __future__ import ( absolute_import, print_function, division, unicode_literals ) from os.path import join import contextlib import os.path import os import shutil import subprocess import glob import re import shlex import click # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ if os.name == "nt": VENV_BIN = "Scripts" else: VENV_BIN = "bin" RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) DIST_DIR = join(RELEASE_DIR, "release") ROOT_DIR = join(RELEASE_DIR, "..") MITMPROXY_DIR = join(ROOT_DIR, "mitmproxy") TEST_VENV_DIR = join(RELEASE_DIR, "venv") PROJECTS = ("netlib", "pathod", "mitmproxy") TOOLS = { "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], "pathod": ["pathod", "pathoc"], "netlib": [] } if os.name == "nt": TOOLS["mitmproxy"].remove("mitmproxy") VERSION_FILES = { "mitmproxy": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"), "pathod": join(ROOT_DIR, "pathod/libpathod/version.py"), "netlib": join(ROOT_DIR, "netlib/netlib/version.py"), } @contextlib.contextmanager def empty_pythonpath(): """ Make sure that the regular python installation is not on the python path, which would give us access to modules installed outside of our virtualenv. """ pythonpath = os.environ["PYTHONPATH"] os.environ["PYTHONPATH"] = "" yield os.environ["PYTHONPATH"] = pythonpath @contextlib.contextmanager def chdir(path): old_dir = os.getcwd() os.chdir(path) yield os.chdir(old_dir) @click.group(chain=True) def cli(): """ mitmproxy build tool """ pass @cli.command("contributors") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) def contributors(projects): """ Update CONTRIBUTORS.md """ for project in PROJECTS: if project not in projects: continue with chdir(os.path.join(ROOT_DIR, project)): print("Updating %s/CONTRIBUTORS..."%project) contributors_data = subprocess.check_output( shlex.split("git shortlog -n -s") ) with open("CONTRIBUTORS", "w+") as f: f.write(contributors_data) @cli.command("docs") def docs(): """ Render the docs """ print("Rendering the docs...") subprocess.check_call([ "cshape", join(MITMPROXY_DIR, "doc-src"), join(MITMPROXY_DIR, "doc") ]) @cli.command("set-version") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) @click.argument('version') def set_version(projects, version): """ Update version information """ print("Update versions...") version = ", ".join(version.split(".")) for project, version_file in VERSION_FILES.items(): if project not in projects: continue print("Update %s..." % version_file) with open(version_file, "rb") as f: content = f.read() new_content = re.sub( r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content ) with open(version_file, "wb") as f: f.write(new_content) @cli.command("git") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) @click.argument('args', nargs=-1, required=True) def git(projects, args): """ Run a git command on every project """ args = ["git"] + list(args) for project in projects: print("%s> %s..." % (project, " ".join(args))) subprocess.check_call( args, cwd=join(ROOT_DIR, project) ) @cli.command("sdist") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) def sdist(projects): """ Build a source distribution """ with empty_pythonpath(): print("Building release...") if os.path.exists(DIST_DIR): shutil.rmtree(DIST_DIR) for project in projects: print("Creating %s source distribution..." % project) subprocess.check_call( [ "python", "./setup.py", "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar" ], cwd=join(ROOT_DIR, project) ) @cli.command("mkvenv") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) @click.pass_context def mkvenv(ctx, projects): """ make a venv and test the source distribution """ ctx.invoke(sdist) with empty_pythonpath(): print("Creating virtualenv for test install...") if os.path.exists(TEST_VENV_DIR): shutil.rmtree(TEST_VENV_DIR) subprocess.check_call(["virtualenv", "-q", TEST_VENV_DIR]) pip = join(TEST_VENV_DIR, VENV_BIN, "pip") with chdir(DIST_DIR): for project in projects: print("Installing %s..." % project) dist = join(ROOT_DIR, project) subprocess.check_call([pip, "install", "-q", dist]) print("Running binaries...") for project in projects: for tool in TOOLS[project]: tool = join(TEST_VENV_DIR, VENV_BIN, tool) print(tool) print(subprocess.check_output([tool, "--version"])) print("Virtualenv available for further testing:") print( "source %s" % os.path.normpath( join(TEST_VENV_DIR, VENV_BIN, "activate") ) ) @cli.command("upload") @click.option('--username', prompt=True) @click.password_option(confirmation_prompt=False) @click.option('--repository', default="pypi") def upload_release(username, password, repository): """ Upload source distributions to PyPI """ print("Uploading distributions...") subprocess.check_call([ "twine", "upload", "-u", username, "-p", password, "-r", repository, "%s/*" % DIST_DIR ]) # TODO: Fully automate build process. # This wizard is missing OSX builds and updating mitmproxy.org. @cli.command("wizard") @click.option('--version', prompt=True) @click.option('--username', prompt="PyPI Username") @click.password_option(confirmation_prompt=False, prompt="PyPI Password") @click.option('--repository', default="pypi") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) @click.pass_context def wizard(ctx, version, username, password, repository, projects): """ Interactive Release Wizard """ for project in projects: if subprocess.check_output( ["git", "status", "--porcelain"], cwd=join(ROOT_DIR, project) ): raise RuntimeError("%s repository is not clean." % project) # Build test release ctx.invoke(sdist, projects=projects) ctx.invoke(test, projects=projects) click.confirm("Please test the release now. Is it ok?", abort=True) # bump version, update docs and contributors ctx.invoke(set_version, version=version, projects=projects) ctx.invoke(docs) ctx.invoke(contributors) # version bump commit + tag ctx.invoke( git, args=["commit", "-a", "-m", "bump version"], projects=projects ) ctx.invoke(git, args=["tag", "v" + version], projects=projects) ctx.invoke(git, args=["push"], projects=projects) ctx.invoke(git, args=["push", "--tags"], projects=projects) # Re-invoke sdist with bumped version ctx.invoke(sdist, projects=projects) click.confirm("All good, can upload to PyPI?", abort=True) ctx.invoke( upload_release, username=username, password=password, repository=repository ) click.echo("All done!") if __name__ == "__main__": cli()