#!/usr/bin/env python from __future__ import ( absolute_import, print_function, division, unicode_literals ) from os.path import join import contextlib import os.path import os import shutil import subprocess import glob import re import shlex import runpy import pprint import click # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ if os.name == "nt": VENV_BIN = "Scripts" else: VENV_BIN = "bin" RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) DIST_DIR = join(RELEASE_DIR, "release") ROOT_DIR = join(RELEASE_DIR, "..") MITMPROXY_DIR = join(ROOT_DIR, "mitmproxy") PYINSTALLER_URL =\ "https://github.com/pyinstaller/pyinstaller/archive/develop.zip" PYINSTALLER_CACHE = os.path.expanduser( "~/Library/Application Support/pyinstaller" ) PYINSTALLER_DIST = join(RELEASE_DIR, "pyinstallerdist") VENV_DIR = join(RELEASE_DIR, "venv") VENV_PIP = join(VENV_DIR, VENV_BIN, "pip") VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") PROJECTS = { "netlib": { "tools": [], "vfile": join(ROOT_DIR, "netlib/netlib/version.py"), "version": None, "dir": None }, "pathod": { "tools": ["pathod", "pathoc"], "vfile": join(ROOT_DIR, "pathod/libpathod/version.py"), "version": None, "dir": None }, "mitmproxy": { "tools": ["mitmproxy", "mitmdump", "mitmweb"], "vfile": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"), "version": None, "dir": None } } if os.name == "nt": PROJECTS["mitmproxy"]["tools"].remove("mitmproxy") for project, settings in PROJECTS.items(): settings["version"] = runpy.run_path(settings["vfile"])["VERSION"] settings["dir"] = join(ROOT_DIR, project) def proj(spec): """ A small helper to iterate over filtered projects. """ for k, v in PROJECTS.items(): if k not in spec: continue yield k, v @contextlib.contextmanager def empty_pythonpath(): """ Make sure that the regular python installation is not on the python path, which would give us access to modules installed outside of our virtualenv. """ pythonpath = os.environ["PYTHONPATH"] os.environ["PYTHONPATH"] = "" yield os.environ["PYTHONPATH"] = pythonpath @contextlib.contextmanager def chdir(path): old_dir = os.getcwd() os.chdir(path) yield os.chdir(old_dir) @click.group(chain=True) def cli(): """ mitmproxy build tool """ pass @cli.command("contributors") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) def contributors(projects): """ Update CONTRIBUTORS.md """ for project, conf in proj(projects): with chdir(conf["dir"]): print("Updating %s/CONTRIBUTORS..."%project) contributors_data = subprocess.check_output( shlex.split("git shortlog -n -s") ) with open("CONTRIBUTORS", "w+") as f: f.write(contributors_data) @cli.command("docs") def docs(): """ Render the docs """ print("Rendering the docs...") subprocess.check_call([ "cshape", join(MITMPROXY_DIR, "doc-src"), join(MITMPROXY_DIR, "doc") ]) @cli.command("set-version") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) @click.argument('version') def set_version(projects, version): """ Update version information """ print("Update versions...") version = ", ".join(version.split(".")) for p, conf in proj(projects): print("Update %s..." % conf["vfile"]) with open(conf["vfile"], "rb") as f: content = f.read() new_content = re.sub( r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content ) with open(conf["vfile"], "wb") as f: f.write(new_content) @cli.command("git") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) @click.argument('args', nargs=-1, required=True) def git(projects, args): """ Run a git command on every project """ args = ["git"] + list(args) for project, conf in proj(projects): print("%s> %s..." % (project, " ".join(args))) subprocess.check_call( args, cwd=conf["dir"] ) @cli.command("sdist") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) def sdist(projects): """ Build a source distribution """ with empty_pythonpath(): print("Building release...") if os.path.exists(DIST_DIR): shutil.rmtree(DIST_DIR) for project, conf in proj(projects): print("Creating %s source distribution..." % project) subprocess.check_call( [ "python", "./setup.py", "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar" ], cwd=conf["dir"] ) @cli.command("osxbin") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) @click.pass_context def osxbin(ctx, projects): if not os.path.exists(VENV_PYINSTALLER): print("Instaling PyInstaller...") subprocess.check_call( [ VENV_PIP, "install", PYINSTALLER_URL ] ) shutil.rmtree(PYINSTALLER_CACHE, ignore_errors=True) shutil.rmtree("./build", ignore_errors=True) shutil.rmtree(PYINSTALLER_DIST, ignore_errors=True) for p, conf in proj(projects): specs = glob.glob(os.path.join(conf["dir"], "release/*.spec")) if specs: for spec in specs: subprocess.check_call( [ VENV_PYINSTALLER, "--distpath", PYINSTALLER_DIST, spec ] ) bins = os.listdir(PYINSTALLER_DIST) base = os.path.join(DIST_DIR, "osx-" + p + "-" + conf["version"]) shutil.rmtree(base, ignore_errors=True) os.makedirs(base) for bin in bins: bin = os.path.join(PYINSTALLER_DIST, bin) subprocess.check_call([bin, "--version"]) shutil.move(bin, base) subprocess.check_call( [ "tar", "-czvf", base + ".tgz", base ] ) @cli.command("mkvenv") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) @click.pass_context def mkvenv(ctx, projects): """ make a venv and test the source distribution """ ctx.invoke(sdist) with empty_pythonpath(): print("Creating virtualenv for test install...") if os.path.exists(VENV_DIR): shutil.rmtree(VENV_DIR) subprocess.check_call(["virtualenv", "-q", VENV_DIR]) with chdir(DIST_DIR): for project, conf in proj(projects): print("Installing %s..." % project) subprocess.check_call([VENV_PIP, "install", "-q", conf["dir"]]) print("Running binaries...") for project, conf in proj(projects): for tool in PROJECTS[project]["tools"]: tool = join(VENV_DIR, VENV_BIN, tool) print(tool) print(subprocess.check_output([tool, "--version"])) print("Virtualenv available for further testing:") print( "source %s" % os.path.normpath( join(VENV_DIR, VENV_BIN, "activate") ) ) @cli.command("upload") @click.option('--username', prompt=True) @click.password_option(confirmation_prompt=False) @click.option('--repository', default="pypi") def upload_release(username, password, repository): """ Upload source distributions to PyPI """ print("Uploading distributions...") subprocess.check_call([ "twine", "upload", "-u", username, "-p", password, "-r", repository, "%s/*" % DIST_DIR ]) # TODO: Fully automate build process. # This wizard is missing OSX builds and updating mitmproxy.org. @cli.command("wizard") @click.option('--version', prompt=True) @click.option('--username', prompt="PyPI Username") @click.password_option(confirmation_prompt=False, prompt="PyPI Password") @click.option('--repository', default="pypi") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) @click.pass_context def wizard(ctx, version, username, password, repository, projects): """ Interactive Release Wizard """ for project, conf in proj(projects): if subprocess.check_output( ["git", "status", "--porcelain"], cwd=conf["dir"] ): raise RuntimeError("%s repository is not clean." % project) # Build test release ctx.invoke(sdist, projects=projects) ctx.invoke(test, projects=projects) click.confirm("Please test the release now. Is it ok?", abort=True) # bump version, update docs and contributors ctx.invoke(set_version, version=version, projects=projects) ctx.invoke(docs) ctx.invoke(contributors) # version bump commit + tag ctx.invoke( git, args=["commit", "-a", "-m", "bump version"], projects=projects ) ctx.invoke(git, args=["tag", "v" + version], projects=projects) ctx.invoke(git, args=["push"], projects=projects) ctx.invoke(git, args=["push", "--tags"], projects=projects) # Re-invoke sdist with bumped version ctx.invoke(sdist, projects=projects) click.confirm("All good, can upload to PyPI?", abort=True) ctx.invoke( upload_release, username=username, password=password, repository=repository ) click.echo("All done!") if __name__ == "__main__": cli()