diff options
author | Thomas Kriechbaumer <thomas@kriechbaumer.name> | 2018-05-18 10:37:56 +0200 |
---|---|---|
committer | Thomas Kriechbaumer <thomas@kriechbaumer.name> | 2018-05-19 12:38:44 +0200 |
commit | 46960fa080c63b912c76321f790573fe4995ed49 (patch) | |
tree | 64d4c3fd8dd4c7c8512ef4dc6539ff19be14a55d /release/cibuild.py | |
parent | eaca8fdaf7b1b3c81451a407593aa7ea7f9ffc31 (diff) | |
download | mitmproxy-46960fa080c63b912c76321f790573fe4995ed49.tar.gz mitmproxy-46960fa080c63b912c76321f790573fe4995ed49.tar.bz2 mitmproxy-46960fa080c63b912c76321f790573fe4995ed49.zip |
add docker to the main repository
Diffstat (limited to 'release/cibuild.py')
-rwxr-xr-x | release/cibuild.py | 321 |
1 files changed, 321 insertions, 0 deletions
diff --git a/release/cibuild.py b/release/cibuild.py new file mode 100755 index 00000000..3a79affe --- /dev/null +++ b/release/cibuild.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 + +import glob +import re +import contextlib +import os +import platform +import sys +import shutil +import subprocess +import tarfile +import zipfile +from os.path import join, abspath, dirname, exists, basename + +import click +import cryptography.fernet + +# ZipFile and tarfile have slightly different APIs. Fix that. +if platform.system() == "Windows": + def Archive(name): + a = zipfile.ZipFile(name, "w") + a.add = a.write + return a +else: + def Archive(name): + return tarfile.open(name, "w:gz") + +PLATFORM_TAG = { + "Darwin": "osx", + "Windows": "windows", + "Linux": "linux", +}.get(platform.system(), platform.system()) + +ROOT_DIR = abspath(join(dirname(__file__), "..")) +RELEASE_DIR = join(ROOT_DIR, "release") +BUILD_DIR = join(RELEASE_DIR, "build") +DIST_DIR = join(RELEASE_DIR, "dist") + +BDISTS = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"] +} +if platform.system() == "Windows": + BDISTS["mitmproxy"].remove("mitmproxy") + +TOOLS = [ + tool + for tools in sorted(BDISTS.values()) + for tool in tools +] + +TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None)) +BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None)) +if TAG: + VERSION = re.sub('^v', '', TAG) + UPLOAD_DIR = VERSION +elif BRANCH: + VERSION = re.sub('^v', '', BRANCH) + UPLOAD_DIR = "branches/%s" % VERSION +else: + print("Could not establish build name - exiting." % BRANCH) + sys.exit(0) + +print("BUILD PLATFORM_TAG=%s" % PLATFORM_TAG) +print("BUILD ROOT_DIR=%s" % ROOT_DIR) +print("BUILD RELEASE_DIR=%s" % RELEASE_DIR) +print("BUILD BUILD_DIR=%s" % BUILD_DIR) +print("BUILD DIST_DIR=%s" % DIST_DIR) +print("BUILD BDISTS=%s" % BDISTS) +print("BUILD TAG=%s" % TAG) +print("BUILD BRANCH=%s" % BRANCH) +print("BUILD VERSION=%s" % VERSION) +print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR) + + +def archive_name(bdist: str) -> str: + if platform.system() == "Windows": + ext = "zip" + else: + ext = "tar.gz" + return "{project}-{version}-{platform}.{ext}".format( + project=bdist, + version=VERSION, + platform=PLATFORM_TAG, + ext=ext + ) + + +@contextlib.contextmanager +def chdir(path: str): + old_dir = os.getcwd() + os.chdir(path) + yield + os.chdir(old_dir) + + +@click.group(chain=True) +def cli(): + """ + mitmproxy build tool + """ + pass + + +@cli.command("build") +def build(): + """ + Build a binary distribution + """ + os.makedirs(DIST_DIR, exist_ok=True) + + if "WHEEL" in os.environ: + whl = build_wheel() + else: + click.echo("Not building wheels.") + + if "WHEEL" in os.environ and "DOCKER" in os.environ: + # Docker image requires wheels + build_docker_image(whl) + else: + click.echo("Not building Docker image.") + + if "PYINSTALLER" in os.environ: + build_pyinstaller() + else: + click.echo("Not building PyInstaller packages.") + + +def build_wheel(): + click.echo("Building wheel...") + subprocess.check_call([ + "python", + "setup.py", + "-q", + "bdist_wheel", + "--dist-dir", DIST_DIR, + ]) + + whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0] + click.echo("Found wheel package: {}".format(whl)) + + subprocess.check_call([ + "tox", + "-e", "wheeltest", + "--", + whl + ]) + + return whl + + +def build_docker_image(whl): + click.echo("Building Docker image...") + subprocess.check_call([ + "docker", + "build", + "--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, ROOT_DIR)), + "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(basename(whl)), + "--file", "docker/Dockerfile", + "." + ]) + + +def build_pyinstaller(): + PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") + # PyInstaller 3.2 does not bundle pydivert's Windivert binaries + PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") + PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") + PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) + + # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes + # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ + if platform.system() == "Windows": + PYINSTALLER_ARGS = [ + # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly. + "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", + ] + else: + PYINSTALLER_ARGS = [] + + if exists(PYINSTALLER_TEMP): + shutil.rmtree(PYINSTALLER_TEMP) + if exists(PYINSTALLER_DIST): + shutil.rmtree(PYINSTALLER_DIST) + + for bdist, tools in sorted(BDISTS.items()): + with Archive(join(DIST_DIR, archive_name(bdist))) as archive: + for tool in tools: + # We can't have a folder and a file with the same name. + if tool == "mitmproxy": + tool = "mitmproxy_main" + # This is PyInstaller, so it messes up paths. + # We need to make sure that we are in the spec folder. + with chdir(PYINSTALLER_SPEC): + click.echo("Building PyInstaller %s binary..." % tool) + excludes = [] + if tool != "mitmweb": + excludes.append("mitmproxy.tools.web") + if tool != "mitmproxy_main": + excludes.append("mitmproxy.tools.console") + + subprocess.check_call( + [ + "pyinstaller", + "--clean", + "--workpath", PYINSTALLER_TEMP, + "--distpath", PYINSTALLER_DIST, + "--additional-hooks-dir", PYINSTALLER_HOOKS, + "--onefile", + "--console", + "--icon", "icon.ico", + # This is PyInstaller, so setting a + # different log level obviously breaks it :-) + # "--log-level", "WARN", + ] + + [x for e in excludes for x in ["--exclude-module", e]] + + PYINSTALLER_ARGS + + [tool] + ) + # Delete the spec file - we're good without. + os.remove("{}.spec".format(tool)) + + # Test if it works at all O:-) + executable = join(PYINSTALLER_DIST, tool) + if platform.system() == "Windows": + executable += ".exe" + + # Remove _main suffix from mitmproxy executable + if "_main" in executable: + shutil.move( + executable, + executable.replace("_main", "") + ) + executable = executable.replace("_main", "") + + click.echo("> %s --version" % executable) + click.echo(subprocess.check_output([executable, "--version"]).decode()) + + archive.add(executable, basename(executable)) + click.echo("Packed {}.".format(archive_name(bdist))) + + +@cli.command("upload") +def upload(): + """ + Upload build artifacts + + Uploads the wheels package to PyPi. + Uploads the Pyinstaller and wheels packages to the snapshot server. + Pushes the Docker image to Docker Hub. + """ + + # Our credentials are only available from within the main repository and not forks. + # We need to prevent uploads from all BUT the branches in the main repository. + # Pull requests and master-branches of forks are not allowed to upload. + is_pull_request = ( + ("TRAVIS_PULL_REQUEST" in os.environ and os.environ["TRAVIS_PULL_REQUEST"] != "false") or + "APPVEYOR_PULL_REQUEST_NUMBER" in os.environ + ) + if is_pull_request: + click.echo("Refusing to upload artifacts from a pull request!") + return + + if "AWS_ACCESS_KEY_ID" in os.environ: + subprocess.check_call([ + "aws", "s3", "cp", + "--acl", "public-read", + DIST_DIR + "/", + "s3://snapshots.mitmproxy.org/{}/".format(UPLOAD_DIR), + "--recursive", + ]) + + upload_pypi = ( + TAG and + "WHEEL" in os.environ and + "TWINE_USERNAME" in os.environ and + "TWINE_PASSWORD" in os.environ + ) + if upload_pypi: + whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0] + click.echo("Uploading {} to PyPi...".format(whl)) + subprocess.check_call([ + "twine", + "upload", + whl + ]) + + upload_docker = ( + (TAG or BRANCH == "master") and + "DOCKER" in os.environ and + "DOCKER_USERNAME" in os.environ and + "DOCKER_PASSWORD" in os.environ + ) + if upload_docker: + docker_tag = "dev" if BRANCH == "master" else VERSION + + click.echo("Uploading Docker image to tag={}...".format(docker_tag)) + subprocess.check_call([ + "docker", + "login", + "-u", os.environ["DOCKER_USERNAME"], + "-p", os.environ["DOCKER_PASSWORD"], + ]) + subprocess.check_call([ + "docker", + "push", + "mitmproxy/mitmproxy:{}".format(docker_tag), + ]) + + +@cli.command("decrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def decrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.decrypt(infile.read())) + + +if __name__ == "__main__": + cli() |