aboutsummaryrefslogtreecommitdiffstats
path: root/release/cibuild.py
diff options
context:
space:
mode:
Diffstat (limited to 'release/cibuild.py')
-rwxr-xr-xrelease/cibuild.py321
1 files changed, 321 insertions, 0 deletions
diff --git a/release/cibuild.py b/release/cibuild.py
new file mode 100755
index 00000000..3a79affe
--- /dev/null
+++ b/release/cibuild.py
@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+
+import glob
+import re
+import contextlib
+import os
+import platform
+import sys
+import shutil
+import subprocess
+import tarfile
+import zipfile
+from os.path import join, abspath, dirname, exists, basename
+
+import click
+import cryptography.fernet
+
+# ZipFile and tarfile have slightly different APIs. Fix that.
+if platform.system() == "Windows":
+ def Archive(name):
+ a = zipfile.ZipFile(name, "w")
+ a.add = a.write
+ return a
+else:
+ def Archive(name):
+ return tarfile.open(name, "w:gz")
+
+PLATFORM_TAG = {
+ "Darwin": "osx",
+ "Windows": "windows",
+ "Linux": "linux",
+}.get(platform.system(), platform.system())
+
+ROOT_DIR = abspath(join(dirname(__file__), ".."))
+RELEASE_DIR = join(ROOT_DIR, "release")
+BUILD_DIR = join(RELEASE_DIR, "build")
+DIST_DIR = join(RELEASE_DIR, "dist")
+
+BDISTS = {
+ "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
+ "pathod": ["pathoc", "pathod"]
+}
+if platform.system() == "Windows":
+ BDISTS["mitmproxy"].remove("mitmproxy")
+
+TOOLS = [
+ tool
+ for tools in sorted(BDISTS.values())
+ for tool in tools
+]
+
+TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
+BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
+if TAG:
+ VERSION = re.sub('^v', '', TAG)
+ UPLOAD_DIR = VERSION
+elif BRANCH:
+ VERSION = re.sub('^v', '', BRANCH)
+ UPLOAD_DIR = "branches/%s" % VERSION
+else:
+ print("Could not establish build name - exiting." % BRANCH)
+ sys.exit(0)
+
+print("BUILD PLATFORM_TAG=%s" % PLATFORM_TAG)
+print("BUILD ROOT_DIR=%s" % ROOT_DIR)
+print("BUILD RELEASE_DIR=%s" % RELEASE_DIR)
+print("BUILD BUILD_DIR=%s" % BUILD_DIR)
+print("BUILD DIST_DIR=%s" % DIST_DIR)
+print("BUILD BDISTS=%s" % BDISTS)
+print("BUILD TAG=%s" % TAG)
+print("BUILD BRANCH=%s" % BRANCH)
+print("BUILD VERSION=%s" % VERSION)
+print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR)
+
+
+def archive_name(bdist: str) -> str:
+ if platform.system() == "Windows":
+ ext = "zip"
+ else:
+ ext = "tar.gz"
+ return "{project}-{version}-{platform}.{ext}".format(
+ project=bdist,
+ version=VERSION,
+ platform=PLATFORM_TAG,
+ ext=ext
+ )
+
+
+@contextlib.contextmanager
+def chdir(path: str):
+ old_dir = os.getcwd()
+ os.chdir(path)
+ yield
+ os.chdir(old_dir)
+
+
+@click.group(chain=True)
+def cli():
+ """
+ mitmproxy build tool
+ """
+ pass
+
+
+@cli.command("build")
+def build():
+ """
+ Build a binary distribution
+ """
+ os.makedirs(DIST_DIR, exist_ok=True)
+
+ if "WHEEL" in os.environ:
+ whl = build_wheel()
+ else:
+ click.echo("Not building wheels.")
+
+ if "WHEEL" in os.environ and "DOCKER" in os.environ:
+ # Docker image requires wheels
+ build_docker_image(whl)
+ else:
+ click.echo("Not building Docker image.")
+
+ if "PYINSTALLER" in os.environ:
+ build_pyinstaller()
+ else:
+ click.echo("Not building PyInstaller packages.")
+
+
+def build_wheel():
+ click.echo("Building wheel...")
+ subprocess.check_call([
+ "python",
+ "setup.py",
+ "-q",
+ "bdist_wheel",
+ "--dist-dir", DIST_DIR,
+ ])
+
+ whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
+ click.echo("Found wheel package: {}".format(whl))
+
+ subprocess.check_call([
+ "tox",
+ "-e", "wheeltest",
+ "--",
+ whl
+ ])
+
+ return whl
+
+
+def build_docker_image(whl):
+ click.echo("Building Docker image...")
+ subprocess.check_call([
+ "docker",
+ "build",
+ "--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, ROOT_DIR)),
+ "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(basename(whl)),
+ "--file", "docker/Dockerfile",
+ "."
+ ])
+
+
+def build_pyinstaller():
+ PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
+ # PyInstaller 3.2 does not bundle pydivert's Windivert binaries
+ PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
+ PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
+ PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
+
+ # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
+ # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
+ if platform.system() == "Windows":
+ PYINSTALLER_ARGS = [
+ # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
+ "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
+ ]
+ else:
+ PYINSTALLER_ARGS = []
+
+ if exists(PYINSTALLER_TEMP):
+ shutil.rmtree(PYINSTALLER_TEMP)
+ if exists(PYINSTALLER_DIST):
+ shutil.rmtree(PYINSTALLER_DIST)
+
+ for bdist, tools in sorted(BDISTS.items()):
+ with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
+ for tool in tools:
+ # We can't have a folder and a file with the same name.
+ if tool == "mitmproxy":
+ tool = "mitmproxy_main"
+ # This is PyInstaller, so it messes up paths.
+ # We need to make sure that we are in the spec folder.
+ with chdir(PYINSTALLER_SPEC):
+ click.echo("Building PyInstaller %s binary..." % tool)
+ excludes = []
+ if tool != "mitmweb":
+ excludes.append("mitmproxy.tools.web")
+ if tool != "mitmproxy_main":
+ excludes.append("mitmproxy.tools.console")
+
+ subprocess.check_call(
+ [
+ "pyinstaller",
+ "--clean",
+ "--workpath", PYINSTALLER_TEMP,
+ "--distpath", PYINSTALLER_DIST,
+ "--additional-hooks-dir", PYINSTALLER_HOOKS,
+ "--onefile",
+ "--console",
+ "--icon", "icon.ico",
+ # This is PyInstaller, so setting a
+ # different log level obviously breaks it :-)
+ # "--log-level", "WARN",
+ ]
+ + [x for e in excludes for x in ["--exclude-module", e]]
+ + PYINSTALLER_ARGS
+ + [tool]
+ )
+ # Delete the spec file - we're good without.
+ os.remove("{}.spec".format(tool))
+
+ # Test if it works at all O:-)
+ executable = join(PYINSTALLER_DIST, tool)
+ if platform.system() == "Windows":
+ executable += ".exe"
+
+ # Remove _main suffix from mitmproxy executable
+ if "_main" in executable:
+ shutil.move(
+ executable,
+ executable.replace("_main", "")
+ )
+ executable = executable.replace("_main", "")
+
+ click.echo("> %s --version" % executable)
+ click.echo(subprocess.check_output([executable, "--version"]).decode())
+
+ archive.add(executable, basename(executable))
+ click.echo("Packed {}.".format(archive_name(bdist)))
+
+
+@cli.command("upload")
+def upload():
+ """
+ Upload build artifacts
+
+ Uploads the wheels package to PyPi.
+ Uploads the Pyinstaller and wheels packages to the snapshot server.
+ Pushes the Docker image to Docker Hub.
+ """
+
+ # Our credentials are only available from within the main repository and not forks.
+ # We need to prevent uploads from all BUT the branches in the main repository.
+ # Pull requests and master-branches of forks are not allowed to upload.
+ is_pull_request = (
+ ("TRAVIS_PULL_REQUEST" in os.environ and os.environ["TRAVIS_PULL_REQUEST"] != "false") or
+ "APPVEYOR_PULL_REQUEST_NUMBER" in os.environ
+ )
+ if is_pull_request:
+ click.echo("Refusing to upload artifacts from a pull request!")
+ return
+
+ if "AWS_ACCESS_KEY_ID" in os.environ:
+ subprocess.check_call([
+ "aws", "s3", "cp",
+ "--acl", "public-read",
+ DIST_DIR + "/",
+ "s3://snapshots.mitmproxy.org/{}/".format(UPLOAD_DIR),
+ "--recursive",
+ ])
+
+ upload_pypi = (
+ TAG and
+ "WHEEL" in os.environ and
+ "TWINE_USERNAME" in os.environ and
+ "TWINE_PASSWORD" in os.environ
+ )
+ if upload_pypi:
+ whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
+ click.echo("Uploading {} to PyPi...".format(whl))
+ subprocess.check_call([
+ "twine",
+ "upload",
+ whl
+ ])
+
+ upload_docker = (
+ (TAG or BRANCH == "master") and
+ "DOCKER" in os.environ and
+ "DOCKER_USERNAME" in os.environ and
+ "DOCKER_PASSWORD" in os.environ
+ )
+ if upload_docker:
+ docker_tag = "dev" if BRANCH == "master" else VERSION
+
+ click.echo("Uploading Docker image to tag={}...".format(docker_tag))
+ subprocess.check_call([
+ "docker",
+ "login",
+ "-u", os.environ["DOCKER_USERNAME"],
+ "-p", os.environ["DOCKER_PASSWORD"],
+ ])
+ subprocess.check_call([
+ "docker",
+ "push",
+ "mitmproxy/mitmproxy:{}".format(docker_tag),
+ ])
+
+
+@cli.command("decrypt")
+@click.argument('infile', type=click.File('rb'))
+@click.argument('outfile', type=click.File('wb'))
+@click.argument('key', envvar='RTOOL_KEY')
+def decrypt(infile, outfile, key):
+ f = cryptography.fernet.Fernet(key.encode())
+ outfile.write(f.decrypt(infile.read()))
+
+
+if __name__ == "__main__":
+ cli()
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-10 02:13:33 +0000