From 09ff5df2fb354a017afa02138b329e615a80c1d0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 2 Jun 2018 21:37:44 +0200 Subject: Such CI, Very Wow. (#3182) --- release/cibuild.py | 190 ++++++++++++++++++++++---------- release/installbuilder/.gitignore | 2 +- release/installbuilder/setup/.gitignore | 2 + 3 files changed, 133 insertions(+), 61 deletions(-) create mode 100644 release/installbuilder/setup/.gitignore (limited to 'release') diff --git a/release/cibuild.py b/release/cibuild.py index 04ca9474..17fb8554 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 -import glob -import re import contextlib +import glob import os +import pathlib import platform -import sys +import re import shutil import subprocess +import sys import tarfile +import urllib.request import zipfile import click @@ -38,26 +40,23 @@ class BuildEnviron: def __init__( self, *, - system = "", - root_dir = "", - - travis_tag = "", - travis_branch = "", - travis_pull_request = "", - - appveyor_repo_tag_name = "", - appveyor_repo_branch = "", - appveyor_pull_request_number = "", - - should_build_wheel = False, - should_build_docker = False, - should_build_pyinstaller = False, - - has_aws_creds = False, - has_twine_creds = False, - - docker_username = "", - docker_password = "", + system="", + root_dir="", + travis_tag="", + travis_branch="", + travis_pull_request="", + appveyor_repo_tag_name="", + appveyor_repo_branch="", + appveyor_pull_request_number="", + should_build_wheel=False, + should_build_docker=False, + should_build_pyinstaller=False, + should_build_wininstaller=False, + has_aws_creds=False, + has_twine_creds=False, + docker_username="", + docker_password="", + rtool_key="", ): self.system = system self.root_dir = root_dir @@ -69,6 +68,7 @@ class BuildEnviron: self.should_build_wheel = should_build_wheel self.should_build_docker = should_build_docker self.should_build_pyinstaller = should_build_pyinstaller + self.should_build_wininstaller = should_build_wininstaller self.appveyor_repo_tag_name = appveyor_repo_tag_name self.appveyor_repo_branch = appveyor_repo_branch @@ -78,33 +78,31 @@ class BuildEnviron: self.has_twine_creds = has_twine_creds self.docker_username = docker_username self.docker_password = docker_password + self.rtool_key = rtool_key @classmethod - def from_env(klass): - return klass( - system = platform.system(), - root_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), "..")), - - travis_tag = os.environ.get("TRAVIS_TAG", ""), - travis_branch = os.environ.get("TRAVIS_BRANCH", ""), - travis_pull_request = os.environ.get("TRAVIS_PULL_REQUEST"), - - appveyor_repo_tag_name = os.environ.get("APPVEYOR_REPO_TAG_NAME", ""), - appveyor_repo_branch = os.environ.get("APPVEYOR_REPO_BRANCH", ""), - appveyor_pull_request_number = os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"), - - should_build_wheel = "WHEEL" in os.environ, - should_build_pyinstaller = "PYINSTALLER" in os.environ, - should_build_docker = "DOCKER" in os.environ, - - has_aws_creds = "AWS_ACCESS_KEY_ID" in os.environ, - has_twine_creds= ( + def from_env(cls): + return cls( + system=platform.system(), + root_dir=os.path.normpath(os.path.join(os.path.dirname(__file__), "..")), + travis_tag=os.environ.get("TRAVIS_TAG", ""), + travis_branch=os.environ.get("TRAVIS_BRANCH", ""), + travis_pull_request=os.environ.get("TRAVIS_PULL_REQUEST"), + appveyor_repo_tag_name=os.environ.get("APPVEYOR_REPO_TAG_NAME", ""), + appveyor_repo_branch=os.environ.get("APPVEYOR_REPO_BRANCH", ""), + appveyor_pull_request_number=os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"), + should_build_wheel="WHEEL" in os.environ, + should_build_pyinstaller="PYINSTALLER" in os.environ, + should_build_wininstaller="WININSTALLER" in os.environ, + should_build_docker="DOCKER" in os.environ, + has_aws_creds="AWS_ACCESS_KEY_ID" in os.environ, + has_twine_creds=( "TWINE_USERNAME" in os.environ and "TWINE_PASSWORD" in os.environ ), - - docker_username = os.environ.get("DOCKER_USERNAME"), - docker_password = os.environ.get("DOCKER_PASSWORD"), + docker_username=os.environ.get("DOCKER_USERNAME"), + docker_password=os.environ.get("DOCKER_PASSWORD"), + rtool_key=os.environ.get("RTOOL_KEY"), ) def archive(self, path): @@ -177,16 +175,45 @@ class BuildEnviron: "should_upload_pypi", ] for attr in lst: - print("cibuild.%s=%s" % (attr, getattr(self, attr)), file=fp) + print(f"cibuild.{attr}={getattr(self, attr)}", file=fp) + + def check_version(self) -> None: + """ + Check that version numbers match our conventions. + Raises a ValueError if there is a mismatch. + """ + with open(pathlib.Path(self.root_dir) / "mitmproxy" / "version.py") as f: + contents = f.read() + + version = re.search(r'^VERSION = "(.+?)"', contents, re.M).group(1) + + if self.tag: + # For (tagged) releases, we are strict: + # 1. The tagname must match the version in mitmproxy/version.py + # 2. The version info must be in canonical form (as recommended in PEP 440). + + if version != self.tag: + raise ValueError(f"Tag is {self.tag}, but mitmproxy/version.py is {version}.") + try: + parver.Version.parse(version, strict=True) + except parver.ParseError as e: + raise ValueError(str(e)) from e + else: + # For snapshots, we only ensure that mitmproxy/version.py contains a dev release. + version_info = parver.Version.parse(version) + if not version_info.is_devrelease: + raise ValueError("Releases must be tagged.") @property def has_docker_creds(self) -> bool: - return self.docker_username and self.docker_password + return bool(self.docker_username and self.docker_password) @property def is_prod_release(self) -> bool: + if not self.tag: + return False try: - v = parver.Version.parse(self.version) + v = parver.Version.parse(self.version, strict=True) except (parver.ParseError, BuildError): return False return not v.is_prerelease @@ -212,7 +239,7 @@ class BuildEnviron: @property def should_upload_docker(self) -> bool: return all([ - (self.tag or self.branch == "master"), + (self.is_prod_release or self.branch == "master"), self.should_build_docker, self.has_docker_creds, ]) @@ -220,7 +247,6 @@ class BuildEnviron: @property def should_upload_pypi(self) -> bool: return all([ - self.tag, self.is_prod_release, self.should_build_wheel, self.has_twine_creds, @@ -242,7 +268,7 @@ class BuildEnviron: name = self.tag or self.branch if not name: raise BuildError("We're on neither a tag nor a branch - could not establish version") - return re.sub('^v', "", name) + return name def build_wheel(be: BuildEnviron): # pragma: no cover @@ -354,6 +380,56 @@ def build_pyinstaller(be: BuildEnviron): # pragma: no cover click.echo("Packed {}.".format(be.archive_name(bdist))) +def build_wininstaller(be: BuildEnviron): # pragma: no cover + click.echo("Building wininstaller package...") + + IB_VERSION = "18.5.2" + IB_DIR = pathlib.Path(be.release_dir) / "installbuilder" + IB_SETUP = IB_DIR / "setup" / f"{IB_VERSION}-installer.exe" + IB_CLI = fr"C:\Program Files (x86)\BitRock InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe" + IB_LICENSE = IB_DIR / "license.xml" + + if True or not os.path.isfile(IB_CLI): + if not os.path.isfile(IB_SETUP): + click.echo("Downloading InstallBuilder...") + + def report(block, blocksize, total): + done = block * blocksize + if round(100 * done / total) != round(100 * (done - blocksize) / total): + click.secho(f"Downloading... {round(100*done/total)}%") + + urllib.request.urlretrieve( + f"https://installbuilder.bitrock.com/installbuilder-enterprise-{IB_VERSION}-windows-installer.exe", + IB_SETUP.with_suffix(".tmp"), + reporthook=report + ) + shutil.move(IB_SETUP.with_suffix(".tmp"), IB_SETUP) + + click.echo("Install InstallBuilder...") + subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"], + check=True) + assert os.path.isfile(IB_CLI) + + click.echo("Decrypt InstallBuilder license...") + f = cryptography.fernet.Fernet(be.rtool_key.encode()) + with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, open(IB_LICENSE, + "wb") as outfile: + outfile.write(f.decrypt(infile.read())) + + click.echo("Run InstallBuilder...") + subprocess.run([ + IB_CLI, + "build", + str(IB_DIR / "mitmproxy.xml"), + "windows", + "--license", str(IB_LICENSE), + "--setvars", f"project.version={be.version}", + "--verbose" + ], check=True) + assert os.path.isfile( + os.path.join(be.dist_dir, f"mitmproxy-{be.version}-windows-installer.exe")) + + @click.group(chain=True) def cli(): # pragma: no cover """ @@ -370,6 +446,7 @@ def build(): # pragma: no cover be = BuildEnviron.from_env() be.dump_info() + be.check_version() os.makedirs(be.dist_dir, exist_ok=True) if be.should_build_wheel: @@ -379,6 +456,8 @@ def build(): # pragma: no cover build_docker_image(be, whl) if be.should_build_pyinstaller: build_pyinstaller(be) + if be.should_build_wininstaller and be.rtool_key: + build_wininstaller(be) @cli.command("upload") @@ -421,14 +500,5 @@ def upload(): # pragma: no cover subprocess.check_call(["docker", "push", be.docker_tag]) -@cli.command("decrypt") -@click.argument('infile', type=click.File('rb')) -@click.argument('outfile', type=click.File('wb')) -@click.argument('key', envvar='RTOOL_KEY') -def decrypt(infile, outfile, key): # pragma: no cover - f = cryptography.fernet.Fernet(key.encode()) - outfile.write(f.decrypt(infile.read())) - - if __name__ == "__main__": # pragma: no cover cli() diff --git a/release/installbuilder/.gitignore b/release/installbuilder/.gitignore index 00c10a2d..b0fdb6d1 100644 --- a/release/installbuilder/.gitignore +++ b/release/installbuilder/.gitignore @@ -1,2 +1,2 @@ license.xml -*.xml.backup \ No newline at end of file +*.xml.backup diff --git a/release/installbuilder/setup/.gitignore b/release/installbuilder/setup/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/release/installbuilder/setup/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore -- cgit v1.2.3