From e92b957e3a7324d79f2ef2a7386ed21549a5cb10 Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Tue, 19 Nov 2019 18:15:08 +0100
Subject: Use Github Actions for CI (#3713)
switch to github actions for CI
---
release/cibuild.py | 167 +++++++++++++++++++++++++++++++++--------------------
1 file changed, 105 insertions(+), 62 deletions(-)
(limited to 'release')
diff --git a/release/cibuild.py b/release/cibuild.py
index 799c7929..46066099 100755
--- a/release/cibuild.py
+++ b/release/cibuild.py
@@ -14,9 +14,10 @@ import urllib.request
import zipfile
import click
-import cryptography.fernet
import parver
+import cryptography.fernet
+
@contextlib.contextmanager
def chdir(path: str): # pragma: no cover
@@ -30,6 +31,14 @@ class BuildError(Exception):
pass
+def bool_from_env(envvar: str) -> bool:
+ val = os.environ.get(envvar, "")
+ if not val or val.lower() in ("0", "false"):
+ return False
+ else:
+ return True
+
+
class BuildEnviron:
PLATFORM_TAGS = {
"Darwin": "osx",
@@ -38,25 +47,27 @@ class BuildEnviron:
}
def __init__(
- self,
- *,
- system="",
- root_dir="",
- travis_tag="",
- travis_branch="",
- travis_pull_request="",
- appveyor_repo_tag_name="",
- appveyor_repo_branch="",
- appveyor_pull_request_number="",
- should_build_wheel=False,
- should_build_docker=False,
- should_build_pyinstaller=False,
- should_build_wininstaller=False,
- has_aws_creds=False,
- has_twine_creds=False,
- docker_username="",
- docker_password="",
- rtool_key="",
+ self,
+ *,
+ system="",
+ root_dir="",
+ travis_tag="",
+ travis_branch="",
+ travis_pull_request="",
+ appveyor_repo_tag_name="",
+ appveyor_repo_branch="",
+ appveyor_pull_request_number="",
+ github_ref="",
+ github_event_name="",
+ should_build_wheel=False,
+ should_build_docker=False,
+ should_build_pyinstaller=False,
+ should_build_wininstaller=False,
+ has_aws_creds=False,
+ has_twine_creds=False,
+ docker_username="",
+ docker_password="",
+ build_key="",
):
self.system = system
self.root_dir = root_dir
@@ -80,11 +91,14 @@ class BuildEnviron:
self.appveyor_repo_branch = appveyor_repo_branch
self.appveyor_pull_request_number = appveyor_pull_request_number
+ self.github_ref = github_ref
+ self.github_event_name = github_event_name
+
self.has_aws_creds = has_aws_creds
self.has_twine_creds = has_twine_creds
self.docker_username = docker_username
self.docker_password = docker_password
- self.rtool_key = rtool_key
+ self.build_key = build_key
@classmethod
def from_env(cls):
@@ -96,19 +110,18 @@ class BuildEnviron:
travis_pull_request=os.environ.get("TRAVIS_PULL_REQUEST"),
appveyor_repo_tag_name=os.environ.get("APPVEYOR_REPO_TAG_NAME", ""),
appveyor_repo_branch=os.environ.get("APPVEYOR_REPO_BRANCH", ""),
- appveyor_pull_request_number=os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"),
- should_build_wheel="WHEEL" in os.environ,
- should_build_pyinstaller="PYINSTALLER" in os.environ,
- should_build_wininstaller="WININSTALLER" in os.environ,
- should_build_docker="DOCKER" in os.environ,
- has_aws_creds="AWS_ACCESS_KEY_ID" in os.environ,
- has_twine_creds=(
- "TWINE_USERNAME" in os.environ and
- "TWINE_PASSWORD" in os.environ
- ),
- docker_username=os.environ.get("DOCKER_USERNAME"),
- docker_password=os.environ.get("DOCKER_PASSWORD"),
- rtool_key=os.environ.get("RTOOL_KEY"),
+ appveyor_pull_request_number=os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER", ""),
+ github_ref=os.environ.get("GITHUB_REF", ""),
+ github_event_name=os.environ.get("GITHUB_EVENT_NAME", ""),
+ should_build_wheel=bool_from_env("CI_BUILD_WHEEL"),
+ should_build_pyinstaller=bool_from_env("CI_BUILD_PYINSTALLER"),
+ should_build_wininstaller=bool_from_env("CI_BUILD_WININSTALLER"),
+ should_build_docker=bool_from_env("CI_BUILD_DOCKER"),
+ has_aws_creds=bool_from_env("AWS_ACCESS_KEY_ID"),
+ has_twine_creds=bool_from_env("TWINE_USERNAME") and bool_from_env("TWINE_PASSWORD"),
+ docker_username=os.environ.get("DOCKER_USERNAME", ""),
+ docker_password=os.environ.get("DOCKER_PASSWORD", ""),
+ build_key=os.environ.get("CI_BUILD_KEY", ""),
)
def archive(self, path):
@@ -143,26 +156,34 @@ class BuildEnviron:
return ret
@property
- def branch(self):
- return self.travis_branch or self.appveyor_repo_branch
+ def branch(self) -> str:
+ if self.travis_branch:
+ return self.travis_branch
+ if self.appveyor_repo_branch:
+ return self.appveyor_repo_branch
+ if self.github_ref and self.github_ref.startswith("refs/heads/"):
+ return self.github_ref.replace("refs/heads/", "")
+ if self.github_ref and self.github_ref.startswith("refs/pull/"):
+ return "pr-" + self.github_ref.split("/")[2]
+ return ""
@property
- def build_dir(self):
+ def build_dir(self) -> str:
return os.path.join(self.release_dir, "build")
@property
- def dist_dir(self):
+ def dist_dir(self) -> str:
return os.path.join(self.release_dir, "dist")
@property
- def docker_tag(self):
+ def docker_tag(self) -> str:
if self.branch == "master":
t = "dev"
else:
t = self.version
return "mitmproxy/mitmproxy:{}".format(t)
- def dump_info(self, fp=sys.stdout):
+ def dump_info(self, fp=sys.stdout) -> None:
lst = [
"version",
"tag",
@@ -176,7 +197,9 @@ class BuildEnviron:
"upload_dir",
"should_build_wheel",
"should_build_pyinstaller",
+ "should_build_wininstaller",
"should_build_docker",
+ "should_upload_aws",
"should_upload_docker",
"should_upload_pypi",
]
@@ -232,6 +255,8 @@ class BuildEnviron:
@property
def is_pull_request(self) -> bool:
+ if self.github_event_name == "pull_request":
+ return True
if self.appveyor_pull_request_number:
return True
if self.travis_pull_request and self.travis_pull_request != "false":
@@ -239,13 +264,13 @@ class BuildEnviron:
return False
@property
- def platform_tag(self):
+ def platform_tag(self) -> str:
if self.system in self.PLATFORM_TAGS:
return self.PLATFORM_TAGS[self.system]
raise BuildError("Unsupported platform: %s" % self.system)
@property
- def release_dir(self):
+ def release_dir(self) -> str:
return os.path.join(self.root_dir, "release")
@property
@@ -256,6 +281,13 @@ class BuildEnviron:
self.has_docker_creds,
])
+ @property
+ def should_upload_aws(self) -> bool:
+ return all([
+ self.has_aws_creds,
+ (self.should_build_wheel or self.should_build_pyinstaller or self.should_build_wininstaller),
+ ])
+
@property
def should_upload_pypi(self) -> bool:
return all([
@@ -265,18 +297,24 @@ class BuildEnviron:
])
@property
- def tag(self):
- return self.travis_tag or self.appveyor_repo_tag_name
+ def tag(self) -> str:
+ if self.travis_tag:
+ return self.travis_tag
+ if self.appveyor_repo_tag_name:
+ return self.appveyor_repo_tag_name
+ if self.github_ref and self.github_ref.startswith("refs/tags/"):
+ return self.github_ref.replace("refs/tags/", "")
+ return ""
@property
- def upload_dir(self):
+ def upload_dir(self) -> str:
if self.tag:
return self.version
else:
return "branches/%s" % self.version
@property
- def version(self):
+ def version(self) -> str:
if self.tag:
if self.tag.startswith("v"):
try:
@@ -300,13 +338,14 @@ def build_wheel(be: BuildEnviron): # pragma: no cover
"bdist_wheel",
"--dist-dir", be.dist_dir,
])
- whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
+ whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))
click.echo("Found wheel package: {}".format(whl))
subprocess.check_call(["tox", "-e", "wheeltest", "--", whl])
return whl
-def build_docker_image(be: BuildEnviron, whl: str): # pragma: no cover
+def build_docker_image(be: BuildEnviron): # pragma: no cover
+ whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))
click.echo("Building Docker images...")
subprocess.check_call([
"docker",
@@ -410,22 +449,25 @@ def build_pyinstaller(be: BuildEnviron): # pragma: no cover
def build_wininstaller(be: BuildEnviron): # pragma: no cover
+ if not be.build_key:
+ click.echo("Cannot build windows installer without secret key.")
+ return
click.echo("Building wininstaller package...")
- IB_VERSION = "18.8.0"
+ IB_VERSION = "19.10.0"
IB_DIR = pathlib.Path(be.release_dir) / "installbuilder"
IB_SETUP = IB_DIR / "setup" / f"{IB_VERSION}-installer.exe"
IB_CLI = fr"C:\Program Files (x86)\BitRock InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe"
IB_LICENSE = IB_DIR / "license.xml"
- if True or not os.path.isfile(IB_CLI):
+ if not os.path.isfile(IB_CLI):
if not os.path.isfile(IB_SETUP):
click.echo("Downloading InstallBuilder...")
def report(block, blocksize, total):
done = block * blocksize
if round(100 * done / total) != round(100 * (done - blocksize) / total):
- click.secho(f"Downloading... {round(100*done/total)}%")
+ click.secho(f"Downloading... {round(100 * done / total)}%")
urllib.request.urlretrieve(
f"https://clients.bitrock.com/installbuilder/installbuilder-enterprise-{IB_VERSION}-windows-installer.exe",
@@ -435,14 +477,13 @@ def build_wininstaller(be: BuildEnviron): # pragma: no cover
shutil.move(IB_SETUP.with_suffix(".tmp"), IB_SETUP)
click.echo("Install InstallBuilder...")
- subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"],
- check=True)
+ subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"], check=True)
assert os.path.isfile(IB_CLI)
click.echo("Decrypt InstallBuilder license...")
- f = cryptography.fernet.Fernet(be.rtool_key.encode())
- with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, open(IB_LICENSE,
- "wb") as outfile:
+ f = cryptography.fernet.Fernet(be.build_key.encode())
+ with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, \
+ open(IB_LICENSE, "wb") as outfile:
outfile.write(f.decrypt(infile.read()))
click.echo("Run InstallBuilder...")
@@ -479,13 +520,12 @@ def build(): # pragma: no cover
os.makedirs(be.dist_dir, exist_ok=True)
if be.should_build_wheel:
- whl = build_wheel(be)
- # Docker image requires wheels
- if be.should_build_docker:
- build_docker_image(be, whl)
+ build_wheel(be)
+ if be.should_build_docker:
+ build_docker_image(be)
if be.should_build_pyinstaller:
build_pyinstaller(be)
- if be.should_build_wininstaller and be.rtool_key:
+ if be.should_build_wininstaller:
build_wininstaller(be)
@@ -499,12 +539,15 @@ def upload(): # pragma: no cover
Pushes the Docker image to Docker Hub.
"""
be = BuildEnviron.from_env()
+ be.dump_info()
if be.is_pull_request:
click.echo("Refusing to upload artifacts from a pull request!")
return
- if be.has_aws_creds:
+ if be.should_upload_aws:
+ num_files = len([name for name in os.listdir(be.dist_dir) if os.path.isfile(name)])
+ click.echo(f"Uploading {num_files} files to AWS dir {be.upload_dir}...")
subprocess.check_call([
"aws", "s3", "cp",
"--acl", "public-read",
--
cgit v1.2.3