cibuild: Consolidate build environment and add tests

2 files changed, 229 insertions, 151 deletions

diff --git a/release/__init__.py b/release/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/release/__init__.py
diff --git a/release/cibuild.py b/release/cibuild.py
index 3a79affe..566c4659 100755
--- a/release/cibuild.py
+++ b/release/cibuild.py
@@ -15,76 +15,6 @@ from os.path import join, abspath, dirname, exists, basename
 import click
 import cryptography.fernet
 
-# ZipFile and tarfile have slightly different APIs. Fix that.
-if platform.system() == "Windows":
-    def Archive(name):
-        a = zipfile.ZipFile(name, "w")
-        a.add = a.write
-        return a
-else:
-    def Archive(name):
-        return tarfile.open(name, "w:gz")
-
-PLATFORM_TAG = {
-    "Darwin": "osx",
-    "Windows": "windows",
-    "Linux": "linux",
-}.get(platform.system(), platform.system())
-
-ROOT_DIR = abspath(join(dirname(__file__), ".."))
-RELEASE_DIR = join(ROOT_DIR, "release")
-BUILD_DIR = join(RELEASE_DIR, "build")
-DIST_DIR = join(RELEASE_DIR, "dist")
-
-BDISTS = {
-    "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
-    "pathod": ["pathoc", "pathod"]
-}
-if platform.system() == "Windows":
-    BDISTS["mitmproxy"].remove("mitmproxy")
-
-TOOLS = [
-    tool
-    for tools in sorted(BDISTS.values())
-    for tool in tools
-]
-
-TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
-BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
-if TAG:
-    VERSION = re.sub('^v', '', TAG)
-    UPLOAD_DIR = VERSION
-elif BRANCH:
-    VERSION = re.sub('^v', '', BRANCH)
-    UPLOAD_DIR = "branches/%s" % VERSION
-else:
-    print("Could not establish build name - exiting." % BRANCH)
-    sys.exit(0)
-
-print("BUILD PLATFORM_TAG=%s" % PLATFORM_TAG)
-print("BUILD ROOT_DIR=%s" % ROOT_DIR)
-print("BUILD RELEASE_DIR=%s" % RELEASE_DIR)
-print("BUILD BUILD_DIR=%s" % BUILD_DIR)
-print("BUILD DIST_DIR=%s" % DIST_DIR)
-print("BUILD BDISTS=%s" % BDISTS)
-print("BUILD TAG=%s" % TAG)
-print("BUILD BRANCH=%s" % BRANCH)
-print("BUILD VERSION=%s" % VERSION)
-print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR)
-
-
-def archive_name(bdist: str) -> str:
-    if platform.system() == "Windows":
-        ext = "zip"
-    else:
-        ext = "tar.gz"
-    return "{project}-{version}-{platform}.{ext}".format(
-        project=bdist,
-        version=VERSION,
-        platform=PLATFORM_TAG,
-        ext=ext
-    )
-
 
 @contextlib.contextmanager
 def chdir(path: str):
@@ -94,79 +24,216 @@ def chdir(path: str):
     os.chdir(old_dir)
 
 
-@click.group(chain=True)
-def cli():
-    """
-    mitmproxy build tool
-    """
+class BuildError(Exception):
     pass
 
 
-@cli.command("build")
-def build():
-    """
-    Build a binary distribution
-    """
-    os.makedirs(DIST_DIR, exist_ok=True)
-
-    if "WHEEL" in os.environ:
-        whl = build_wheel()
-    else:
-        click.echo("Not building wheels.")
-
-    if "WHEEL" in os.environ and "DOCKER" in os.environ:
-        # Docker image requires wheels
-        build_docker_image(whl)
-    else:
-        click.echo("Not building Docker image.")
-
-    if "PYINSTALLER" in os.environ:
-        build_pyinstaller()
-    else:
-        click.echo("Not building PyInstaller packages.")
-
-
-def build_wheel():
+class BuildEnviron:
+    PLATFORM_TAGS = {
+        "Darwin": "osx",
+        "Windows": "windows",
+        "Linux": "linux",
+    }
+
+    def __init__(
+        self,
+        *,
+        system = "",
+        root_dir = "",
+
+        travis_tag = "",
+        travis_branch = "",
+        travis_pull_request = "",
+
+        appveyor_repo_tag_name = "",
+        appveyor_repo_branch = "",
+        appveyor_pull_request_number = "",
+
+        build_wheel = "",
+        build_docker = "",
+        build_pyinstaller = "",
+
+        has_aws_creds = False,
+        has_twine_creds = False,
+
+        docker_username = "",
+        docker_password = "",
+    ):
+        self.system = system
+        self.root_dir = root_dir
+
+        self.travis_tag = travis_tag
+        self.travis_branch = travis_branch
+        self.travis_pull_request = travis_pull_request
+
+        self.appveyor_repo_tag_name = appveyor_repo_tag_name
+        self.appveyor_repo_branch = appveyor_repo_branch
+        self.appveyor_pull_request_number = appveyor_pull_request_number
+
+        self.has_aws_creds = has_aws_creds
+        self.has_twine_creds = has_twine_creds
+        self.docker_username = docker_username
+        self.docker_password = docker_password
+
+    @classmethod
+    def from_env(klass):
+        return klass(
+            system = platform.system,
+            root_dir = dirname(__file__),
+
+            travis_tag = os.environ.get("TRAVIS_TAG", ""),
+            travis_branch = os.environ.get("TRAVIS_BRANCH", ""),
+            travis_pull_request = os.environ.get("TRAVIS_PULL_REQUEST"),
+
+            appveyor_repo_tag_name = os.environ.get("APPVEYOR_REPO_TAG_NAME", ""),
+            appveyor_repo_branch = os.environ.get("APPVEYOR_REPO_BRANCH", ""),
+            appveyor_pull_request_number = os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"),
+
+            build_wheel = "WHEEL" in os.environ,
+            build_pyinstaller = "PYINSTALLER" in os.environ,
+            build_docker = "DOCKER" in os.environ,
+
+            has_aws_creds = "AWS_ACCESS_KEY_ID" in os.environ,
+            has_twine_creds= (
+                "TWINE_USERNAME" in os.environ and
+                "TWINE_PASSWORD" in os.environ
+            ),
+
+            docker_username = os.environ.get("DOCKER_USERNAME"),
+            docker_password = os.environ.get("DOCKER_PASSWORD"),
+        )
+
+    @property
+    def has_docker_creds(self) -> bool:
+        return self.docker_username and self.docker_password
+
+    @property
+    def is_pull_request(self) -> bool:
+        if self.appveyor_pull_request_number:
+            return True
+        if self.travis_pull_request and self.travis_pull_request != "false":
+            return True
+        return False
+
+    @property
+    def tag(self):
+        return self.travis_tag or self.appveyor_repo_tag_name
+
+    @property
+    def branch(self):
+        return self.travis_branch or self.appveyor_repo_branch
+
+    @property
+    def version(self):
+        name = self.tag or self.branch
+        if not name:
+            raise BuildError("Could not establish build name")
+        return re.sub('^v', "", name)
+
+    @property
+    def upload_dir(self):
+        if self.tag:
+            return self.version
+        else:
+            return "branches/%s" % self.version
+
+    @property
+    def platform_tag(self):
+        if self.system in self.PLATFORM_TAGS:
+            return self.PLATFORM_TAGS[self.system]
+        raise BuildError("Unsupported platform: %s" % self.system)
+
+    @property
+    def release_dir(self):
+        return os.path.join(self.root_dir, "release")
+
+    @property
+    def build_dir(self):
+        return os.path.join(self.release_dir, "build")
+
+    @property
+    def dist_dir(self):
+        return os.path.join(self.release_dir, "dist")
+
+    def archive(self, name):
+        # ZipFile and tarfile have slightly different APIs. Fix that.
+        if self.system == "Windows":
+            a = zipfile.ZipFile(name, "w")
+            a.add = a.write
+            return a
+        else:
+            return tarfile.open(name, "w:gz")
+
+    def archive_name(self, bdist: str) -> str:
+        if self.system == "Windows":
+            ext = "zip"
+        else:
+            ext = "tar.gz"
+        return "{project}-{version}-{platform}.{ext}".format(
+            project=bdist,
+            version=self.version,
+            platform=self.platform_tag,
+            ext=ext
+        )
+
+    @property
+    def bdists(self):
+        ret = {
+            "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
+            "pathod": ["pathoc", "pathod"]
+        }
+        if self.system == "Windows":
+            ret["mitmproxy"].remove("mitmproxy")
+        return ret
+
+    def dump_info(self, fp=sys.stdout):
+        print("BUILD PLATFORM_TAG=%s" % self.platform_tag, file=fp)
+        print("BUILD ROOT_DIR=%s" % self.root_dir, file=fp)
+        print("BUILD RELEASE_DIR=%s" % self.release_dir, file=fp)
+        print("BUILD BUILD_DIR=%s" % self.build_dir, file=fp)
+        print("BUILD DIST_DIR=%s" % self.dist_dir, file=fp)
+        print("BUILD BDISTS=%s" % self.bdists, file=fp)
+        print("BUILD TAG=%s" % self.tag, file=fp)
+        print("BUILD BRANCH=%s" % self.branch, file=fp)
+        print("BUILD VERSION=%s" % self.version, file=fp)
+        print("BUILD UPLOAD_DIR=%s" % self.upload_dir, file=fp)
+
+
+def build_wheel(be: BuildEnviron):
     click.echo("Building wheel...")
     subprocess.check_call([
         "python",
         "setup.py",
         "-q",
         "bdist_wheel",
-        "--dist-dir", DIST_DIR,
+        "--dist-dir", be.dist_dir,
     ])
-
-    whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
+    whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
     click.echo("Found wheel package: {}".format(whl))
-
-    subprocess.check_call([
-        "tox",
-        "-e", "wheeltest",
-        "--",
-        whl
-    ])
-
+    subprocess.check_call(["tox", "-e", "wheeltest", "--", whl])
     return whl
 
 
-def build_docker_image(whl):
+def build_docker_image(be: BuildEnviron, whl: str):
     click.echo("Building Docker image...")
     subprocess.check_call([
         "docker",
         "build",
-        "--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, ROOT_DIR)),
+        "--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, be.root_dir)),
         "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(basename(whl)),
         "--file", "docker/Dockerfile",
         "."
     ])
 
 
-def build_pyinstaller():
-    PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
+def build_pyinstaller(be: BuildEnviron):
+    click.echo("Building pyinstaller package...")
+
+    PYINSTALLER_SPEC = join(be.release_dir, "specs")
     # PyInstaller 3.2 does not bundle pydivert's Windivert binaries
-    PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
-    PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
-    PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
+    PYINSTALLER_HOOKS = join(be.release_dir, "hooks")
+    PYINSTALLER_TEMP = join(be.build_dir, "pyinstaller")
+    PYINSTALLER_DIST = join(be.build_dir, "binaries", be.platform_tag)
 
     # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
     # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
@@ -183,8 +250,8 @@ def build_pyinstaller():
     if exists(PYINSTALLER_DIST):
         shutil.rmtree(PYINSTALLER_DIST)
 
-    for bdist, tools in sorted(BDISTS.items()):
-        with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
+    for bdist, tools in sorted(be.bdists.items()):
+        with be.archive(join(be.dist_dir, be.archive_name(bdist))) as archive:
             for tool in tools:
                 # We can't have a folder and a file with the same name.
                 if tool == "mitmproxy":
@@ -237,7 +304,34 @@ def build_pyinstaller():
                 click.echo(subprocess.check_output([executable, "--version"]).decode())
 
                 archive.add(executable, basename(executable))
-        click.echo("Packed {}.".format(archive_name(bdist)))
+        click.echo("Packed {}.".format(be.archive_name(bdist)))
+
+
+@click.group(chain=True)
+def cli():
+    """
+    mitmproxy build tool
+    """
+    pass
+
+
+@cli.command("build")
+def build():
+    """
+        Build a binary distribution
+    """
+    be = BuildEnviron.from_env()
+    be.dump_info()
+
+    os.makedirs(be.dist_dir, exist_ok=True)
+
+    if be.build_wheel:
+        whl = build_wheel(be)
+        # Docker image requires wheels
+        if be.build_docker:
+            build_docker_image(whl)
+    if be.build_pyinstaller:
+        build_pyinstaller()
 
 
 @cli.command("upload")
@@ -249,57 +343,41 @@ def upload():
         Uploads the Pyinstaller and wheels packages to the snapshot server.
         Pushes the Docker image to Docker Hub.
     """
+    be = BuildEnviron.from_env()
 
-    # Our credentials are only available from within the main repository and not forks.
-    # We need to prevent uploads from all BUT the branches in the main repository.
-    # Pull requests and master-branches of forks are not allowed to upload.
-    is_pull_request = (
-        ("TRAVIS_PULL_REQUEST" in os.environ and os.environ["TRAVIS_PULL_REQUEST"] != "false") or
-        "APPVEYOR_PULL_REQUEST_NUMBER" in os.environ
-    )
-    if is_pull_request:
+    if be.is_pull_request:
         click.echo("Refusing to upload artifacts from a pull request!")
         return
 
-    if "AWS_ACCESS_KEY_ID" in os.environ:
+    if be.has_aws_creds:
         subprocess.check_call([
             "aws", "s3", "cp",
             "--acl", "public-read",
-            DIST_DIR + "/",
-            "s3://snapshots.mitmproxy.org/{}/".format(UPLOAD_DIR),
+            be.dist_dir + "/",
+            "s3://snapshots.mitmproxy.org/{}/".format(be.upload_dir),
             "--recursive",
         ])
 
-    upload_pypi = (
-        TAG and
-        "WHEEL" in os.environ and
-        "TWINE_USERNAME" in os.environ and
-        "TWINE_PASSWORD" in os.environ
-    )
+    upload_pypi = (be.tag and be.build_wheel and be.has_twine_creds)
     if upload_pypi:
-        whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
+        whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
         click.echo("Uploading {} to PyPi...".format(whl))
-        subprocess.check_call([
-            "twine",
-            "upload",
-            whl
-        ])
+        subprocess.check_call(["twine", "upload", whl])
 
     upload_docker = (
-        (TAG or BRANCH == "master") and
-        "DOCKER" in os.environ and
-        "DOCKER_USERNAME" in os.environ and
-        "DOCKER_PASSWORD" in os.environ
+        (be.tag or be.branch == "master") and
+        be.build_docker,
+        be.has_docker_creds,
     )
     if upload_docker:
-        docker_tag = "dev" if BRANCH == "master" else VERSION
+        docker_tag = "dev" if be.branch == "master" else be.version
 
         click.echo("Uploading Docker image to tag={}...".format(docker_tag))
         subprocess.check_call([
             "docker",
             "login",
-            "-u", os.environ["DOCKER_USERNAME"],
-            "-p", os.environ["DOCKER_PASSWORD"],
+            "-u", be.docker_username,
+            "-p", be.docker_password,
         ])
         subprocess.check_call([
             "docker",