Update release.py to use azure for wheel building (#4878)

* Initial stab at this script

* Convert to the old style artifact publish

* Update script based on some testing

* Remove this

* Adapt release.py to combine azure and jenkins wheels

3 files changed, 90 insertions, 139 deletions

diff --git a/.jenkins/Jenkinsfile-cryptography-wheel-builder b/.jenkins/Jenkinsfile-cryptography-wheel-builder
index 72158f3c..907f06e9 100644
--- a/.jenkins/Jenkinsfile-cryptography-wheel-builder
+++ b/.jenkins/Jenkinsfile-cryptography-wheel-builder
@@ -14,30 +14,6 @@ def configs = [
         label: 'windows64',
         versions: ['py27', 'py34', 'py35', 'py36', 'py37'],
     ],
-    [
-        label: 'sierra',
-        // The py3x version listed here corresponds to the minimum ABI version
-        // the wheels will support. e.g. py34 supports py34+
-        versions: ['py27', 'py34'],
-    ],
-    [
-        label: 'docker',
-        imageName: 'pyca/cryptography-manylinux1:i686',
-        // The py3x version listed here corresponds to the minimum ABI version
-        // the wheels will support. e.g. cp34-cp34m supports py34+
-        versions: [
-            'cp27-cp27m', 'cp27-cp27mu', 'cp34-cp34m',
-        ],
-    ],
-    [
-        label: 'docker',
-        imageName: 'pyca/cryptography-manylinux1:x86_64',
-        // The py3x version listed here corresponds to the minimum ABI version
-        // the wheels will support. e.g. cp34-cp34m supports py34+
-        versions: [
-            'cp27-cp27m', 'cp27-cp27mu', 'cp34-cp34m',
-        ],
-    ],
 ]
 
 
@@ -89,73 +65,6 @@ def build(version, label, imageName) {
                     pip install -f wheelhouse cryptography --no-index
                     python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))"
                 """
-            } else if (label.contains("sierra")) {
-                def pythonPath = [
-                    py27: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7",
-                    py34: "/Library/Frameworks/Python.framework/Versions/3.4/bin/python3.4",
-                ]
-                ansiColor {
-                    sh """#!/usr/bin/env bash
-                        set -xe
-                        # output the list of things we've installed as a point in time check of how up
-                        # to date the builder is
-                        /usr/sbin/system_profiler SPInstallHistoryDataType
-
-                        # Jenkins logs in as a non-interactive shell, so we don't even have /usr/local/bin in PATH
-                        export PATH="/usr/local/bin:\${PATH}"
-                        export PATH="/Users/jenkins/.pyenv/shims:\${PATH}"
-
-                        printenv
-
-                        virtualenv .venv -p ${pythonPath[version]}
-                        source .venv/bin/activate
-                        pip install -U wheel # upgrade wheel to latest before we use it to build the wheel
-                        pip install cffi six idna asn1crypto ipaddress enum34
-                        REGEX="py3([0-9])*"
-                        if [[ "${version}" =~ \$REGEX ]]; then
-                            PY_LIMITED_API="--build-option --py-limited-api=cp3\${BASH_REMATCH[1]}"
-                        fi
-                        CRYPTOGRAPHY_SUPPRESS_LINK_FLAGS="1" \
-                            LDFLAGS="/usr/local/opt/openssl@1.1/lib/libcrypto.a /usr/local/opt/openssl@1.1/lib/libssl.a" \
-                            CFLAGS="-I/usr/local/opt/openssl@1.1/include -mmacosx-version-min=10.9" \
-                            pip wheel cryptography==$BUILD_VERSION --no-use-pep517 --wheel-dir=wheelhouse --no-binary cryptography --no-deps \$PY_LIMITED_API
-                        pip install -f wheelhouse cryptography --no-index
-                        python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))"
-                        otool -L `find .venv -name '_openssl*.so'`
-                        lipo -info `find .venv -name '*.so'`
-                        otool -L `find .venv -name '_openssl*.so'` | grep -vG "libcrypto\\|libssl"
-                    """
-                }
-            } else if (label.contains("docker")) {
-                linux32 = ""
-                if (imageName.contains("i686")) {
-                    linux32 = "linux32"
-                }
-                sh """#!/usr/bin/env bash
-                    set -x -e
-
-                    $linux32 /opt/python/$version/bin/pip install cffi six idna asn1crypto ipaddress enum34
-                    # Because we are doing this as root in the container, but we write to a mounted dir that is outside the container
-                    # we need to make sure we set these files writable such that the jenkins user can delete them afterwards
-                    mkdir -p tmpwheelhouse
-                    mkdir -p wheelhouse
-                    chmod -R 777 tmpwheelhouse
-                    chmod -R 777 wheelhouse
-
-                    REGEX="cp3([0-9])*"
-                    if [[ "${version}" =~ \$REGEX ]]; then
-                        PY_LIMITED_API="--build-option --py-limited-api=cp3\${BASH_REMATCH[1]}"
-                    fi
-                    LDFLAGS="-L/opt/pyca/cryptography/openssl/lib" \
-                        CFLAGS="-I/opt/pyca/cryptography/openssl/include -Wl,--exclude-libs,ALL" \
-                        $linux32 /opt/python/$version/bin/pip wheel cryptography==$BUILD_VERSION --no-use-pep517 --no-binary cryptography --no-deps --wheel-dir=tmpwheelhouse \$PY_LIMITED_API
-                    $linux32 auditwheel repair tmpwheelhouse/cryptography*.whl -w wheelhouse/
-                    unzip wheelhouse/*.whl -d execstack.check
-                    chmod -R 777 execstack.check
-                    (execstack execstack.check/cryptography/hazmat/bindings/*.so | grep '^X') && exit 1
-                    $linux32 /opt/python/$version/bin/pip install cryptography==$BUILD_VERSION --no-index -f wheelhouse/
-                    $linux32 /opt/python/$version/bin/python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))"
-                """
             }
             archiveArtifacts artifacts: "wheelhouse/cryptography*.whl"
         }
@@ -172,28 +81,11 @@ for (config in configs) {
 
     for (_version in versions) {
         def version = _version
-
-        if (label.contains("docker")) {
-            def imageName = config["imageName"]
-            def combinedName = "${imageName}-${version}"
-            builders[combinedName] = {
-                node(label) {
-                    stage(combinedName) {
-                        def buildImage = docker.image(imageName)
-                        buildImage.pull()
-                        buildImage.inside("-u root") {
-                            build(version, label, imageName)
-                        }
-                    }
-                }
-            }
-        } else {
-            def combinedName = "${label}-${version}"
-            builders[combinedName] = {
-                node(label) {
-                    stage(combinedName) {
-                        build(version, label, "")
-                    }
+        def combinedName = "${label}-${version}"
+        builders[combinedName] = {
+            node(label) {
+                stage(combinedName) {
+                    build(version, label, "")
                 }
             }
         }
diff --git a/dev-requirements.txt b/dev-requirements.txt
index 58827ed4..60819739 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,5 +1,5 @@
+azure-devops
 click
-clint
 coverage
 requests
 tox >= 2.4.1
diff --git a/release.py b/release.py
index d7c18d10..b8269114 100644
--- a/release.py
+++ b/release.py
@@ -7,13 +7,19 @@ from __future__ import absolute_import, division, print_function
 import getpass
 import glob
 import io
+import json
 import os
 import subprocess
+import tempfile
 import time
+import zipfile
+
+from azure.devops.connection import Connection
+from azure.devops.v5_1.build.models import Build
 
 import click
 
-from clint.textui.progress import Bar as ProgressBar
+from msrest.authentication import BasicAuthentication
 
 import requests
 
@@ -29,7 +35,63 @@ def run(*args, **kwargs):
     subprocess.check_call(list(args), **kwargs)
 
 
-def wait_for_build_completed(session):
+def wait_for_build_completed_azure(build_client, build_id):
+    while True:
+        build = build_client.get_build("cryptography", build_id)
+        if build.finish_time is not None:
+            break
+        time.sleep(3)
+
+
+def download_artifacts_azure(build_client, build_id):
+    artifacts = build_client.get_artifacts("cryptography", build_id)
+    paths = []
+    for artifact in artifacts:
+        contents = build_client.get_artifact_content_zip(
+            "cryptography", build_id, artifact.name
+        )
+        with tempfile.NamedTemporaryFile() as f:
+            for chunk in contents:
+                f.write(chunk)
+            f.flush()
+            with zipfile.ZipFile(f.name) as z:
+                for name in z.namelist():
+                    if not name.endswith(".whl"):
+                        continue
+                    p = z.open(name)
+                    out_path = os.path.join(
+                        os.path.dirname(__file__),
+                        "dist",
+                        os.path.basename(name),
+                    )
+                    with open(out_path, "wb") as f:
+                        f.write(p.read())
+                    paths.append(out_path)
+    return paths
+
+
+def build_wheels_azure(version):
+    token = getpass.getpass("Azure personal access token: ")
+    credentials = BasicAuthentication("", token)
+    connection = Connection(
+        base_url="https://dev.azure.com/pyca", creds=credentials
+    )
+    build_client = connection.clients.get_build_client()
+    [definition] = build_client.get_definitions(
+        "cryptography", "wheel builder"
+    )
+    build_description = Build(
+        definition=definition,
+        parameters=json.dumps({"BUILD_VERSION": version}),
+    )
+    build = build_client.queue_build(
+        project="cryptography", build=build_description
+    )
+    wait_for_build_completed_azure(build_client, build.id)
+    return download_artifacts_azure(build_client, build.id)
+
+
+def wait_for_build_completed_jenkins(session):
     # Wait 20 seconds before actually checking if the build is complete, to
     # ensure that it had time to really start.
     time.sleep(20)
@@ -47,7 +109,7 @@ def wait_for_build_completed(session):
         time.sleep(0.1)
 
 
-def download_artifacts(session):
+def download_artifacts_jenkins(session):
     response = session.get(
         "{0}/lastBuild/api/json/".format(JENKINS_URL),
         headers={
@@ -69,16 +131,9 @@ def download_artifacts(session):
         )
         assert response.headers["content-length"]
         print("Downloading {0}".format(artifact["fileName"]))
-        bar = ProgressBar(
-            expected_size=int(response.headers["content-length"]),
-            filled_char="="
-        )
         content = io.BytesIO()
         for data in response.iter_content(chunk_size=8192):
             content.write(data)
-            bar.show(content.tell())
-        assert bar.expected_size == content.tell()
-        bar.done()
         out_path = os.path.join(
             os.path.dirname(__file__),
             "dist",
@@ -90,6 +145,22 @@ def download_artifacts(session):
     return paths
 
 
+def build_wheels_jenkins(version):
+    token = getpass.getpass("Input the Jenkins token: ")
+    session = requests.Session()
+    response = session.get(
+        "{0}/buildWithParameters".format(JENKINS_URL),
+        params={
+            "token": token,
+            "BUILD_VERSION": version,
+            "cause": "Building wheels for {0}".format(version)
+        }
+    )
+    response.raise_for_status()
+    wait_for_build_completed_jenkins(session)
+    return download_artifacts_jenkins(session)
+
+
 @click.command()
 @click.argument("version")
 def release(version):
@@ -108,21 +179,9 @@ def release(version):
     )
     run("twine", "upload", "-s", *packages)
 
-    session = requests.Session()
-
-    token = getpass.getpass("Input the Jenkins token: ")
-    response = session.get(
-        "{0}/buildWithParameters".format(JENKINS_URL),
-        params={
-            "token": token,
-            "BUILD_VERSION": version,
-            "cause": "Building wheels for {0}".format(version)
-        }
-    )
-    response.raise_for_status()
-    wait_for_build_completed(session)
-    paths = download_artifacts(session)
-    run("twine", "upload", *paths)
+    azure_wheel_paths = build_wheels_azure(version)
+    jenkins_wheel_paths = build_wheels_jenkins(version)
+    run("twine", "upload", *(azure_wheel_paths + jenkins_wheel_paths))
 
 
 if __name__ == "__main__":