diff options
| -rw-r--r-- | .jenkins/Jenkinsfile-cryptography-wheel-builder | 118 | ||||
| -rw-r--r-- | dev-requirements.txt | 2 | ||||
| -rw-r--r-- | release.py | 109 |
3 files changed, 90 insertions, 139 deletions
diff --git a/.jenkins/Jenkinsfile-cryptography-wheel-builder b/.jenkins/Jenkinsfile-cryptography-wheel-builder index 72158f3c..907f06e9 100644 --- a/.jenkins/Jenkinsfile-cryptography-wheel-builder +++ b/.jenkins/Jenkinsfile-cryptography-wheel-builder @@ -14,30 +14,6 @@ def configs = [ label: 'windows64', versions: ['py27', 'py34', 'py35', 'py36', 'py37'], ], - [ - label: 'sierra', - // The py3x version listed here corresponds to the minimum ABI version - // the wheels will support. e.g. py34 supports py34+ - versions: ['py27', 'py34'], - ], - [ - label: 'docker', - imageName: 'pyca/cryptography-manylinux1:i686', - // The py3x version listed here corresponds to the minimum ABI version - // the wheels will support. e.g. cp34-cp34m supports py34+ - versions: [ - 'cp27-cp27m', 'cp27-cp27mu', 'cp34-cp34m', - ], - ], - [ - label: 'docker', - imageName: 'pyca/cryptography-manylinux1:x86_64', - // The py3x version listed here corresponds to the minimum ABI version - // the wheels will support. e.g. cp34-cp34m supports py34+ - versions: [ - 'cp27-cp27m', 'cp27-cp27mu', 'cp34-cp34m', - ], - ], ] @@ -89,73 +65,6 @@ def build(version, label, imageName) { pip install -f wheelhouse cryptography --no-index python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))" """ - } else if (label.contains("sierra")) { - def pythonPath = [ - py27: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7", - py34: "/Library/Frameworks/Python.framework/Versions/3.4/bin/python3.4", - ] - ansiColor { - sh """#!/usr/bin/env bash - set -xe - # output the list of things we've installed as a point in time check of how up - # to date the builder is - /usr/sbin/system_profiler SPInstallHistoryDataType - - # Jenkins logs in as a non-interactive shell, so we don't even have /usr/local/bin in PATH - export PATH="/usr/local/bin:\${PATH}" - export PATH="/Users/jenkins/.pyenv/shims:\${PATH}" - - printenv - - virtualenv .venv -p ${pythonPath[version]} - source .venv/bin/activate - pip install -U wheel # upgrade wheel to latest before we use it to build the wheel - pip install cffi six idna asn1crypto ipaddress enum34 - REGEX="py3([0-9])*" - if [[ "${version}" =~ \$REGEX ]]; then - PY_LIMITED_API="--build-option --py-limited-api=cp3\${BASH_REMATCH[1]}" - fi - CRYPTOGRAPHY_SUPPRESS_LINK_FLAGS="1" \ - LDFLAGS="/usr/local/opt/openssl@1.1/lib/libcrypto.a /usr/local/opt/openssl@1.1/lib/libssl.a" \ - CFLAGS="-I/usr/local/opt/openssl@1.1/include -mmacosx-version-min=10.9" \ - pip wheel cryptography==$BUILD_VERSION --no-use-pep517 --wheel-dir=wheelhouse --no-binary cryptography --no-deps \$PY_LIMITED_API - pip install -f wheelhouse cryptography --no-index - python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))" - otool -L `find .venv -name '_openssl*.so'` - lipo -info `find .venv -name '*.so'` - otool -L `find .venv -name '_openssl*.so'` | grep -vG "libcrypto\\|libssl" - """ - } - } else if (label.contains("docker")) { - linux32 = "" - if (imageName.contains("i686")) { - linux32 = "linux32" - } - sh """#!/usr/bin/env bash - set -x -e - - $linux32 /opt/python/$version/bin/pip install cffi six idna asn1crypto ipaddress enum34 - # Because we are doing this as root in the container, but we write to a mounted dir that is outside the container - # we need to make sure we set these files writable such that the jenkins user can delete them afterwards - mkdir -p tmpwheelhouse - mkdir -p wheelhouse - chmod -R 777 tmpwheelhouse - chmod -R 777 wheelhouse - - REGEX="cp3([0-9])*" - if [[ "${version}" =~ \$REGEX ]]; then - PY_LIMITED_API="--build-option --py-limited-api=cp3\${BASH_REMATCH[1]}" - fi - LDFLAGS="-L/opt/pyca/cryptography/openssl/lib" \ - CFLAGS="-I/opt/pyca/cryptography/openssl/include -Wl,--exclude-libs,ALL" \ - $linux32 /opt/python/$version/bin/pip wheel cryptography==$BUILD_VERSION --no-use-pep517 --no-binary cryptography --no-deps --wheel-dir=tmpwheelhouse \$PY_LIMITED_API - $linux32 auditwheel repair tmpwheelhouse/cryptography*.whl -w wheelhouse/ - unzip wheelhouse/*.whl -d execstack.check - chmod -R 777 execstack.check - (execstack execstack.check/cryptography/hazmat/bindings/*.so | grep '^X') && exit 1 - $linux32 /opt/python/$version/bin/pip install cryptography==$BUILD_VERSION --no-index -f wheelhouse/ - $linux32 /opt/python/$version/bin/python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))" - """ } archiveArtifacts artifacts: "wheelhouse/cryptography*.whl" } @@ -172,28 +81,11 @@ for (config in configs) { for (_version in versions) { def version = _version - - if (label.contains("docker")) { - def imageName = config["imageName"] - def combinedName = "${imageName}-${version}" - builders[combinedName] = { - node(label) { - stage(combinedName) { - def buildImage = docker.image(imageName) - buildImage.pull() - buildImage.inside("-u root") { - build(version, label, imageName) - } - } - } - } - } else { - def combinedName = "${label}-${version}" - builders[combinedName] = { - node(label) { - stage(combinedName) { - build(version, label, "") - } + def combinedName = "${label}-${version}" + builders[combinedName] = { + node(label) { + stage(combinedName) { + build(version, label, "") } } } diff --git a/dev-requirements.txt b/dev-requirements.txt index 58827ed4..60819739 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ +azure-devops click -clint coverage requests tox >= 2.4.1 @@ -7,13 +7,19 @@ from __future__ import absolute_import, division, print_function import getpass import glob import io +import json import os import subprocess +import tempfile import time +import zipfile + +from azure.devops.connection import Connection +from azure.devops.v5_1.build.models import Build import click -from clint.textui.progress import Bar as ProgressBar +from msrest.authentication import BasicAuthentication import requests @@ -29,7 +35,63 @@ def run(*args, **kwargs): subprocess.check_call(list(args), **kwargs) -def wait_for_build_completed(session): +def wait_for_build_completed_azure(build_client, build_id): + while True: + build = build_client.get_build("cryptography", build_id) + if build.finish_time is not None: + break + time.sleep(3) + + +def download_artifacts_azure(build_client, build_id): + artifacts = build_client.get_artifacts("cryptography", build_id) + paths = [] + for artifact in artifacts: + contents = build_client.get_artifact_content_zip( + "cryptography", build_id, artifact.name + ) + with tempfile.NamedTemporaryFile() as f: + for chunk in contents: + f.write(chunk) + f.flush() + with zipfile.ZipFile(f.name) as z: + for name in z.namelist(): + if not name.endswith(".whl"): + continue + p = z.open(name) + out_path = os.path.join( + os.path.dirname(__file__), + "dist", + os.path.basename(name), + ) + with open(out_path, "wb") as f: + f.write(p.read()) + paths.append(out_path) + return paths + + +def build_wheels_azure(version): + token = getpass.getpass("Azure personal access token: ") + credentials = BasicAuthentication("", token) + connection = Connection( + base_url="https://dev.azure.com/pyca", creds=credentials + ) + build_client = connection.clients.get_build_client() + [definition] = build_client.get_definitions( + "cryptography", "wheel builder" + ) + build_description = Build( + definition=definition, + parameters=json.dumps({"BUILD_VERSION": version}), + ) + build = build_client.queue_build( + project="cryptography", build=build_description + ) + wait_for_build_completed_azure(build_client, build.id) + return download_artifacts_azure(build_client, build.id) + + +def wait_for_build_completed_jenkins(session): # Wait 20 seconds before actually checking if the build is complete, to # ensure that it had time to really start. time.sleep(20) @@ -47,7 +109,7 @@ def wait_for_build_completed(session): time.sleep(0.1) -def download_artifacts(session): +def download_artifacts_jenkins(session): response = session.get( "{0}/lastBuild/api/json/".format(JENKINS_URL), headers={ @@ -69,16 +131,9 @@ def download_artifacts(session): ) assert response.headers["content-length"] print("Downloading {0}".format(artifact["fileName"])) - bar = ProgressBar( - expected_size=int(response.headers["content-length"]), - filled_char="=" - ) content = io.BytesIO() for data in response.iter_content(chunk_size=8192): content.write(data) - bar.show(content.tell()) - assert bar.expected_size == content.tell() - bar.done() out_path = os.path.join( os.path.dirname(__file__), "dist", @@ -90,6 +145,22 @@ def download_artifacts(session): return paths +def build_wheels_jenkins(version): + token = getpass.getpass("Input the Jenkins token: ") + session = requests.Session() + response = session.get( + "{0}/buildWithParameters".format(JENKINS_URL), + params={ + "token": token, + "BUILD_VERSION": version, + "cause": "Building wheels for {0}".format(version) + } + ) + response.raise_for_status() + wait_for_build_completed_jenkins(session) + return download_artifacts_jenkins(session) + + @click.command() @click.argument("version") def release(version): @@ -108,21 +179,9 @@ def release(version): ) run("twine", "upload", "-s", *packages) - session = requests.Session() - - token = getpass.getpass("Input the Jenkins token: ") - response = session.get( - "{0}/buildWithParameters".format(JENKINS_URL), - params={ - "token": token, - "BUILD_VERSION": version, - "cause": "Building wheels for {0}".format(version) - } - ) - response.raise_for_status() - wait_for_build_completed(session) - paths = download_artifacts(session) - run("twine", "upload", *paths) + azure_wheel_paths = build_wheels_azure(version) + jenkins_wheel_paths = build_wheels_jenkins(version) + run("twine", "upload", *(azure_wheel_paths + jenkins_wheel_paths)) if __name__ == "__main__": |